def run(self):
# Setup SPDB instance
config = configuration.BossConfig()
kvio_config = {"cache_host": config['aws']['cache'],
"cache_db": config['aws']['cache-state-db'],
"read_timeout": 86400}
state_config = {"cache_state_host": config['aws']['cache-state'],
"cache_state_db": config['aws']['cache-db']}
object_store_config = {"s3_flush_queue": config['aws']["s3-flush-queue"],
"cuboid_bucket": config['aws']['cuboid_bucket'],
"page_in_lambda_function": config['lambda']['page_in_function'],
"page_out_lambda_function": config['lambda']['flush_function'],
"s3_index_table": config['aws']['s3-index-table'],
"id_index_table": config['aws']['id-index-table'],
"id_count_table": config['aws']['id-count-table']}
sp = SpatialDB(kvio_config,
state_config,
object_store_config)
while True:
self.log.info("Checking for delayed write operations.")
try:
self.process(sp)
time.sleep(5)
except Exception as err:
self.log.error("An error occurred running the process() method! \n {}".format(err))
def setUpClass(cls):
""" Create a diction of configuration values for the test resource. """
# Create resource
cls.setup_helper = SetupTests()
cls.data = cls.setup_helper.get_anno64_dict()
cls.resource = BossResourceBasic(cls.data)
# Load config
cls.config = configuration.BossConfig()
cls.object_store_config = {
"s3_flush_queue": 'https://mytestqueue.com',
"cuboid_bucket": "test_bucket",
"page_in_lambda_function": "page_in.test.boss",
"page_out_lambda_function": "page_out.test.boss",
"s3_index_table": "test_s3_table",
"id_index_table": "test_id_table",
"id_count_table": "test_count_table",
}
# Create AWS Resources needed for tests while mocking
cls.setup_helper.start_mocking()
cls.setup_helper.create_index_table(
cls.object_store_config["id_count_table"],
cls.setup_helper.ID_COUNT_SCHEMA)
cls.obj_ind = ObjectIndices(cls.object_store_config["s3_index_table"],
cls.object_store_config["id_index_table"],
cls.object_store_config["id_count_table"],
'us-east-1')
def setUpClass(cls):
"""Setup the redis client at the start of the test"""
cls.data = get_image_dict()
cls.resource = BossResourceBasic(cls.data)
cls.config = configuration.BossConfig()
cls.cache_client = redis.StrictRedis(host=cls.config["aws"]["cache"], port=6379, db=1,
decode_responses=False)
cls.config_data = {"cache_client": cls.cache_client, "read_timeout": 86400}
def get_test_configuration():
"""Method to get the integration test configuration info for spdb
Returns:
(dict, dict, dict): A tuple of dictionaries (kvio_config, state_config, object_store_config, s3_flush_queue_name)
"""
config = configuration.BossConfig()
# Get domain info
parts = config['aws']['cache'].split('.')
domain = "{}.{}".format(parts[1], parts[2])
# kvio settings
kvio_config = {
"cache_host": config['aws']['cache'],
"cache_db": 1,
"read_timeout": 86400
}
# state settings
state_config = {
"cache_state_host": config['aws']['cache-state'],
"cache_state_db": 1
}
_, domain = config['aws']['cuboid_bucket'].split('.', 1)
s3_flush_queue_name = "intTest.S3FlushQueue.{}".format(domain).replace(
'.', '-')
account_id = "{}".format(get_account_id())
account_id = account_id[:5]
object_store_config = {
"s3_flush_queue":
"https://queue.amazonaws.com/{}/{}".format(get_account_id(),
s3_flush_queue_name),
"cuboid_bucket":
"inttest.{}.{}".format(account_id, config['aws']['cuboid_bucket']),
"page_in_lambda_function":
config['lambda']['page_in_function'],
"page_out_lambda_function":
config['lambda']['flush_function'],
"s3_index_table":
"intTest.{}".format(config['aws']['s3-index-table']),
"id_index_table":
"intTest.{}".format(config['aws']['id-index-table']),
"id_count_table":
"intTest.{}".format(config['aws']['id-count-table'])
}
return kvio_config, state_config, object_store_config, s3_flush_queue_name
def __init__(self):
# Load boss config file
config = configuration.BossConfig()
# Set Properties
self.__sessions = queue.Queue()
if config['aws_mngr']['num_sessions'] == 'auto':
self.__num_sessions = multiprocessing.cpu_count()
else:
self.__num_sessions = config['aws_mngr']['num_sessions']
# Initialize the credentials and sessions
self.__init_sessions()
def setUpParams(self):
self.setup_helper = SetupTests()
# Don't use mock Amazon resources.
self.setup_helper.mock = False
self.data = self.setup_helper.get_image8_dict()
self.resource = BossResourceBasic(self.data)
self.config = configuration.BossConfig()
# kvio settings, 1 is the test DB.
self.kvio_config = {
"cache_host": self.config['aws']['cache'],
"cache_db": 1,
"read_timeout": 86400
}
# state settings, 1 is the test DB.
self.state_config = {
"cache_state_host": self.config['aws']['cache-state'],
"cache_state_db": 1
}
# object store settings
_, self.domain = self.config['aws']['cuboid_bucket'].split('.', 1)
self.s3_flush_queue_name = "intTest.S3FlushQueue.{}".format(
self.domain).replace('.', '-')
self.object_store_config = {
"s3_flush_queue":
'', # This will get updated after the queue is created.
"cuboid_bucket":
"intTest{}.{}".format(random.randint(0, 9999),
self.config['aws']['cuboid_bucket']),
"page_in_lambda_function":
self.config['lambda']['page_in_function'],
"page_out_lambda_function":
self.config['lambda']['flush_function'],
"s3_index_table":
"intTest.{}".format(self.config['aws']['s3-index-table']),
"id_index_table":
"intTest.{}".format(self.config['aws']['id-index-table']),
"id_count_table":
"intTest.{}".format(self.config['aws']['id-count-table'])
}
def __init__(self, pid_file_name, pid_dir="/var/run"):
super().__init__(pid_file_name, pid_dir)
self.log = logger.bossLogger()
self.config = configuration.BossConfig()
# kvio settings
kvio_config = {
"cache_host": self.config['aws']['cache'],
"cache_db": self.config['aws']['cache-db'],
"read_timeout": 86400
}
# state settings
state_config = {
"cache_state_host": self.config['aws']['cache-state'],
"cache_state_db": self.config['aws']['cache-state-db']
}
# object store settings
object_store_config = {
"s3_flush_queue": self.config["aws"]["s3-flush-queue"],
"cuboid_bucket": self.config['aws']['cuboid_bucket'],
"page_in_lambda_function":
self.config['lambda']['page_in_function'],
"page_out_lambda_function":
self.config['lambda']['flush_function'],
"s3_index_table": self.config['aws']['s3-index-table'],
"id_index_table": self.config['aws']['id-index-table'],
"id_count_table": self.config['aws']['id-count-table']
}
config_data = {
"kv_config": kvio_config,
"state_config": state_config,
"object_store_config": object_store_config
}
self.lambda_data = {"config": config_data, "lambda-name": "s3_flush"}
self.sqs_watcher = SqsWatcher(self.lambda_data)
