def run(self): # Setup SPDB instance config = configuration.BossConfig() kvio_config = {"cache_host": config['aws']['cache'], "cache_db": config['aws']['cache-state-db'], "read_timeout": 86400} state_config = {"cache_state_host": config['aws']['cache-state'], "cache_state_db": config['aws']['cache-db']} object_store_config = {"s3_flush_queue": config['aws']["s3-flush-queue"], "cuboid_bucket": config['aws']['cuboid_bucket'], "page_in_lambda_function": config['lambda']['page_in_function'], "page_out_lambda_function": config['lambda']['flush_function'], "s3_index_table": config['aws']['s3-index-table'], "id_index_table": config['aws']['id-index-table'], "id_count_table": config['aws']['id-count-table']} sp = SpatialDB(kvio_config, state_config, object_store_config) while True: self.log.info("Checking for delayed write operations.") try: self.process(sp) time.sleep(5) except Exception as err: self.log.error("An error occurred running the process() method! \n {}".format(err))
def setUpClass(cls): """ Create a diction of configuration values for the test resource. """ # Create resource cls.setup_helper = SetupTests() cls.data = cls.setup_helper.get_anno64_dict() cls.resource = BossResourceBasic(cls.data) # Load config cls.config = configuration.BossConfig() cls.object_store_config = { "s3_flush_queue": 'https://mytestqueue.com', "cuboid_bucket": "test_bucket", "page_in_lambda_function": "page_in.test.boss", "page_out_lambda_function": "page_out.test.boss", "s3_index_table": "test_s3_table", "id_index_table": "test_id_table", "id_count_table": "test_count_table", } # Create AWS Resources needed for tests while mocking cls.setup_helper.start_mocking() cls.setup_helper.create_index_table( cls.object_store_config["id_count_table"], cls.setup_helper.ID_COUNT_SCHEMA) cls.obj_ind = ObjectIndices(cls.object_store_config["s3_index_table"], cls.object_store_config["id_index_table"], cls.object_store_config["id_count_table"], 'us-east-1')
def setUpClass(cls): """Setup the redis client at the start of the test""" cls.data = get_image_dict() cls.resource = BossResourceBasic(cls.data) cls.config = configuration.BossConfig() cls.cache_client = redis.StrictRedis(host=cls.config["aws"]["cache"], port=6379, db=1, decode_responses=False) cls.config_data = {"cache_client": cls.cache_client, "read_timeout": 86400}
def get_test_configuration(): """Method to get the integration test configuration info for spdb Returns: (dict, dict, dict): A tuple of dictionaries (kvio_config, state_config, object_store_config, s3_flush_queue_name) """ config = configuration.BossConfig() # Get domain info parts = config['aws']['cache'].split('.') domain = "{}.{}".format(parts[1], parts[2]) # kvio settings kvio_config = { "cache_host": config['aws']['cache'], "cache_db": 1, "read_timeout": 86400 } # state settings state_config = { "cache_state_host": config['aws']['cache-state'], "cache_state_db": 1 } _, domain = config['aws']['cuboid_bucket'].split('.', 1) s3_flush_queue_name = "intTest.S3FlushQueue.{}".format(domain).replace( '.', '-') account_id = "{}".format(get_account_id()) account_id = account_id[:5] object_store_config = { "s3_flush_queue": "https://queue.amazonaws.com/{}/{}".format(get_account_id(), s3_flush_queue_name), "cuboid_bucket": "inttest.{}.{}".format(account_id, config['aws']['cuboid_bucket']), "page_in_lambda_function": config['lambda']['page_in_function'], "page_out_lambda_function": config['lambda']['flush_function'], "s3_index_table": "intTest.{}".format(config['aws']['s3-index-table']), "id_index_table": "intTest.{}".format(config['aws']['id-index-table']), "id_count_table": "intTest.{}".format(config['aws']['id-count-table']) } return kvio_config, state_config, object_store_config, s3_flush_queue_name
def __init__(self): # Load boss config file config = configuration.BossConfig() # Set Properties self.__sessions = queue.Queue() if config['aws_mngr']['num_sessions'] == 'auto': self.__num_sessions = multiprocessing.cpu_count() else: self.__num_sessions = config['aws_mngr']['num_sessions'] # Initialize the credentials and sessions self.__init_sessions()
def setUpParams(self): self.setup_helper = SetupTests() # Don't use mock Amazon resources. self.setup_helper.mock = False self.data = self.setup_helper.get_image8_dict() self.resource = BossResourceBasic(self.data) self.config = configuration.BossConfig() # kvio settings, 1 is the test DB. self.kvio_config = { "cache_host": self.config['aws']['cache'], "cache_db": 1, "read_timeout": 86400 } # state settings, 1 is the test DB. self.state_config = { "cache_state_host": self.config['aws']['cache-state'], "cache_state_db": 1 } # object store settings _, self.domain = self.config['aws']['cuboid_bucket'].split('.', 1) self.s3_flush_queue_name = "intTest.S3FlushQueue.{}".format( self.domain).replace('.', '-') self.object_store_config = { "s3_flush_queue": '', # This will get updated after the queue is created. "cuboid_bucket": "intTest{}.{}".format(random.randint(0, 9999), self.config['aws']['cuboid_bucket']), "page_in_lambda_function": self.config['lambda']['page_in_function'], "page_out_lambda_function": self.config['lambda']['flush_function'], "s3_index_table": "intTest.{}".format(self.config['aws']['s3-index-table']), "id_index_table": "intTest.{}".format(self.config['aws']['id-index-table']), "id_count_table": "intTest.{}".format(self.config['aws']['id-count-table']) }
def __init__(self, pid_file_name, pid_dir="/var/run"): super().__init__(pid_file_name, pid_dir) self.log = logger.bossLogger() self.config = configuration.BossConfig() # kvio settings kvio_config = { "cache_host": self.config['aws']['cache'], "cache_db": self.config['aws']['cache-db'], "read_timeout": 86400 } # state settings state_config = { "cache_state_host": self.config['aws']['cache-state'], "cache_state_db": self.config['aws']['cache-state-db'] } # object store settings object_store_config = { "s3_flush_queue": self.config["aws"]["s3-flush-queue"], "cuboid_bucket": self.config['aws']['cuboid_bucket'], "page_in_lambda_function": self.config['lambda']['page_in_function'], "page_out_lambda_function": self.config['lambda']['flush_function'], "s3_index_table": self.config['aws']['s3-index-table'], "id_index_table": self.config['aws']['id-index-table'], "id_count_table": self.config['aws']['id-count-table'] } config_data = { "kv_config": kvio_config, "state_config": state_config, "object_store_config": object_store_config } self.lambda_data = {"config": config_data, "lambda-name": "s3_flush"} self.sqs_watcher = SqsWatcher(self.lambda_data)