def test_get_topic_map( get_container_func, ports_config, query, expected_topic_id_map, expected_topic_name_map, ): get_container, image = get_container_func with get_container(image, ports=ports_config["ports"], env=ENV_MONGODB) as container: wait_for_connection(container) query_database(container, query) actual_topic_id_map, actual_topic_name_map = mongoutils.get_topic_map( mongo_client(ports_config["port_on_host"]), "topics") assert actual_topic_id_map == expected_topic_id_map assert actual_topic_name_map == expected_topic_name_map
def historian_setup(self): _log.debug("HISTORIAN SETUP") self._client = mongoutils.get_mongo_client(self._connection_params) db = self._client.get_default_database() db[self._data_collection].create_index( [('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)], unique=True, background=True) self._topic_id_map, self._topic_name_map = \ mongoutils.get_topic_map( self._client, self._topic_collection) self._load_meta_map() if self._agg_topic_collection in db.collection_names(): _log.debug("found agg_topics_collection ") self._agg_topic_id_map = mongoutils.get_agg_topic_map( self._client, self._agg_topic_collection) else: _log.debug("no agg topics to load") self._agg_topic_id_map = {}
def historian_setup(self): _log.debug("HISTORIAN SETUP") self._client = mongoutils.get_mongo_client(self._connection_params) db = self._client.get_default_database() db[self._data_collection].create_index( [('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)], unique=True, background=True) self._topic_id_map, self._topic_name_map = \ mongoutils.get_topic_map( self._client, self._topic_collection) self._load_meta_map() if self._agg_topic_collection in db.collection_names(): _log.debug("found agg_topics_collection ") self._agg_topic_id_map = mongoutils.get_agg_topic_map( self._client, self._agg_topic_collection) else: _log.debug("no agg topics to load") self._agg_topic_id_map = {}
def historian_setup(self): _log.debug("HISTORIAN SETUP") self._client = mongoutils.get_mongo_client(self._connection_params, minPoolSize=10) _log.info("Mongo client created with min pool size {}".format( self._client.min_pool_size)) db = self._client.get_default_database() col_list = db.collection_names() create_index1 = True create_index2 = True if self._readonly: create_index1 = False create_index2 = False # if data collection exists check if necessary indexes exists elif self._data_collection in col_list: index_info = db[self._data_collection].index_information() index_list = [value['key'] for value in index_info.viewvalues()] index_new_list = [] for index in index_list: keys = set() for key in index: keys.add(key[0]) index_new_list.append(keys) _log.debug("Index list got from db is {}. formatted list is ".format( index_list, index_new_list)) i1 = {'topic_id', 'ts'} if i1 in index_new_list: create_index1 = False i2 = {'ts'} if i2 in index_new_list: create_index2 = False # create data indexes if needed if create_index1: db[self._data_collection].create_index( [('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)], unique=True, background=True) if create_index2: db[self._data_collection].create_index( [('ts', pymongo.DESCENDING)], background=True) self._topic_id_map, self._topic_name_map = \ mongoutils.get_topic_map( self._client, self._topic_collection) self._load_meta_map() if self._agg_topic_collection in db.collection_names(): _log.debug("found agg_topics_collection ") self._agg_topic_id_map = mongoutils.get_agg_topic_map( self._client, self._agg_topic_collection) else: _log.debug("no agg topics to load") self._agg_topic_id_map = {} if not self._readonly: db[self.HOURLY_COLLECTION].create_index( [('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)], unique=True, background=True) db[self.HOURLY_COLLECTION].create_index( [('last_updated_data', pymongo.DESCENDING)], background=True) db[self.DAILY_COLLECTION].create_index( [('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)], unique=True, background=True) db[self.DAILY_COLLECTION].create_index( [('last_updated_data', pymongo.DESCENDING)], background=True)
def get_topic_map(self): return mongoutils.get_topic_map(self.dbclient, self._topic_collection)
def get_topic_map(self): return mongoutils.get_topic_map(self.dbclient, self._topic_collection)
def historian_setup(self): _log.debug("HISTORIAN SETUP") self._client = mongoutils.get_mongo_client(self._connection_params, minPoolSize=10) _log.info("Mongo client created with min pool size {}".format( self._client.min_pool_size)) db = self._client.get_default_database() col_list = db.collection_names() create_index1 = True create_index2 = True if self._readonly: create_index1 = False create_index2 = False # if data collection exists check if necessary indexes exists elif self._data_collection in col_list: index_info = db[self._data_collection].index_information() index_list = [value['key'] for value in index_info.viewvalues()] index_new_list = [] for index in index_list: keys = set() for key in index: keys.add(key[0]) index_new_list.append(keys) _log.debug("Index list got from db is {}. formatted list is ".format( index_list, index_new_list)) i1 = {'topic_id', 'ts'} if i1 in index_new_list: create_index1 = False i2 = {'ts'} if i2 in index_new_list: create_index2 = False # create data indexes if needed if create_index1: db[self._data_collection].create_index( [('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)], unique=True, background=True) if create_index2: db[self._data_collection].create_index( [('ts', pymongo.DESCENDING)], background=True) self._topic_id_map, self._topic_name_map = \ mongoutils.get_topic_map( self._client, self._topic_collection) self._load_meta_map() if self._agg_topic_collection in db.collection_names(): _log.debug("found agg_topics_collection ") self._agg_topic_id_map = mongoutils.get_agg_topic_map( self._client, self._agg_topic_collection) else: _log.debug("no agg topics to load") self._agg_topic_id_map = {} if not self._readonly: db[self.HOURLY_COLLECTION].create_index( [('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)], unique=True, background=True) db[self.HOURLY_COLLECTION].create_index( [('last_updated_data', pymongo.DESCENDING)], background=True) db[self.DAILY_COLLECTION].create_index( [('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)], unique=True, background=True) db[self.DAILY_COLLECTION].create_index( [('last_updated_data', pymongo.DESCENDING)], background=True)