def test_get_topic_map(
get_container_func,
ports_config,
query,
expected_topic_id_map,
expected_topic_name_map,
):
get_container, image = get_container_func
with get_container(image, ports=ports_config["ports"],
env=ENV_MONGODB) as container:
wait_for_connection(container)
query_database(container, query)
actual_topic_id_map, actual_topic_name_map = mongoutils.get_topic_map(
mongo_client(ports_config["port_on_host"]), "topics")
assert actual_topic_id_map == expected_topic_id_map
assert actual_topic_name_map == expected_topic_name_map
def historian_setup(self):
_log.debug("HISTORIAN SETUP")
self._client = mongoutils.get_mongo_client(self._connection_params)
db = self._client.get_default_database()
db[self._data_collection].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True, background=True)
self._topic_id_map, self._topic_name_map = \
mongoutils.get_topic_map(
self._client, self._topic_collection)
self._load_meta_map()
if self._agg_topic_collection in db.collection_names():
_log.debug("found agg_topics_collection ")
self._agg_topic_id_map = mongoutils.get_agg_topic_map(
self._client, self._agg_topic_collection)
else:
_log.debug("no agg topics to load")
self._agg_topic_id_map = {}
def historian_setup(self):
_log.debug("HISTORIAN SETUP")
self._client = mongoutils.get_mongo_client(self._connection_params)
db = self._client.get_default_database()
db[self._data_collection].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True,
background=True)
self._topic_id_map, self._topic_name_map = \
mongoutils.get_topic_map(
self._client, self._topic_collection)
self._load_meta_map()
if self._agg_topic_collection in db.collection_names():
_log.debug("found agg_topics_collection ")
self._agg_topic_id_map = mongoutils.get_agg_topic_map(
self._client, self._agg_topic_collection)
else:
_log.debug("no agg topics to load")
self._agg_topic_id_map = {}
def historian_setup(self):
_log.debug("HISTORIAN SETUP")
self._client = mongoutils.get_mongo_client(self._connection_params,
minPoolSize=10)
_log.info("Mongo client created with min pool size {}".format(
self._client.min_pool_size))
db = self._client.get_default_database()
col_list = db.collection_names()
create_index1 = True
create_index2 = True
if self._readonly:
create_index1 = False
create_index2 = False
# if data collection exists check if necessary indexes exists
elif self._data_collection in col_list:
index_info = db[self._data_collection].index_information()
index_list = [value['key'] for value in index_info.viewvalues()]
index_new_list = []
for index in index_list:
keys = set()
for key in index:
keys.add(key[0])
index_new_list.append(keys)
_log.debug("Index list got from db is {}. formatted list is ".format(
index_list, index_new_list))
i1 = {'topic_id', 'ts'}
if i1 in index_new_list:
create_index1 = False
i2 = {'ts'}
if i2 in index_new_list:
create_index2 = False
# create data indexes if needed
if create_index1:
db[self._data_collection].create_index(
[('topic_id', pymongo.DESCENDING),
('ts', pymongo.DESCENDING)],
unique=True, background=True)
if create_index2:
db[self._data_collection].create_index(
[('ts', pymongo.DESCENDING)], background=True)
self._topic_id_map, self._topic_name_map = \
mongoutils.get_topic_map(
self._client, self._topic_collection)
self._load_meta_map()
if self._agg_topic_collection in db.collection_names():
_log.debug("found agg_topics_collection ")
self._agg_topic_id_map = mongoutils.get_agg_topic_map(
self._client, self._agg_topic_collection)
else:
_log.debug("no agg topics to load")
self._agg_topic_id_map = {}
if not self._readonly:
db[self.HOURLY_COLLECTION].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True, background=True)
db[self.HOURLY_COLLECTION].create_index(
[('last_updated_data', pymongo.DESCENDING)], background=True)
db[self.DAILY_COLLECTION].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True, background=True)
db[self.DAILY_COLLECTION].create_index(
[('last_updated_data', pymongo.DESCENDING)],
background=True)
def get_topic_map(self):
return mongoutils.get_topic_map(self.dbclient, self._topic_collection)
def get_topic_map(self):
return mongoutils.get_topic_map(self.dbclient, self._topic_collection)
def historian_setup(self):
_log.debug("HISTORIAN SETUP")
self._client = mongoutils.get_mongo_client(self._connection_params,
minPoolSize=10)
_log.info("Mongo client created with min pool size {}".format(
self._client.min_pool_size))
db = self._client.get_default_database()
col_list = db.collection_names()
create_index1 = True
create_index2 = True
if self._readonly:
create_index1 = False
create_index2 = False
# if data collection exists check if necessary indexes exists
elif self._data_collection in col_list:
index_info = db[self._data_collection].index_information()
index_list = [value['key'] for value in index_info.viewvalues()]
index_new_list = []
for index in index_list:
keys = set()
for key in index:
keys.add(key[0])
index_new_list.append(keys)
_log.debug("Index list got from db is {}. formatted list is ".format(
index_list, index_new_list))
i1 = {'topic_id', 'ts'}
if i1 in index_new_list:
create_index1 = False
i2 = {'ts'}
if i2 in index_new_list:
create_index2 = False
# create data indexes if needed
if create_index1:
db[self._data_collection].create_index(
[('topic_id', pymongo.DESCENDING),
('ts', pymongo.DESCENDING)],
unique=True, background=True)
if create_index2:
db[self._data_collection].create_index(
[('ts', pymongo.DESCENDING)], background=True)
self._topic_id_map, self._topic_name_map = \
mongoutils.get_topic_map(
self._client, self._topic_collection)
self._load_meta_map()
if self._agg_topic_collection in db.collection_names():
_log.debug("found agg_topics_collection ")
self._agg_topic_id_map = mongoutils.get_agg_topic_map(
self._client, self._agg_topic_collection)
else:
_log.debug("no agg topics to load")
self._agg_topic_id_map = {}
if not self._readonly:
db[self.HOURLY_COLLECTION].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True, background=True)
db[self.HOURLY_COLLECTION].create_index(
[('last_updated_data', pymongo.DESCENDING)], background=True)
db[self.DAILY_COLLECTION].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True, background=True)
db[self.DAILY_COLLECTION].create_index(
[('last_updated_data', pymongo.DESCENDING)],
background=True)
