def __init__(self, connection, table_prefix=None, **kwargs):
"""Initialise the tagging service.
:param connection: dictionary object containing the database
connection details
:param table_prefix: optional prefix to be used for all tag tables
:param kwargs: additional keyword arguments. (optional identity and
topic_replace_list used by parent classes)
"""
super(MongodbTaggingService, self).__init__(**kwargs)
self.connection = connection
self._client = mongoutils.get_mongo_client(connection['params'])
self.tags_collection = "tags"
self.tag_refs_collection = "tag_refs"
#self.units_table = "units" #in version 2
self.categories_collection = "categories"
self.topic_tags_collection = "topic_tags"
if table_prefix:
self.tags_collection = table_prefix + "_" + \
self.tags_collection
self.tag_refs_collection = table_prefix + "_" + \
self.tag_refs_collection
self.categories_collection = table_prefix + "_" + \
self.categories_collection
self.topic_tags_collection = table_prefix + "_" + \
self.topic_tags_collection
def __init__(self, connection, table_prefix=None, **kwargs):
"""Initialise the tagging service.
:param connection: dictionary object containing the database
connection details
:param table_prefix: optional prefix to be used for all tag tables
:param kwargs: additional keyword arguments. (optional identity and
topic_replace_list used by parent classes)
"""
super(MongodbTaggingService, self).__init__(**kwargs)
self.connection = connection
self._client = mongoutils.get_mongo_client(connection['params'])
self.tags_collection = "tags"
self.tag_refs_collection = "tag_refs"
#self.units_table = "units" #in version 2
self.categories_collection = "categories"
self.topic_tags_collection = "topic_tags"
if table_prefix:
self.tags_collection = table_prefix + "_" + \
self.tags_collection
self.tag_refs_collection = table_prefix + "_" + \
self.tag_refs_collection
self.categories_collection = table_prefix + "_" + \
self.categories_collection
self.topic_tags_collection = table_prefix + "_" + \
self.topic_tags_collection
def mongo_client(port):
connection_params = {
"host": "localhost",
"port": port,
"database": TEST_DATABASE,
"user": ROOT_USERNAME,
"passwd": ROOT_PASSWORD,
"authSource": "admin",
}
return mongoutils.get_mongo_client(connection_params)
def historian_setup(self):
_log.debug("HISTORIAN SETUP")
self._client = mongoutils.get_mongo_client(self._connection_params)
db = self._client.get_default_database()
db[self._data_collection].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True, background=True)
self._topic_id_map, self._topic_name_map = \
mongoutils.get_topic_map(
self._client, self._topic_collection)
self._load_meta_map()
if self._agg_topic_collection in db.collection_names():
_log.debug("found agg_topics_collection ")
self._agg_topic_id_map = mongoutils.get_agg_topic_map(
self._client, self._agg_topic_collection)
else:
_log.debug("no agg topics to load")
self._agg_topic_id_map = {}
def historian_setup(self):
_log.debug("HISTORIAN SETUP")
self._client = mongoutils.get_mongo_client(self._connection_params)
db = self._client.get_default_database()
db[self._data_collection].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True,
background=True)
self._topic_id_map, self._topic_name_map = \
mongoutils.get_topic_map(
self._client, self._topic_collection)
self._load_meta_map()
if self._agg_topic_collection in db.collection_names():
_log.debug("found agg_topics_collection ")
self._agg_topic_id_map = mongoutils.get_agg_topic_map(
self._client, self._agg_topic_collection)
else:
_log.debug("no agg topics to load")
self._agg_topic_id_map = {}
def configure(self, config_name, action, config):
if not config or not isinstance(config, dict):
raise ValueError("Configuration should be a valid json")
connection = config.get('connection')
self.dbclient = mongoutils.get_mongo_client(connection['params'])
# Why are we not letting users configure data and topic collection
# names in mongo similar to sqlhistorian
# tables_def = sqlutils.get_table_def(self.config)
db = self.dbclient.get_default_database()
cursor = db[self.volttron_table_defs].find()
table_map = {}
prefix = ""
for document in cursor:
table_map[document['table_id'].lower()] = document['table_name']
prefix = document.get('table_prefix') + "_" if document.get(
'table_prefix') else ''
self._data_collection = prefix + table_map.get('data_table', 'data')
self._meta_collection = prefix + table_map.get('meta_table', 'meta')
self._topic_collection = prefix + table_map.get(
'topics_table', 'topics')
self._agg_meta_collection = prefix + 'aggregate_' \
+ table_map.get('meta_table', 'meta')
self._agg_topic_collection = prefix + 'aggregate_' \
+ table_map.get('topics_table', 'topics')
db[self._agg_topic_collection].create_index(
[('agg_topic_name', pymongo.DESCENDING),
('agg_type', pymongo.DESCENDING),
('agg_time_period', pymongo.DESCENDING)],
unique=True,
background=True)
# 2. load topic name and topic id.
self.topic_id_map, name_map = self.get_topic_map()
super(MongodbAggregateHistorian,
self).configure(config_name, action, config)
def configure(self, config_name, action, config):
if not config or not isinstance(config, dict):
raise ValueError("Configuration should be a valid json")
connection = config.get('connection')
self.dbclient = mongoutils.get_mongo_client(connection['params'])
# Why are we not letting users configure data and topic collection
# names in mongo similar to sqlhistorian
# tables_def = sqlutils.get_table_def(self.config)
db = self.dbclient.get_default_database()
cursor = db[self.volttron_table_defs].find()
table_map = {}
prefix = ""
for document in cursor:
table_map[document['table_id'].lower()] = document[
'table_name']
prefix = document.get('table_prefix') + "_" if document.get(
'table_prefix') else ''
self._data_collection = prefix + table_map.get('data_table', 'data')
self._meta_collection = prefix + table_map.get('meta_table', 'meta')
self._topic_collection = prefix + table_map.get('topics_table',
'topics')
self._agg_meta_collection = prefix + 'aggregate_' \
+ table_map.get('meta_table', 'meta')
self._agg_topic_collection = prefix + 'aggregate_' \
+ table_map.get('topics_table', 'topics')
db[self._agg_topic_collection].create_index(
[('agg_topic_name', pymongo.DESCENDING),
('agg_type', pymongo.DESCENDING),
('agg_time_period', pymongo.DESCENDING)],
unique=True, background=True)
# 2. load topic name and topic id.
self.topic_id_map, name_map = self.get_topic_map()
super(MongodbAggregateHistorian, self).configure(config_name,
action, config)
def historian_setup(self):
_log.debug("HISTORIAN SETUP")
self._client = mongoutils.get_mongo_client(self._connection_params,
minPoolSize=10)
_log.info("Mongo client created with min pool size {}".format(
self._client.min_pool_size))
db = self._client.get_default_database()
col_list = db.collection_names()
create_index1 = True
create_index2 = True
if self._readonly:
create_index1 = False
create_index2 = False
# if data collection exists check if necessary indexes exists
elif self._data_collection in col_list:
index_info = db[self._data_collection].index_information()
index_list = [value['key'] for value in index_info.viewvalues()]
index_new_list = []
for index in index_list:
keys = set()
for key in index:
keys.add(key[0])
index_new_list.append(keys)
_log.debug("Index list got from db is {}. formatted list is ".format(
index_list, index_new_list))
i1 = {'topic_id', 'ts'}
if i1 in index_new_list:
create_index1 = False
i2 = {'ts'}
if i2 in index_new_list:
create_index2 = False
# create data indexes if needed
if create_index1:
db[self._data_collection].create_index(
[('topic_id', pymongo.DESCENDING),
('ts', pymongo.DESCENDING)],
unique=True, background=True)
if create_index2:
db[self._data_collection].create_index(
[('ts', pymongo.DESCENDING)], background=True)
self._topic_id_map, self._topic_name_map = \
mongoutils.get_topic_map(
self._client, self._topic_collection)
self._load_meta_map()
if self._agg_topic_collection in db.collection_names():
_log.debug("found agg_topics_collection ")
self._agg_topic_id_map = mongoutils.get_agg_topic_map(
self._client, self._agg_topic_collection)
else:
_log.debug("no agg topics to load")
self._agg_topic_id_map = {}
if not self._readonly:
db[self.HOURLY_COLLECTION].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True, background=True)
db[self.HOURLY_COLLECTION].create_index(
[('last_updated_data', pymongo.DESCENDING)], background=True)
db[self.DAILY_COLLECTION].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True, background=True)
db[self.DAILY_COLLECTION].create_index(
[('last_updated_data', pymongo.DESCENDING)],
background=True)
def get_mongo_db():
mongo_client = mongoutils.get_mongo_client(mongo_params["connection"]["params"])
return mongo_client.get_default_database()
def get_mongo_db():
mongo_client = mongoutils.get_mongo_client(
mongo_params["connection"]["params"])
return mongo_client.get_default_database()
def historian_setup(self):
_log.debug("HISTORIAN SETUP")
self._client = mongoutils.get_mongo_client(self._connection_params,
minPoolSize=10)
_log.info("Mongo client created with min pool size {}".format(
self._client.min_pool_size))
db = self._client.get_default_database()
col_list = db.collection_names()
create_index1 = True
create_index2 = True
if self._readonly:
create_index1 = False
create_index2 = False
# if data collection exists check if necessary indexes exists
elif self._data_collection in col_list:
index_info = db[self._data_collection].index_information()
index_list = [value['key'] for value in index_info.viewvalues()]
index_new_list = []
for index in index_list:
keys = set()
for key in index:
keys.add(key[0])
index_new_list.append(keys)
_log.debug("Index list got from db is {}. formatted list is ".format(
index_list, index_new_list))
i1 = {'topic_id', 'ts'}
if i1 in index_new_list:
create_index1 = False
i2 = {'ts'}
if i2 in index_new_list:
create_index2 = False
# create data indexes if needed
if create_index1:
db[self._data_collection].create_index(
[('topic_id', pymongo.DESCENDING),
('ts', pymongo.DESCENDING)],
unique=True, background=True)
if create_index2:
db[self._data_collection].create_index(
[('ts', pymongo.DESCENDING)], background=True)
self._topic_id_map, self._topic_name_map = \
mongoutils.get_topic_map(
self._client, self._topic_collection)
self._load_meta_map()
if self._agg_topic_collection in db.collection_names():
_log.debug("found agg_topics_collection ")
self._agg_topic_id_map = mongoutils.get_agg_topic_map(
self._client, self._agg_topic_collection)
else:
_log.debug("no agg topics to load")
self._agg_topic_id_map = {}
if not self._readonly:
db[self.HOURLY_COLLECTION].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True, background=True)
db[self.HOURLY_COLLECTION].create_index(
[('last_updated_data', pymongo.DESCENDING)], background=True)
db[self.DAILY_COLLECTION].create_index(
[('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)],
unique=True, background=True)
db[self.DAILY_COLLECTION].create_index(
[('last_updated_data', pymongo.DESCENDING)],
background=True)
