def add_document(self, table, document, disconnect_finish = False) :
table = table.replace('-',"dash")
self.lastrow_mutex.acquire()
lastrowid = -1
try :
cursor = self.conn_check()
if "_id" in document and isinstance(document["_id"], bytes) :
document["_id"] = document["_id"].decode("utf-8")
statement = "insert into " + table + " (document) values ('" + json.dumps(document) + "')"
result = cursor.execute(statement)
if cursor.rowcount != 1 :
MysqlMgdConn.catalogs.cbtool["conn"].rollback()
raise MetricStoreMgdConnException("Add failed w/ statement: " + statement, 65)
cursor.close()
MysqlMgdConn.catalogs.cbtool["conn"].commit()
lastrowid = cursor.lastrowid
if disconnect_finish :
self.disconnect()
except mysql.connector.Error as err :
self.lastrow_mutex.release()
_msg = "Unable to insert document into table \"" + table + "\": "
_msg += str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 6)
except Exception as e :
self.lastrow_mutex.release()
raise MetricStoreMgdConnException(str(e), 64)
self.lastrow_mutex.release()
return lastrowid
def connect(self, tout) :
try:
#if tout and tout > 0:
# MysqlMgdConn.conn.set_connection_timeout(tout)
if not MysqlMgdConn.catalogs.cbtool["conn"] :
cbdebug("Opening to: " + self.database)
MysqlMgdConn.catalogs.cbtool["conn"] = mysql.connector.connect(host = self.host, port = self.port, user = self.username, password = self.password)
cursor = MysqlMgdConn.catalogs.cbtool["conn"].cursor()
try :
cursor.execute("use " + self.database)
MysqlMgdConn.catalogs.cbtool["database"] = True
except mysql.connector.Error as err :
if err.errno == mysql.connector.errorcode.ER_BAD_DB_ERROR:
cbwarn("Database not found. Will create later.")
cursor.close()
_msg = "A connection to MySQL running on host "
_msg += self.host + ", port " + str(self.port) + ", database"
_msg += ' ' + str(MysqlMgdConn.catalogs.cbtool["database"]) + ", with a timeout of "
_msg += str(tout) + "s was established."
cbdebug(_msg)
except mysql.connector.Error as err :
if err.errno == mysql.connector.errorcode.ER_ACCESS_DENIED_ERROR:
_msg = "Something is wrong with your MySQL user name or password."
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
else:
_msg = "Unknown MySQL error: " + str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 2)
def conn_check(self, hostov=False, dbov=False, tout=False):
'''
TBD
'''
if not self.mongodb_conn:
if hostov:
self.host = hostov
if dbov:
self.database = dbov
if tout:
self.timeout = tout
try:
self.connect(self.timeout)
except MetricStoreMgdConnException as obj:
raise MetricStoreMgdConnException(obj.msg, 2)
if self.password and len(self.password) > 2 and str(
self.password).lower() != "false":
try:
_auth_cmd = "mongo -u \"<YOUR ADMIN\" -p \"<YOUR ADMINPASS>\" "
_auth_cmd += "--authenticationDatabase \"admin\" --eval "
_auth_cmd += "\"db.createUser({user: '******', pwd: '" + self.password + "', roles: [ { role:"
_auth_cmd += " 'readWrite', db: 'metrics' } ]})\" "
_auth_cmd += self.host + ":" + str(
self.port) + '/' + self.database
self.mongodb_conn[self.database].authenticate(
self.username, self.password, mechanism='MONGODB-CR')
except PymongoException as errmsg:
_msg = "Unable to authenticate against the database \"" + self.database
_msg += "\":" + str(
errmsg
) + ". \nPlease create the user there (i.e., directly on "
_msg += self.host + ") using the following command:\n"
_msg += _auth_cmd
raise MetricStoreMgdConnException(_msg, 2)
except Exception as e:
_msg = "Unable to authenticate against the database \"" + self.database
_msg += "\":" + str(
e
) + ". \nPlease create the user there (i.e., directly on "
_msg += self.host + ") using the following command:\n"
_msg += _auth_cmd
raise MetricStoreMgdConnException(_msg, 2)
def update_document(self, table, document, disconnect_finish = False) :
table = table.replace('-',"dash")
self.update_mutex.acquire()
try :
cursor = self.conn_check()
if "_id" in document and isinstance(document["_id"], bytes) :
document["_id"] = document["_id"].decode("utf-8")
if "original_mysql_id" not in document :
if "_id" in document :
# Attempt to find the original ID first
statement = "select id from " + table + " where _id = '" + document["_id"] + "'"
cursor.execute(statement)
while True :
rows = cursor.fetchmany(1)
if not len(rows) :
break
for (original_mysql_id,) in rows :
document["original_mysql_id"] = original_mysql_id
if "original_mysql_id" not in document :
cursor.close()
cbwarn("This document does not have a pre-existing identifier. Cannot update. Will insert first")
document["original_mysql_id"] = self.add_document(table, document, disconnect_finish = disconnect_finish)
self.update_mutex.release()
return
statement = "update " + table + " set document = '" + json.dumps(document) + "' where id = " + str(document["original_mysql_id"])
result = cursor.execute(statement)
cursor.close()
MysqlMgdConn.catalogs.cbtool["conn"].commit()
if disconnect_finish :
self.disconnect()
except mysql.connector.Error as err :
self.update_mutex.release()
_msg = "Unable to update documents from the table \""
_msg += table + ": " + str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 8)
except Exception as e :
self.update_mutex.release()
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 67)
self.update_mutex.release()
def get_info(self):
'''
TBD
'''
self.conn_check()
try:
_buildinfo = self.mongodb_conn[self.database].command("buildinfo")
_dbstats = self.mongodb_conn[self.database].command("dbstats")
_output = []
_output.append(["MongoDB Version", _buildinfo["version"]])
_output.append(["Storage Size", str(_dbstats["storageSize"])])
#_output.append(["File Size", _buildinfo["fileSize"]])
_output.append(["Data Size", str(_dbstats["dataSize"])])
_output.append(["Index Size", str(_dbstats["indexSize"])])
_output.append(
["Average Object Size",
str(_dbstats["avgObjSize"])])
_output.append(["Collections", str(_dbstats["collections"])])
return _output
except PymongoException as msg:
_msg = "Unable to get info database " + self.database + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def get_time_boundaries(self, collection, disconnect_finish=False):
'''
TBD
'''
self.conn_check()
collection = collection.replace('-', "dash")
try:
_collection_handle = self.mongodb_conn[self.database][collection]
_start_time = _collection_handle.find(spec={},
fields={"time": "1"},
sort=[("time", 1)],
limit=1)[0]["time"]
_end_time = _collection_handle.find(spec={},
fields={"time": "1"},
sort=[("time", -1)],
limit=1)[0]["time"]
if disconnect_finish:
self.disconnect()
return _start_time, _end_time
except PymongoException as msg:
_msg = "Unable to get time boundaries on the collection \""
_msg += collection + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def get_info(self) :
try :
_output = []
cursor = self.conn_check()
cursor.execute("show variables")
while True :
rows = cursor.fetchmany(4)
if not len(rows) :
break
for row in rows :
if row[0] in ["version"] :
_output.append([row[0], row[1]])
cursor.execute("select sum(data_length + index_length)/1024/1024 'size' FROM information_schema.TABLES")
while True :
rows = cursor.fetchmany(4)
if not len(rows) :
break
for row in rows :
_output.append(["Data Size (MB)", str(float(row[0]))])
cursor.close()
return _output
except mysql.connector.Error as err :
_msg = "Unable to get info for database " + self.database + ": "
_msg += str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 15)
except Exception as e :
cbdebug("No workey: " + str(e))
def get_experiment_list(self, table, disconnect_finish = False) :
table = table.replace('-',"dash")
_experiment_list = []
try :
cursor = self.conn_check()
statement = "select distinct(expid) from " + table + " where expid is not NULL"
cursor.execute(statement)
while True :
rows = cursor.fetchmany(4)
if not len(rows) :
break
for (expid) in rows :
_experiment_list.append(expid)
cursor.close()
if disconnect_finish :
self.disconnect()
return _experiment_list
except mysql.connector.Error as err :
_msg = "Unable to get time experiment list for table \""
_msg += table + ": " + str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 14)
def delete_document(self, collection, criteria, disconnect_finish=False):
'''
TBD
'''
self.conn_check()
collection = collection.replace('-', "dash")
try:
_collection_handle = self.mongodb_conn[self.database][collection]
if int(self.version) < 3:
_collection_handle.remove(criteria)
else:
_collection_handle.delete_one(criteria)
if disconnect_finish:
self.disconnect()
except PymongoException as msg:
_msg = "Unable to remove document from the collection \""
_msg += collection + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def update_document(self, collection, document, disconnect_finish=False):
'''
TBD
'''
self.conn_check()
collection = collection.replace('-', "dash")
try:
_collection_handle = self.mongodb_conn[self.database][collection]
if int(self.version) < 3:
_collection_handle.save(document)
else:
# This insane behavior is supposed to be the "expected behavior"
# according to https://jira.mongodb.org/browse/SERVER-14322
try:
_collection_handle.replace_one({'_id': document["_id"]},
document,
upsert=True)
except:
_collection_handle.replace_one({'_id': document["_id"]},
document,
upsert=True)
if disconnect_finish:
self.disconnect()
except PymongoException as msg:
_msg = "Unable to update documents from the collection \""
_msg += collection + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def add_document(self, collection, document, disconnect_finish=False):
'''
TBD
'''
self.conn_check()
collection = collection.replace('-', "dash")
try:
_collection_handle = self.mongodb_conn[self.database][collection]
if int(self.version) < 3:
_collection_handle.insert(document)
else:
_collection_handle.insert_one(document)
if disconnect_finish:
self.disconnect()
return True
except PymongoException as msg:
_msg = "Unable to insert document \"" + document
_msg += "\" on collection \"" + collection + "\": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def get_experiment_list(self, collection, disconnect_finish=False):
'''
TBD
'''
self.conn_check()
collection = collection.replace('-', "dash")
_experiment_list = None
try:
_collection_handle = self.mongodb_conn[self.database][collection]
#_experiment_list = _collection_handle.distinct('expid')
# The document is getting too big, but a workaround was found.
# TODO: Find a more permanent solution to this.
_experiment_list_agg = _collection_handle.aggregate([{
"$group": {
"_id": '$expid'
}
}])
_experiment_list = ([_v['_id'] for _v in _experiment_list_agg])
if disconnect_finish:
self.disconnect()
return _experiment_list
except PymongoException as msg:
_msg = "Unable to get time boundaries on the collection \""
_msg += collection + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def connect(self, tout):
'''
TBD
'''
try:
if tout > 0:
_conn = MongoClient(host = self.host, port = self.port, \
maxPoolSize=10)
else:
_conn = MongoClient(host = self.host, port = self.port, \
max_pool_size=10)
self.mongodb_conn = _conn
_msg = "A connection to MongoDB running on host "
_msg += self.host + ", port " + str(self.port) + ", database"
_msg += ' ' + str(self.database) + ", with a timeout of "
_msg += str(tout) + "s was established."
cbdebug(_msg)
return self.mongodb_conn
except PymongoException as msg:
True
except:
True
# This was added here just because some MongoDBs don't accept the
# "max_pool_size" parameter
try:
if tout > 0:
_conn = MongoClient(host=self.host, port=self.port)
else:
_conn = MongoClient(host=self.host, port=self.port)
self.mongodb_conn = _conn
_msg = "A connection to MongoDB running on host "
_msg += self.host + ", port " + str(self.port) + ", database"
_msg += ' ' + str(self.database) + ", with a timeout of "
_msg += str(tout) + "s was established."
cbdebug(_msg)
return self.mongodb_conn
except PymongoException as msg:
_msg = "Unable to establish a connection with the MongoDB "
_msg += "server on host " + self.host + " port "
_msg += str(self.port) + "database " + str(self.database) + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def initialize_metric_store(self, username):
'''
TBD
'''
self.conn_check()
username = username.replace('-', "dash")
try:
_collections = [ \
"latest_management_VM_" + username, \
"latest_management_HOST_" + username, \
"latest_runtime_os_VM_" + username, \
"latest_runtime_os_HOST_" + username, \
"latest_runtime_app_VM_" + username \
]
for _collection in _collections:
_collection_handle = self.mongodb_conn[
self.database][_collection]
#_collection_handle.drop()
_collections = [ "trace_" + username, \
"management_HOST_" + username, \
"management_VM_" + username, \
"runtime_os_VM_" + username, \
"runtime_app_VM_" + username, \
"runtime_os_HOST_" + username ]
for _collection in _collections:
_collection_handle = self.mongodb_conn[
self.database][_collection]
if int(self.version) < 3:
_collection_handle.ensure_index("dashboard_polled")
_collection_handle.ensure_index("expid")
_collection_handle.ensure_index("time")
_collection_handle.ensure_index("uuid")
else:
_collection_handle.create_index("dashboard_polled")
_collection_handle.create_index("expid")
_collection_handle.create_index("time")
_collection_handle.create_index("uuid")
self.disconnect()
return True
except PymongoException as msg:
_msg = "Unable to initialize all documents on "
_msg += "\" on collection \"" + _collection + "\": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def count_document(self, table, criteria, disconnect_finish = False) :
table = table.replace('-',"dash")
try :
cursor = self.conn_check()
statement = "select * from " + table + self.make_restrictions(criteria)
count = cursor.execute(statement).rowcount
cursor.close()
if disconnect_finish :
self.disconnect()
return count
except mysql.connector.Error as err :
_msg = "Unable to count documents on the table \""
_msg += table + ": " + str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 11)
def disconnect(self) :
try:
if "disconnect" in dir(MysqlMgdConn.catalogs.cbtool["conn"]) :
MysqlMgdConn.catalogs.cbtool["conn"].disconnect()
MysqlMgdConn.catalogs.cbtool["conn"] = False
_msg = "A connection to MySQL running on host "
_msg += self.host + ", port " + str(self.port) + ", database"
_msg += ' ' + str(MysqlMgdConn.catalogs.cbtool["database"]) + ", was terminated."
cbdebug(_msg)
except mysql.connector.Error as err :
_msg = "Unable to terminate a connection with MySQL "
_msg += "server on host " + self.host + " port "
_msg += str(self.port) + "database " + str(MysqlMgdConn.catalogs.cbtool["database"]) + ": "
_msg += str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 3)
def cleanup_collection(self, table, disconnect_finish = False) :
table = table.replace('-',"dash")
try :
cursor = self.conn_check()
statement = "delete from " + table
cursor.execute(statement)
cursor.close()
MysqlMgdConn.catalogs.cbtool["conn"].commit()
if disconnect_finish :
self.disconnect()
return True
except mysql.connector.Error as err :
_msg = "Unable to drop all documents from the table \""
_msg += table + ": " + str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 10)
def delete_document(self, table, criteria, disconnect_finish = False) :
table = table.replace('-',"dash")
try :
cursor = self.conn_check()
statement = "delete from " + table + self.make_restrictions(criteria)
cursor.execute(statement)
cursor.close()
MysqlMgdConn.catalogs.cbtool["conn"].commit()
if disconnect_finish :
self.disconnect()
except mysql.connector.Error as err :
_msg = "Unable to remove document from the table \""
_msg += table + ": " + str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 9)
def flush_metric_store(self, username, partial=False, criteria={}):
'''
TBD
'''
self.conn_check()
username = username.replace('-', "dash")
try:
_collections = ["latest_management_VM_" + username, \
"latest_management_HOST_" + username, \
"latest_runtime_os_VM_" + username, \
"latest_runtime_os_HOST_" + username, \
"latest_runtime_app_VM_" + username, \
"trace_" + username, \
"management_HOST_" + username, \
"management_VM_" + username, \
"runtime_os_VM_" + username, \
"runtime_app_VM_" + username, \
"runtime_os_HOST_" + username, \
"reported_management_VM_metric_names_" + username, \
"reported_runtime_app_VM_metric_names_" + username, \
"reported_runtime_os_HOST_metric_names_" + username, \
"reported_runtime_os_VM_metric_names_" + username ]
for _collection in _collections:
_collection_handle = self.mongodb_conn[
self.database][_collection]
if partial:
_collection_handle.remove(criteria)
else:
_collection_handle.drop()
self.disconnect()
return True
except PymongoException as msg:
_msg = "Unable to initialize all documents on "
_msg += "\" on collection \"" + _collection + "\": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def get_reported_objects(self, collection, disconnect_finish=False):
'''
TBD
'''
self.conn_check()
collection = collection.replace('-', "dash")
try:
_result = {}
_attributes = [
"vm_name", "role", "ai_name", "type", "aidrs_name", "pattern"
]
for _attribute in _attributes:
_result[_attribute + 's'] = []
_collection_handle = self.mongodb_conn[self.database][collection]
_documents = _collection_handle.find()
for _document in _documents:
for _attribute in _attributes:
if _attribute == "vm_name":
_attribute_r = "name"
else:
_attribute_r = _attribute
if _attribute_r in _document:
if not _result[_attribute + 's'].count(
_document[_attribute_r]):
_result[_attribute + 's'].append(
_document[_attribute_r])
if disconnect_finish:
self.disconnect()
return _result
except PymongoException as msg:
_msg = "Unable to get reported attributes on the collection \""
_msg += collection + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def cleanup_collection(self, collection, disconnect_finish=False):
'''
TBD
'''
self.conn_check()
collection = collection.replace('-', "dash")
try:
_collection_handle = self.mongodb_conn[self.database][collection]
_collection_handle.drop()
if disconnect_finish:
self.disconnect()
return True
except PymongoException as msg:
_msg = "Unable to drop all documents from the collection \""
_msg += collection + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def count_document(self, collection, criteria, disconnect_finish=False):
'''
TBD
'''
self.conn_check()
collection = collection.replace('-', "dash")
try:
_collection_handle = self.mongodb_conn[self.database][collection]
_matches = _collection_handle.find(criteria)
if disconnect_finish:
self.disconnect()
return _matches.count()
except PymongoException as msg:
_msg = "Unable to count documents on the collection \""
_msg += collection + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def disconnect(self):
'''
TBD
'''
try:
if "disconnect" in dir(self.mongodb_conn):
self.mongodb_conn.disconnect()
self.mongodb_conn = False
_msg = "A connection to MongoDB running on host "
_msg += self.host + ", port " + str(self.port) + ", database"
_msg += ' ' + str(self.database) + ", was terminated."
cbdebug(_msg)
return self.mongodb_conn
except PymongoException as msg:
_msg = "Unable to terminate a connection with the MongoDB "
_msg += "server on host " + self.host + " port "
_msg += str(self.port) + "database " + str(self.database) + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def find_document(self, collection, criteria, allmatches = False, \
sortkeypairs = None, limitdocuments = 0, \
documentfields = None, disconnect_finish = False) :
'''
TBD
'''
self.conn_check()
collection = collection.replace('-', "dash")
try:
_collection_handle = self.mongodb_conn[self.database][collection]
if allmatches:
_results = _collection_handle.find(criteria, \
sort = sortkeypairs, \
limit = limitdocuments, \
projection = documentfields)
else:
_results = _collection_handle.find_one(criteria, \
sort = sortkeypairs, \
projection = documentfields)
if disconnect_finish:
self.disconnect()
return _results
except PymongoException as msg:
_msg = "Unable to retrieve documents from the collection \""
_msg += collection + ": "
_msg += str(msg) + '.'
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 1)
def conn_check(self, hostov = False, dbov = False, tout = False) :
self.conn_mutex.acquire()
try :
getattr(MysqlMgdConn.catalogs, "cbtool")
except AttributeError as e :
cbdebug("Initializing thread local connection: ")
MysqlMgdConn.catalogs.cbtool = {}
if "database" not in MysqlMgdConn.catalogs.cbtool :
MysqlMgdConn.catalogs.cbtool["database"] = False
if "conn" not in MysqlMgdConn.catalogs.cbtool :
MysqlMgdConn.catalogs.cbtool["conn"] = False
if not MysqlMgdConn.catalogs.cbtool["conn"] or not MysqlMgdConn.catalogs.cbtool["conn"].is_connected() :
MysqlMgdConn.catalogs.cbtool["conn"] = False
if hostov :
self.host = hostov
if dbov :
MysqlMgdConn.catalogs.cbtool["database"] = dbov
if tout :
self.timeout = tout
try :
self.connect(self.timeout)
except MetricStoreMgdConnException as obj :
self.conn_mutex.release()
raise MetricStoreMgdConnException(obj.msg, 2)
except Exception as e :
self.conn_mutex.release()
raise(e)
assert(MysqlMgdConn.catalogs.cbtool["conn"])
assert(MysqlMgdConn.catalogs.cbtool["conn"].is_connected())
cursor = MysqlMgdConn.catalogs.cbtool["conn"].cursor()
self.conn_mutex.release()
return cursor
def flush_metric_store(self, username, partial = False, criteria = {}) :
username = username.replace('-',"dash")
try :
cursor = self.conn_check()
_tables = ["latest_management_VM_" + username, \
"latest_management_HOST_" + username, \
"latest_runtime_os_VM_" + username, \
"latest_runtime_os_HOST_" + username, \
"latest_runtime_app_VM_" + username, \
"trace_" + username, \
"management_HOST_" + username, \
"management_VM_" + username, \
"runtime_os_VM_" + username, \
"runtime_app_VM_" + username, \
"runtime_os_HOST_" + username, \
"reported_management_VM_metric_names_" + username, \
"reported_runtime_app_VM_metric_names_" + username, \
"reported_runtime_os_HOST_metric_names_" + username, \
"reported_runtime_os_VM_metric_names_" + username ]
for _table in _tables :
if partial and len(criteria) :
statement = "delete from " + _table + self.make_restrictions(criteria)
cursor.execute(statement)
else :
cursor.execute("delete from " + _table)
cursor.close()
MysqlMgdConn.catalogs.cbtool["conn"].commit()
self.disconnect()
return True
except mysql.connector.Error as err :
self.disconnect()
_msg = "Unable to flush metric store: " + str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 5)
def get_reported_objects(self, table, disconnect_finish = False) :
table = table.replace('-',"dash")
try :
cursor = self.conn_check()
_result = {}
_attributes = [ "vm_name", "role", "ai_name", "type", "aidrs_name", "pattern" ]
for _attribute in _attributes :
_result[_attribute + 's'] = []
statement = "select id, document from " + table
cursor.execute(statement)
while True :
rows = cursor.fetchmany(4)
if not len(rows) :
break
for (original_mysql_id, _document) in rows :
for _attribute in _attributes :
if _attribute == "vm_name" :
_attribute_r = "name"
else :
_attribute_r = _attribute
if _attribute_r in _document :
if not _result[_attribute + 's'].count(_document[_attribute_r]) :
_result[_attribute + 's'].append(_document[_attribute_r])
cursor.close()
if disconnect_finish :
self.disconnect()
return _result
except mysql.connector.Error as err :
_msg = "Unable to get reported attributes on the table \""
_msg += table + ": " + str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 12)
def initialize_metric_store(self, username) :
username = username.replace('-',"dash")
try :
cursor = self.conn_check()
if not MysqlMgdConn.catalogs.cbtool["database"] :
cursor.execute("create database " + self.database)
cursor.execute("use " + self.database)
MysqlMgdConn.catalogs.cbtool["database"] = True
_latest_tables = [ \
"latest_management_VM_" + username, \
"latest_management_HOST_" + username, \
"latest_runtime_os_VM_" + username, \
"latest_runtime_os_HOST_" + username, \
"latest_runtime_app_VM_" + username, \
"reported_management_VM_metric_names_" + username, \
"reported_runtime_app_VM_metric_names_" + username, \
"reported_runtime_os_HOST_metric_names_" + username, \
"reported_runtime_os_VM_metric_names_" + username \
]
_indexed_tables = [ "trace_" + username, \
"management_HOST_" + username, \
"management_VM_" + username, \
"runtime_os_VM_" + username, \
"runtime_app_VM_" + username, \
"runtime_os_HOST_" + username ]
cursor.execute("show tables")
_tables_found = []
for x in cursor:
_tables_found.append(x[0])
for _table in (_latest_tables + _indexed_tables) :
if _table not in _tables_found :
statement = "create table " + _table + "(" + \
"id int auto_increment primary key," + \
"document json NOT NULL," + \
"`expid` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.expid')," + \
"`_id` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$._id')," + \
"`time` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.time')," + \
"`uuid` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.uuid')," + \
"`dashboard_polled` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.dashboard_polled')" + \
")"
cursor.execute(statement)
if _table in _indexed_tables :
cursor.execute("CREATE INDEX `expid_idx` ON `" + _table + "`(`expid`)")
cursor.execute("CREATE INDEX `time_idx` ON `" + _table + "`(`time`)")
cursor.execute("CREATE INDEX `uuid_idx` ON `" + _table + "`(`uuid`)")
cursor.execute("CREATE INDEX `dashboard_polled_idx` ON `" + _table + "`(`dashboard_polled`)")
cursor.close()
MysqlMgdConn.catalogs.cbtool["conn"].commit()
self.disconnect()
return True
except mysql.connector.Error as err :
self.disconnect()
_msg = "Unable to complete database initialization: "
_msg += str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 4)
def make_restrictions(self, criteria, join = "and", level = 0) :
full_list = ""
restrictions = []
for _key in criteria.keys() :
_value = criteria[_key]
if isinstance(_value, set) :
_msg = "1) We cannot yet handle this criteria: " + str(criteria)
cberr(_msg)
raise MetricStoreMgdConnException(_msg, 41)
elif isinstance(_value, dict) :
for subkey in _value.keys() :
if subkey.lower() == "$exists" :
if not isinstance(_value[subkey], bool) :
_msg = "2) We cannot yet handle this criteria: " + str(_value)
cberr(_msg)
raise MetricStoreMgdConnException(_msg, 41)
if _value[subkey] :
restrictions.append("document->>'$." + _key + "' IS NOT NULL")
else :
restrictions.append("document->>'$." + _key + "' IS NULL")
else :
_msg = "3) We cannot yet handle this criteria: " + str(subkey)
cberr(_msg)
raise MetricStoreMgdConnException(_msg, 41)
elif isinstance(_value, list) :
# Handle this group below
continue
else :
_newvalue = _value
if isinstance(_value, bytes) :
_newvalue = _value.decode("utf-8")
restrictions.append("document->>'$." + _key + "' = '" + str(_newvalue) + "'")
if len(restrictions) :
full_list += (" " + join + " ").join(restrictions)
for _key in criteria.keys() :
if _key.lower() == "$or" or _key.lower() == "$and" :
_value = criteria[_key]
if isinstance(_value, list) :
subdict = {}
for subitem in _value :
if not isinstance(subitem, dict) :
_msg = "4) We cannot yet handle this criteria: " + str(subitem)
cberr(_msg)
raise MetricStoreMgdConnException(_msg, 41)
subdict.update(subitem)
sub_restrictions = self.make_restrictions(subdict, join = _key[1:], level = level + 1)
if sub_restrictions.strip() != "" :
full_list += " and (" + sub_restrictions + ")"
else :
_msg = "5) We cannot yet handle this criteria: " + str(_value)
cberr(_msg)
raise MetricStoreMgdConnException(_msg, 41)
if full_list.strip() != "" :
if level == 0 :
return " where " + full_list
else :
return full_list
return ""
def find_document(self, table, criteria, allmatches = False, \
sortkeypairs = None, limitdocuments = 0, \
documentfields = None, disconnect_finish = False) :
table = table.replace('-',"dash")
try :
cursor = self.conn_check()
statement = "select "
if documentfields is not None :
convertedfields = []
for field in documentfields :
convertedfields.append("document->>'$." + field + "'")
statement += ",".join(["id"] + convertedfields)
else :
statement += " id,document "
statement += " from " + table + " " + self.make_restrictions(criteria)
if sortkeypairs :
keylist = []
for keypair in sortkeypairs :
# FIXME: I'm unsure of how to have different directional sort criteria for multiple
# sorted keys. Will have to look into that later, so for the time being,
# I'm dropping the direction.
keylist.append("document->>'$." + keypair[0] + "'")
statement += " order by " + ",".join(keylist)
if not allmatches or limitdocuments :
if limitdocuments > 0 :
statement += " limit " + str(limitdocuments)
else :
statement += " limit 1"
_results = []
# FIXME: We need to figure out how to safely allow iterators over
# the live connection. But for now, let's just extract all the results
result = cursor.execute(statement)
while True :
rows = cursor.fetchmany(4)
if not len(rows) :
break
for resultset in rows :
original_mysql_id = resultset[0]
document = False
if documentfields is not None :
document = {}
for idx in range(1, len(resultset)) :
document[documentfields[idx - 1]] = resultset[idx].decode()
else :
if isinstance(resultset[1], str) :
document = json.loads(resultset[1])
else :
assert(isinstance(resultset[1], dict))
document = resultset[1]
document["original_mysql_id"] = original_mysql_id
_results.append(document)
cursor.close()
if allmatches :
return _results
else :
if len(_results) >= 1 :
return _results[0]
return None
except mysql.connector.Error as err :
_msg = "Unable to retrieve documents from the table \""
_msg += table + ": " + str(err)
cberr(_msg)
raise MetricStoreMgdConnException(str(_msg), 7)