def add_document(self, table, document, disconnect_finish = False) : table = table.replace('-',"dash") self.lastrow_mutex.acquire() lastrowid = -1 try : cursor = self.conn_check() if "_id" in document and isinstance(document["_id"], bytes) : document["_id"] = document["_id"].decode("utf-8") statement = "insert into " + table + " (document) values ('" + json.dumps(document) + "')" result = cursor.execute(statement) if cursor.rowcount != 1 : MysqlMgdConn.catalogs.cbtool["conn"].rollback() raise MetricStoreMgdConnException("Add failed w/ statement: " + statement, 65) cursor.close() MysqlMgdConn.catalogs.cbtool["conn"].commit() lastrowid = cursor.lastrowid if disconnect_finish : self.disconnect() except mysql.connector.Error as err : self.lastrow_mutex.release() _msg = "Unable to insert document into table \"" + table + "\": " _msg += str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 6) except Exception as e : self.lastrow_mutex.release() raise MetricStoreMgdConnException(str(e), 64) self.lastrow_mutex.release() return lastrowid
def connect(self, tout) : try: #if tout and tout > 0: # MysqlMgdConn.conn.set_connection_timeout(tout) if not MysqlMgdConn.catalogs.cbtool["conn"] : cbdebug("Opening to: " + self.database) MysqlMgdConn.catalogs.cbtool["conn"] = mysql.connector.connect(host = self.host, port = self.port, user = self.username, password = self.password) cursor = MysqlMgdConn.catalogs.cbtool["conn"].cursor() try : cursor.execute("use " + self.database) MysqlMgdConn.catalogs.cbtool["database"] = True except mysql.connector.Error as err : if err.errno == mysql.connector.errorcode.ER_BAD_DB_ERROR: cbwarn("Database not found. Will create later.") cursor.close() _msg = "A connection to MySQL running on host " _msg += self.host + ", port " + str(self.port) + ", database" _msg += ' ' + str(MysqlMgdConn.catalogs.cbtool["database"]) + ", with a timeout of " _msg += str(tout) + "s was established." cbdebug(_msg) except mysql.connector.Error as err : if err.errno == mysql.connector.errorcode.ER_ACCESS_DENIED_ERROR: _msg = "Something is wrong with your MySQL user name or password." cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1) else: _msg = "Unknown MySQL error: " + str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 2)
def conn_check(self, hostov=False, dbov=False, tout=False): ''' TBD ''' if not self.mongodb_conn: if hostov: self.host = hostov if dbov: self.database = dbov if tout: self.timeout = tout try: self.connect(self.timeout) except MetricStoreMgdConnException as obj: raise MetricStoreMgdConnException(obj.msg, 2) if self.password and len(self.password) > 2 and str( self.password).lower() != "false": try: _auth_cmd = "mongo -u \"<YOUR ADMIN\" -p \"<YOUR ADMINPASS>\" " _auth_cmd += "--authenticationDatabase \"admin\" --eval " _auth_cmd += "\"db.createUser({user: '******', pwd: '" + self.password + "', roles: [ { role:" _auth_cmd += " 'readWrite', db: 'metrics' } ]})\" " _auth_cmd += self.host + ":" + str( self.port) + '/' + self.database self.mongodb_conn[self.database].authenticate( self.username, self.password, mechanism='MONGODB-CR') except PymongoException as errmsg: _msg = "Unable to authenticate against the database \"" + self.database _msg += "\":" + str( errmsg ) + ". \nPlease create the user there (i.e., directly on " _msg += self.host + ") using the following command:\n" _msg += _auth_cmd raise MetricStoreMgdConnException(_msg, 2) except Exception as e: _msg = "Unable to authenticate against the database \"" + self.database _msg += "\":" + str( e ) + ". \nPlease create the user there (i.e., directly on " _msg += self.host + ") using the following command:\n" _msg += _auth_cmd raise MetricStoreMgdConnException(_msg, 2)
def update_document(self, table, document, disconnect_finish = False) : table = table.replace('-',"dash") self.update_mutex.acquire() try : cursor = self.conn_check() if "_id" in document and isinstance(document["_id"], bytes) : document["_id"] = document["_id"].decode("utf-8") if "original_mysql_id" not in document : if "_id" in document : # Attempt to find the original ID first statement = "select id from " + table + " where _id = '" + document["_id"] + "'" cursor.execute(statement) while True : rows = cursor.fetchmany(1) if not len(rows) : break for (original_mysql_id,) in rows : document["original_mysql_id"] = original_mysql_id if "original_mysql_id" not in document : cursor.close() cbwarn("This document does not have a pre-existing identifier. Cannot update. Will insert first") document["original_mysql_id"] = self.add_document(table, document, disconnect_finish = disconnect_finish) self.update_mutex.release() return statement = "update " + table + " set document = '" + json.dumps(document) + "' where id = " + str(document["original_mysql_id"]) result = cursor.execute(statement) cursor.close() MysqlMgdConn.catalogs.cbtool["conn"].commit() if disconnect_finish : self.disconnect() except mysql.connector.Error as err : self.update_mutex.release() _msg = "Unable to update documents from the table \"" _msg += table + ": " + str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 8) except Exception as e : self.update_mutex.release() cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 67) self.update_mutex.release()
def get_info(self): ''' TBD ''' self.conn_check() try: _buildinfo = self.mongodb_conn[self.database].command("buildinfo") _dbstats = self.mongodb_conn[self.database].command("dbstats") _output = [] _output.append(["MongoDB Version", _buildinfo["version"]]) _output.append(["Storage Size", str(_dbstats["storageSize"])]) #_output.append(["File Size", _buildinfo["fileSize"]]) _output.append(["Data Size", str(_dbstats["dataSize"])]) _output.append(["Index Size", str(_dbstats["indexSize"])]) _output.append( ["Average Object Size", str(_dbstats["avgObjSize"])]) _output.append(["Collections", str(_dbstats["collections"])]) return _output except PymongoException as msg: _msg = "Unable to get info database " + self.database + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def get_time_boundaries(self, collection, disconnect_finish=False): ''' TBD ''' self.conn_check() collection = collection.replace('-', "dash") try: _collection_handle = self.mongodb_conn[self.database][collection] _start_time = _collection_handle.find(spec={}, fields={"time": "1"}, sort=[("time", 1)], limit=1)[0]["time"] _end_time = _collection_handle.find(spec={}, fields={"time": "1"}, sort=[("time", -1)], limit=1)[0]["time"] if disconnect_finish: self.disconnect() return _start_time, _end_time except PymongoException as msg: _msg = "Unable to get time boundaries on the collection \"" _msg += collection + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def get_info(self) : try : _output = [] cursor = self.conn_check() cursor.execute("show variables") while True : rows = cursor.fetchmany(4) if not len(rows) : break for row in rows : if row[0] in ["version"] : _output.append([row[0], row[1]]) cursor.execute("select sum(data_length + index_length)/1024/1024 'size' FROM information_schema.TABLES") while True : rows = cursor.fetchmany(4) if not len(rows) : break for row in rows : _output.append(["Data Size (MB)", str(float(row[0]))]) cursor.close() return _output except mysql.connector.Error as err : _msg = "Unable to get info for database " + self.database + ": " _msg += str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 15) except Exception as e : cbdebug("No workey: " + str(e))
def get_experiment_list(self, table, disconnect_finish = False) : table = table.replace('-',"dash") _experiment_list = [] try : cursor = self.conn_check() statement = "select distinct(expid) from " + table + " where expid is not NULL" cursor.execute(statement) while True : rows = cursor.fetchmany(4) if not len(rows) : break for (expid) in rows : _experiment_list.append(expid) cursor.close() if disconnect_finish : self.disconnect() return _experiment_list except mysql.connector.Error as err : _msg = "Unable to get time experiment list for table \"" _msg += table + ": " + str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 14)
def delete_document(self, collection, criteria, disconnect_finish=False): ''' TBD ''' self.conn_check() collection = collection.replace('-', "dash") try: _collection_handle = self.mongodb_conn[self.database][collection] if int(self.version) < 3: _collection_handle.remove(criteria) else: _collection_handle.delete_one(criteria) if disconnect_finish: self.disconnect() except PymongoException as msg: _msg = "Unable to remove document from the collection \"" _msg += collection + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def update_document(self, collection, document, disconnect_finish=False): ''' TBD ''' self.conn_check() collection = collection.replace('-', "dash") try: _collection_handle = self.mongodb_conn[self.database][collection] if int(self.version) < 3: _collection_handle.save(document) else: # This insane behavior is supposed to be the "expected behavior" # according to https://jira.mongodb.org/browse/SERVER-14322 try: _collection_handle.replace_one({'_id': document["_id"]}, document, upsert=True) except: _collection_handle.replace_one({'_id': document["_id"]}, document, upsert=True) if disconnect_finish: self.disconnect() except PymongoException as msg: _msg = "Unable to update documents from the collection \"" _msg += collection + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def add_document(self, collection, document, disconnect_finish=False): ''' TBD ''' self.conn_check() collection = collection.replace('-', "dash") try: _collection_handle = self.mongodb_conn[self.database][collection] if int(self.version) < 3: _collection_handle.insert(document) else: _collection_handle.insert_one(document) if disconnect_finish: self.disconnect() return True except PymongoException as msg: _msg = "Unable to insert document \"" + document _msg += "\" on collection \"" + collection + "\": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def get_experiment_list(self, collection, disconnect_finish=False): ''' TBD ''' self.conn_check() collection = collection.replace('-', "dash") _experiment_list = None try: _collection_handle = self.mongodb_conn[self.database][collection] #_experiment_list = _collection_handle.distinct('expid') # The document is getting too big, but a workaround was found. # TODO: Find a more permanent solution to this. _experiment_list_agg = _collection_handle.aggregate([{ "$group": { "_id": '$expid' } }]) _experiment_list = ([_v['_id'] for _v in _experiment_list_agg]) if disconnect_finish: self.disconnect() return _experiment_list except PymongoException as msg: _msg = "Unable to get time boundaries on the collection \"" _msg += collection + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def connect(self, tout): ''' TBD ''' try: if tout > 0: _conn = MongoClient(host = self.host, port = self.port, \ maxPoolSize=10) else: _conn = MongoClient(host = self.host, port = self.port, \ max_pool_size=10) self.mongodb_conn = _conn _msg = "A connection to MongoDB running on host " _msg += self.host + ", port " + str(self.port) + ", database" _msg += ' ' + str(self.database) + ", with a timeout of " _msg += str(tout) + "s was established." cbdebug(_msg) return self.mongodb_conn except PymongoException as msg: True except: True # This was added here just because some MongoDBs don't accept the # "max_pool_size" parameter try: if tout > 0: _conn = MongoClient(host=self.host, port=self.port) else: _conn = MongoClient(host=self.host, port=self.port) self.mongodb_conn = _conn _msg = "A connection to MongoDB running on host " _msg += self.host + ", port " + str(self.port) + ", database" _msg += ' ' + str(self.database) + ", with a timeout of " _msg += str(tout) + "s was established." cbdebug(_msg) return self.mongodb_conn except PymongoException as msg: _msg = "Unable to establish a connection with the MongoDB " _msg += "server on host " + self.host + " port " _msg += str(self.port) + "database " + str(self.database) + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def initialize_metric_store(self, username): ''' TBD ''' self.conn_check() username = username.replace('-', "dash") try: _collections = [ \ "latest_management_VM_" + username, \ "latest_management_HOST_" + username, \ "latest_runtime_os_VM_" + username, \ "latest_runtime_os_HOST_" + username, \ "latest_runtime_app_VM_" + username \ ] for _collection in _collections: _collection_handle = self.mongodb_conn[ self.database][_collection] #_collection_handle.drop() _collections = [ "trace_" + username, \ "management_HOST_" + username, \ "management_VM_" + username, \ "runtime_os_VM_" + username, \ "runtime_app_VM_" + username, \ "runtime_os_HOST_" + username ] for _collection in _collections: _collection_handle = self.mongodb_conn[ self.database][_collection] if int(self.version) < 3: _collection_handle.ensure_index("dashboard_polled") _collection_handle.ensure_index("expid") _collection_handle.ensure_index("time") _collection_handle.ensure_index("uuid") else: _collection_handle.create_index("dashboard_polled") _collection_handle.create_index("expid") _collection_handle.create_index("time") _collection_handle.create_index("uuid") self.disconnect() return True except PymongoException as msg: _msg = "Unable to initialize all documents on " _msg += "\" on collection \"" + _collection + "\": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def count_document(self, table, criteria, disconnect_finish = False) : table = table.replace('-',"dash") try : cursor = self.conn_check() statement = "select * from " + table + self.make_restrictions(criteria) count = cursor.execute(statement).rowcount cursor.close() if disconnect_finish : self.disconnect() return count except mysql.connector.Error as err : _msg = "Unable to count documents on the table \"" _msg += table + ": " + str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 11)
def disconnect(self) : try: if "disconnect" in dir(MysqlMgdConn.catalogs.cbtool["conn"]) : MysqlMgdConn.catalogs.cbtool["conn"].disconnect() MysqlMgdConn.catalogs.cbtool["conn"] = False _msg = "A connection to MySQL running on host " _msg += self.host + ", port " + str(self.port) + ", database" _msg += ' ' + str(MysqlMgdConn.catalogs.cbtool["database"]) + ", was terminated." cbdebug(_msg) except mysql.connector.Error as err : _msg = "Unable to terminate a connection with MySQL " _msg += "server on host " + self.host + " port " _msg += str(self.port) + "database " + str(MysqlMgdConn.catalogs.cbtool["database"]) + ": " _msg += str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 3)
def cleanup_collection(self, table, disconnect_finish = False) : table = table.replace('-',"dash") try : cursor = self.conn_check() statement = "delete from " + table cursor.execute(statement) cursor.close() MysqlMgdConn.catalogs.cbtool["conn"].commit() if disconnect_finish : self.disconnect() return True except mysql.connector.Error as err : _msg = "Unable to drop all documents from the table \"" _msg += table + ": " + str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 10)
def delete_document(self, table, criteria, disconnect_finish = False) : table = table.replace('-',"dash") try : cursor = self.conn_check() statement = "delete from " + table + self.make_restrictions(criteria) cursor.execute(statement) cursor.close() MysqlMgdConn.catalogs.cbtool["conn"].commit() if disconnect_finish : self.disconnect() except mysql.connector.Error as err : _msg = "Unable to remove document from the table \"" _msg += table + ": " + str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 9)
def flush_metric_store(self, username, partial=False, criteria={}): ''' TBD ''' self.conn_check() username = username.replace('-', "dash") try: _collections = ["latest_management_VM_" + username, \ "latest_management_HOST_" + username, \ "latest_runtime_os_VM_" + username, \ "latest_runtime_os_HOST_" + username, \ "latest_runtime_app_VM_" + username, \ "trace_" + username, \ "management_HOST_" + username, \ "management_VM_" + username, \ "runtime_os_VM_" + username, \ "runtime_app_VM_" + username, \ "runtime_os_HOST_" + username, \ "reported_management_VM_metric_names_" + username, \ "reported_runtime_app_VM_metric_names_" + username, \ "reported_runtime_os_HOST_metric_names_" + username, \ "reported_runtime_os_VM_metric_names_" + username ] for _collection in _collections: _collection_handle = self.mongodb_conn[ self.database][_collection] if partial: _collection_handle.remove(criteria) else: _collection_handle.drop() self.disconnect() return True except PymongoException as msg: _msg = "Unable to initialize all documents on " _msg += "\" on collection \"" + _collection + "\": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def get_reported_objects(self, collection, disconnect_finish=False): ''' TBD ''' self.conn_check() collection = collection.replace('-', "dash") try: _result = {} _attributes = [ "vm_name", "role", "ai_name", "type", "aidrs_name", "pattern" ] for _attribute in _attributes: _result[_attribute + 's'] = [] _collection_handle = self.mongodb_conn[self.database][collection] _documents = _collection_handle.find() for _document in _documents: for _attribute in _attributes: if _attribute == "vm_name": _attribute_r = "name" else: _attribute_r = _attribute if _attribute_r in _document: if not _result[_attribute + 's'].count( _document[_attribute_r]): _result[_attribute + 's'].append( _document[_attribute_r]) if disconnect_finish: self.disconnect() return _result except PymongoException as msg: _msg = "Unable to get reported attributes on the collection \"" _msg += collection + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def cleanup_collection(self, collection, disconnect_finish=False): ''' TBD ''' self.conn_check() collection = collection.replace('-', "dash") try: _collection_handle = self.mongodb_conn[self.database][collection] _collection_handle.drop() if disconnect_finish: self.disconnect() return True except PymongoException as msg: _msg = "Unable to drop all documents from the collection \"" _msg += collection + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def count_document(self, collection, criteria, disconnect_finish=False): ''' TBD ''' self.conn_check() collection = collection.replace('-', "dash") try: _collection_handle = self.mongodb_conn[self.database][collection] _matches = _collection_handle.find(criteria) if disconnect_finish: self.disconnect() return _matches.count() except PymongoException as msg: _msg = "Unable to count documents on the collection \"" _msg += collection + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def disconnect(self): ''' TBD ''' try: if "disconnect" in dir(self.mongodb_conn): self.mongodb_conn.disconnect() self.mongodb_conn = False _msg = "A connection to MongoDB running on host " _msg += self.host + ", port " + str(self.port) + ", database" _msg += ' ' + str(self.database) + ", was terminated." cbdebug(_msg) return self.mongodb_conn except PymongoException as msg: _msg = "Unable to terminate a connection with the MongoDB " _msg += "server on host " + self.host + " port " _msg += str(self.port) + "database " + str(self.database) + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def find_document(self, collection, criteria, allmatches = False, \ sortkeypairs = None, limitdocuments = 0, \ documentfields = None, disconnect_finish = False) : ''' TBD ''' self.conn_check() collection = collection.replace('-', "dash") try: _collection_handle = self.mongodb_conn[self.database][collection] if allmatches: _results = _collection_handle.find(criteria, \ sort = sortkeypairs, \ limit = limitdocuments, \ projection = documentfields) else: _results = _collection_handle.find_one(criteria, \ sort = sortkeypairs, \ projection = documentfields) if disconnect_finish: self.disconnect() return _results except PymongoException as msg: _msg = "Unable to retrieve documents from the collection \"" _msg += collection + ": " _msg += str(msg) + '.' cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 1)
def conn_check(self, hostov = False, dbov = False, tout = False) : self.conn_mutex.acquire() try : getattr(MysqlMgdConn.catalogs, "cbtool") except AttributeError as e : cbdebug("Initializing thread local connection: ") MysqlMgdConn.catalogs.cbtool = {} if "database" not in MysqlMgdConn.catalogs.cbtool : MysqlMgdConn.catalogs.cbtool["database"] = False if "conn" not in MysqlMgdConn.catalogs.cbtool : MysqlMgdConn.catalogs.cbtool["conn"] = False if not MysqlMgdConn.catalogs.cbtool["conn"] or not MysqlMgdConn.catalogs.cbtool["conn"].is_connected() : MysqlMgdConn.catalogs.cbtool["conn"] = False if hostov : self.host = hostov if dbov : MysqlMgdConn.catalogs.cbtool["database"] = dbov if tout : self.timeout = tout try : self.connect(self.timeout) except MetricStoreMgdConnException as obj : self.conn_mutex.release() raise MetricStoreMgdConnException(obj.msg, 2) except Exception as e : self.conn_mutex.release() raise(e) assert(MysqlMgdConn.catalogs.cbtool["conn"]) assert(MysqlMgdConn.catalogs.cbtool["conn"].is_connected()) cursor = MysqlMgdConn.catalogs.cbtool["conn"].cursor() self.conn_mutex.release() return cursor
def flush_metric_store(self, username, partial = False, criteria = {}) : username = username.replace('-',"dash") try : cursor = self.conn_check() _tables = ["latest_management_VM_" + username, \ "latest_management_HOST_" + username, \ "latest_runtime_os_VM_" + username, \ "latest_runtime_os_HOST_" + username, \ "latest_runtime_app_VM_" + username, \ "trace_" + username, \ "management_HOST_" + username, \ "management_VM_" + username, \ "runtime_os_VM_" + username, \ "runtime_app_VM_" + username, \ "runtime_os_HOST_" + username, \ "reported_management_VM_metric_names_" + username, \ "reported_runtime_app_VM_metric_names_" + username, \ "reported_runtime_os_HOST_metric_names_" + username, \ "reported_runtime_os_VM_metric_names_" + username ] for _table in _tables : if partial and len(criteria) : statement = "delete from " + _table + self.make_restrictions(criteria) cursor.execute(statement) else : cursor.execute("delete from " + _table) cursor.close() MysqlMgdConn.catalogs.cbtool["conn"].commit() self.disconnect() return True except mysql.connector.Error as err : self.disconnect() _msg = "Unable to flush metric store: " + str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 5)
def get_reported_objects(self, table, disconnect_finish = False) : table = table.replace('-',"dash") try : cursor = self.conn_check() _result = {} _attributes = [ "vm_name", "role", "ai_name", "type", "aidrs_name", "pattern" ] for _attribute in _attributes : _result[_attribute + 's'] = [] statement = "select id, document from " + table cursor.execute(statement) while True : rows = cursor.fetchmany(4) if not len(rows) : break for (original_mysql_id, _document) in rows : for _attribute in _attributes : if _attribute == "vm_name" : _attribute_r = "name" else : _attribute_r = _attribute if _attribute_r in _document : if not _result[_attribute + 's'].count(_document[_attribute_r]) : _result[_attribute + 's'].append(_document[_attribute_r]) cursor.close() if disconnect_finish : self.disconnect() return _result except mysql.connector.Error as err : _msg = "Unable to get reported attributes on the table \"" _msg += table + ": " + str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 12)
def initialize_metric_store(self, username) : username = username.replace('-',"dash") try : cursor = self.conn_check() if not MysqlMgdConn.catalogs.cbtool["database"] : cursor.execute("create database " + self.database) cursor.execute("use " + self.database) MysqlMgdConn.catalogs.cbtool["database"] = True _latest_tables = [ \ "latest_management_VM_" + username, \ "latest_management_HOST_" + username, \ "latest_runtime_os_VM_" + username, \ "latest_runtime_os_HOST_" + username, \ "latest_runtime_app_VM_" + username, \ "reported_management_VM_metric_names_" + username, \ "reported_runtime_app_VM_metric_names_" + username, \ "reported_runtime_os_HOST_metric_names_" + username, \ "reported_runtime_os_VM_metric_names_" + username \ ] _indexed_tables = [ "trace_" + username, \ "management_HOST_" + username, \ "management_VM_" + username, \ "runtime_os_VM_" + username, \ "runtime_app_VM_" + username, \ "runtime_os_HOST_" + username ] cursor.execute("show tables") _tables_found = [] for x in cursor: _tables_found.append(x[0]) for _table in (_latest_tables + _indexed_tables) : if _table not in _tables_found : statement = "create table " + _table + "(" + \ "id int auto_increment primary key," + \ "document json NOT NULL," + \ "`expid` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.expid')," + \ "`_id` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$._id')," + \ "`time` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.time')," + \ "`uuid` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.uuid')," + \ "`dashboard_polled` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.dashboard_polled')" + \ ")" cursor.execute(statement) if _table in _indexed_tables : cursor.execute("CREATE INDEX `expid_idx` ON `" + _table + "`(`expid`)") cursor.execute("CREATE INDEX `time_idx` ON `" + _table + "`(`time`)") cursor.execute("CREATE INDEX `uuid_idx` ON `" + _table + "`(`uuid`)") cursor.execute("CREATE INDEX `dashboard_polled_idx` ON `" + _table + "`(`dashboard_polled`)") cursor.close() MysqlMgdConn.catalogs.cbtool["conn"].commit() self.disconnect() return True except mysql.connector.Error as err : self.disconnect() _msg = "Unable to complete database initialization: " _msg += str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 4)
def make_restrictions(self, criteria, join = "and", level = 0) : full_list = "" restrictions = [] for _key in criteria.keys() : _value = criteria[_key] if isinstance(_value, set) : _msg = "1) We cannot yet handle this criteria: " + str(criteria) cberr(_msg) raise MetricStoreMgdConnException(_msg, 41) elif isinstance(_value, dict) : for subkey in _value.keys() : if subkey.lower() == "$exists" : if not isinstance(_value[subkey], bool) : _msg = "2) We cannot yet handle this criteria: " + str(_value) cberr(_msg) raise MetricStoreMgdConnException(_msg, 41) if _value[subkey] : restrictions.append("document->>'$." + _key + "' IS NOT NULL") else : restrictions.append("document->>'$." + _key + "' IS NULL") else : _msg = "3) We cannot yet handle this criteria: " + str(subkey) cberr(_msg) raise MetricStoreMgdConnException(_msg, 41) elif isinstance(_value, list) : # Handle this group below continue else : _newvalue = _value if isinstance(_value, bytes) : _newvalue = _value.decode("utf-8") restrictions.append("document->>'$." + _key + "' = '" + str(_newvalue) + "'") if len(restrictions) : full_list += (" " + join + " ").join(restrictions) for _key in criteria.keys() : if _key.lower() == "$or" or _key.lower() == "$and" : _value = criteria[_key] if isinstance(_value, list) : subdict = {} for subitem in _value : if not isinstance(subitem, dict) : _msg = "4) We cannot yet handle this criteria: " + str(subitem) cberr(_msg) raise MetricStoreMgdConnException(_msg, 41) subdict.update(subitem) sub_restrictions = self.make_restrictions(subdict, join = _key[1:], level = level + 1) if sub_restrictions.strip() != "" : full_list += " and (" + sub_restrictions + ")" else : _msg = "5) We cannot yet handle this criteria: " + str(_value) cberr(_msg) raise MetricStoreMgdConnException(_msg, 41) if full_list.strip() != "" : if level == 0 : return " where " + full_list else : return full_list return ""
def find_document(self, table, criteria, allmatches = False, \ sortkeypairs = None, limitdocuments = 0, \ documentfields = None, disconnect_finish = False) : table = table.replace('-',"dash") try : cursor = self.conn_check() statement = "select " if documentfields is not None : convertedfields = [] for field in documentfields : convertedfields.append("document->>'$." + field + "'") statement += ",".join(["id"] + convertedfields) else : statement += " id,document " statement += " from " + table + " " + self.make_restrictions(criteria) if sortkeypairs : keylist = [] for keypair in sortkeypairs : # FIXME: I'm unsure of how to have different directional sort criteria for multiple # sorted keys. Will have to look into that later, so for the time being, # I'm dropping the direction. keylist.append("document->>'$." + keypair[0] + "'") statement += " order by " + ",".join(keylist) if not allmatches or limitdocuments : if limitdocuments > 0 : statement += " limit " + str(limitdocuments) else : statement += " limit 1" _results = [] # FIXME: We need to figure out how to safely allow iterators over # the live connection. But for now, let's just extract all the results result = cursor.execute(statement) while True : rows = cursor.fetchmany(4) if not len(rows) : break for resultset in rows : original_mysql_id = resultset[0] document = False if documentfields is not None : document = {} for idx in range(1, len(resultset)) : document[documentfields[idx - 1]] = resultset[idx].decode() else : if isinstance(resultset[1], str) : document = json.loads(resultset[1]) else : assert(isinstance(resultset[1], dict)) document = resultset[1] document["original_mysql_id"] = original_mysql_id _results.append(document) cursor.close() if allmatches : return _results else : if len(_results) >= 1 : return _results[0] return None except mysql.connector.Error as err : _msg = "Unable to retrieve documents from the table \"" _msg += table + ": " + str(err) cberr(_msg) raise MetricStoreMgdConnException(str(_msg), 7)