Ejemplo n.º 1
0
    def add_document(self, table, document, disconnect_finish = False) :
        table = table.replace('-',"dash")
        self.lastrow_mutex.acquire()
        lastrowid = -1

        try :
            cursor = self.conn_check()

            if "_id" in document and isinstance(document["_id"], bytes) :
                document["_id"] = document["_id"].decode("utf-8")
            statement = "insert into " + table + " (document) values ('" + json.dumps(document) + "')"
            result = cursor.execute(statement)
            if cursor.rowcount != 1 :
                MysqlMgdConn.catalogs.cbtool["conn"].rollback()
                raise MetricStoreMgdConnException("Add failed w/ statement: " + statement, 65)
            cursor.close()
            MysqlMgdConn.catalogs.cbtool["conn"].commit()
            lastrowid = cursor.lastrowid
            if disconnect_finish :
                self.disconnect()

        except mysql.connector.Error as err :
            self.lastrow_mutex.release()
            _msg = "Unable to insert document into table \"" + table + "\": " 
            _msg += str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 6)
        except Exception as e :
            self.lastrow_mutex.release()
            raise MetricStoreMgdConnException(str(e), 64)

        self.lastrow_mutex.release()
        return lastrowid 
Ejemplo n.º 2
0
    def connect(self, tout) :
        try:
            #if tout and tout > 0:            
            #    MysqlMgdConn.conn.set_connection_timeout(tout)

            if not MysqlMgdConn.catalogs.cbtool["conn"] :
                cbdebug("Opening to: " + self.database)
                MysqlMgdConn.catalogs.cbtool["conn"] = mysql.connector.connect(host = self.host, port = self.port, user = self.username, password = self.password)
                cursor = MysqlMgdConn.catalogs.cbtool["conn"].cursor()
                try :
                    cursor.execute("use " + self.database)
                    MysqlMgdConn.catalogs.cbtool["database"] = True 
                except mysql.connector.Error as err :
                    if err.errno == mysql.connector.errorcode.ER_BAD_DB_ERROR:
                        cbwarn("Database not found. Will create later.")
                cursor.close()

            _msg = "A connection to MySQL running on host "
            _msg += self.host + ", port " + str(self.port) + ", database"
            _msg += ' ' + str(MysqlMgdConn.catalogs.cbtool["database"]) + ", with a timeout of "
            _msg += str(tout) + "s was established."
            cbdebug(_msg)

        except mysql.connector.Error as err :
            if err.errno == mysql.connector.errorcode.ER_ACCESS_DENIED_ERROR:
                _msg = "Something is wrong with your MySQL user name or password."
                cberr(_msg)
                raise MetricStoreMgdConnException(str(_msg), 1)
            else:
                _msg = "Unknown MySQL error: " + str(err)
                cberr(_msg)
                raise MetricStoreMgdConnException(str(_msg), 2)
Ejemplo n.º 3
0
    def conn_check(self, hostov=False, dbov=False, tout=False):
        '''
        TBD
        '''
        if not self.mongodb_conn:
            if hostov:
                self.host = hostov

            if dbov:
                self.database = dbov

            if tout:
                self.timeout = tout

            try:
                self.connect(self.timeout)

            except MetricStoreMgdConnException as obj:
                raise MetricStoreMgdConnException(obj.msg, 2)

            if self.password and len(self.password) > 2 and str(
                    self.password).lower() != "false":
                try:
                    _auth_cmd = "mongo -u \"<YOUR ADMIN\" -p \"<YOUR ADMINPASS>\" "
                    _auth_cmd += "--authenticationDatabase \"admin\" --eval "
                    _auth_cmd += "\"db.createUser({user: '******', pwd: '" + self.password + "', roles: [ { role:"
                    _auth_cmd += " 'readWrite', db: 'metrics' } ]})\" "
                    _auth_cmd += self.host + ":" + str(
                        self.port) + '/' + self.database

                    self.mongodb_conn[self.database].authenticate(
                        self.username, self.password, mechanism='MONGODB-CR')

                except PymongoException as errmsg:
                    _msg = "Unable to authenticate against the database \"" + self.database
                    _msg += "\":" + str(
                        errmsg
                    ) + ". \nPlease create the user there (i.e., directly on "
                    _msg += self.host + ") using the following command:\n"
                    _msg += _auth_cmd
                    raise MetricStoreMgdConnException(_msg, 2)

                except Exception as e:
                    _msg = "Unable to authenticate against the database \"" + self.database
                    _msg += "\":" + str(
                        e
                    ) + ". \nPlease create the user there (i.e., directly on "
                    _msg += self.host + ") using the following command:\n"
                    _msg += _auth_cmd
                    raise MetricStoreMgdConnException(_msg, 2)
Ejemplo n.º 4
0
    def update_document(self, table, document, disconnect_finish = False) :
        table = table.replace('-',"dash")

        self.update_mutex.acquire()
        try :
            cursor = self.conn_check()

            if "_id" in document and isinstance(document["_id"], bytes) :
                document["_id"] = document["_id"].decode("utf-8")

            if "original_mysql_id" not in document :
                if "_id" in document :
                    # Attempt to find the original ID first
                    statement = "select id from " + table + " where _id = '" + document["_id"] + "'"
                    cursor.execute(statement)
                    while True :
                        rows = cursor.fetchmany(1)
                        if not len(rows) :
                            break
                        for (original_mysql_id,) in rows :
                            document["original_mysql_id"] = original_mysql_id

                if "original_mysql_id" not in document :
                    cursor.close()
                    cbwarn("This document does not have a pre-existing identifier. Cannot update. Will insert first")
                    document["original_mysql_id"] = self.add_document(table, document, disconnect_finish = disconnect_finish)
                    self.update_mutex.release()
                    return

            statement = "update " + table + " set document = '" + json.dumps(document) + "' where id = " + str(document["original_mysql_id"])
            result = cursor.execute(statement)
            cursor.close()
            MysqlMgdConn.catalogs.cbtool["conn"].commit()

            if disconnect_finish :
                self.disconnect()

        except mysql.connector.Error as err :
            self.update_mutex.release()
            _msg = "Unable to update documents from the table \""
            _msg += table + ": " + str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 8)
        except Exception as e :
            self.update_mutex.release()
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 67)

        self.update_mutex.release()
Ejemplo n.º 5
0
    def get_info(self):
        '''
        TBD
        '''
        self.conn_check()

        try:
            _buildinfo = self.mongodb_conn[self.database].command("buildinfo")
            _dbstats = self.mongodb_conn[self.database].command("dbstats")

            _output = []
            _output.append(["MongoDB Version", _buildinfo["version"]])
            _output.append(["Storage Size", str(_dbstats["storageSize"])])
            #_output.append(["File Size", _buildinfo["fileSize"]])
            _output.append(["Data Size", str(_dbstats["dataSize"])])
            _output.append(["Index Size", str(_dbstats["indexSize"])])
            _output.append(
                ["Average Object Size",
                 str(_dbstats["avgObjSize"])])
            _output.append(["Collections", str(_dbstats["collections"])])

            return _output

        except PymongoException as msg:
            _msg = "Unable to get info database " + self.database + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 6
0
    def get_time_boundaries(self, collection, disconnect_finish=False):
        '''
        TBD
        '''
        self.conn_check()

        collection = collection.replace('-', "dash")

        try:
            _collection_handle = self.mongodb_conn[self.database][collection]
            _start_time = _collection_handle.find(spec={},
                                                  fields={"time": "1"},
                                                  sort=[("time", 1)],
                                                  limit=1)[0]["time"]
            _end_time = _collection_handle.find(spec={},
                                                fields={"time": "1"},
                                                sort=[("time", -1)],
                                                limit=1)[0]["time"]

            if disconnect_finish:
                self.disconnect()

            return _start_time, _end_time

        except PymongoException as msg:
            _msg = "Unable to get time boundaries on the collection \""
            _msg += collection + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 7
0
    def get_info(self) :
        try :
            _output = []
            cursor = self.conn_check()
            cursor.execute("show variables")
            while True :
                rows = cursor.fetchmany(4)
                if not len(rows) :
                    break
                for row in rows :
                    if row[0] in ["version"] : 
                        _output.append([row[0], row[1]])
            
            cursor.execute("select sum(data_length + index_length)/1024/1024 'size' FROM information_schema.TABLES")
            while True :
                rows = cursor.fetchmany(4)
                if not len(rows) :
                    break
                for row in rows :
                    _output.append(["Data Size (MB)", str(float(row[0]))])

            cursor.close()
            return _output

        except mysql.connector.Error as err :
            _msg = "Unable to get info for database " + self.database + ": " 
            _msg += str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 15)
        except Exception as e :
            cbdebug("No workey: " + str(e))
Ejemplo n.º 8
0
    def get_experiment_list(self, table, disconnect_finish = False) :
        table = table.replace('-',"dash")
        _experiment_list = [] 

        try :
            cursor = self.conn_check()
            
            statement = "select distinct(expid) from " + table + " where expid is not NULL"
            cursor.execute(statement)

            while True :
                rows = cursor.fetchmany(4)
                if not len(rows) :
                    break
                for (expid) in rows :
                    _experiment_list.append(expid)

            cursor.close()
            if disconnect_finish :
                self.disconnect()

            return _experiment_list

        except mysql.connector.Error as err :
            _msg = "Unable to get time experiment list for table \""
            _msg += table + ": " + str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 14)
Ejemplo n.º 9
0
    def delete_document(self, collection, criteria, disconnect_finish=False):
        '''
        TBD
        '''
        self.conn_check()

        collection = collection.replace('-', "dash")

        try:
            _collection_handle = self.mongodb_conn[self.database][collection]

            if int(self.version) < 3:
                _collection_handle.remove(criteria)
            else:
                _collection_handle.delete_one(criteria)

            if disconnect_finish:
                self.disconnect()

        except PymongoException as msg:
            _msg = "Unable to remove document from the collection \""
            _msg += collection + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 10
0
    def update_document(self, collection, document, disconnect_finish=False):
        '''
        TBD
        '''
        self.conn_check()

        collection = collection.replace('-', "dash")

        try:
            _collection_handle = self.mongodb_conn[self.database][collection]

            if int(self.version) < 3:
                _collection_handle.save(document)
            else:
                # This insane behavior is supposed to be the "expected behavior"
                # according to https://jira.mongodb.org/browse/SERVER-14322
                try:
                    _collection_handle.replace_one({'_id': document["_id"]},
                                                   document,
                                                   upsert=True)
                except:
                    _collection_handle.replace_one({'_id': document["_id"]},
                                                   document,
                                                   upsert=True)

            if disconnect_finish:
                self.disconnect()

        except PymongoException as msg:
            _msg = "Unable to update documents from the collection \""
            _msg += collection + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 11
0
    def add_document(self, collection, document, disconnect_finish=False):
        '''
        TBD
        '''
        self.conn_check()

        collection = collection.replace('-', "dash")

        try:
            _collection_handle = self.mongodb_conn[self.database][collection]

            if int(self.version) < 3:
                _collection_handle.insert(document)
            else:
                _collection_handle.insert_one(document)
            if disconnect_finish:
                self.disconnect()
            return True

        except PymongoException as msg:
            _msg = "Unable to insert document \"" + document
            _msg += "\" on collection \"" + collection + "\": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 12
0
    def get_experiment_list(self, collection, disconnect_finish=False):
        '''
        TBD
        '''
        self.conn_check()

        collection = collection.replace('-', "dash")

        _experiment_list = None
        try:
            _collection_handle = self.mongodb_conn[self.database][collection]

            #_experiment_list = _collection_handle.distinct('expid')

            # The document is getting too big, but a workaround was found.
            # TODO: Find a more permanent solution to this.
            _experiment_list_agg = _collection_handle.aggregate([{
                "$group": {
                    "_id": '$expid'
                }
            }])
            _experiment_list = ([_v['_id'] for _v in _experiment_list_agg])

            if disconnect_finish:
                self.disconnect()

            return _experiment_list

        except PymongoException as msg:
            _msg = "Unable to get time boundaries on the collection \""
            _msg += collection + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 13
0
    def connect(self, tout):
        '''
        TBD
        '''
        try:

            if tout > 0:

                _conn = MongoClient(host = self.host, port = self.port, \
                                    maxPoolSize=10)
            else:
                _conn = MongoClient(host = self.host, port = self.port, \
                                    max_pool_size=10)

            self.mongodb_conn = _conn

            _msg = "A connection to MongoDB running on host "
            _msg += self.host + ", port " + str(self.port) + ", database"
            _msg += ' ' + str(self.database) + ", with a timeout of "
            _msg += str(tout) + "s was established."
            cbdebug(_msg)
            return self.mongodb_conn

        except PymongoException as msg:
            True

        except:
            True
        # This was added here just because some MongoDBs don't accept the
        # "max_pool_size" parameter
        try:

            if tout > 0:
                _conn = MongoClient(host=self.host, port=self.port)
            else:
                _conn = MongoClient(host=self.host, port=self.port)

            self.mongodb_conn = _conn

            _msg = "A connection to MongoDB running on host "
            _msg += self.host + ", port " + str(self.port) + ", database"
            _msg += ' ' + str(self.database) + ", with a timeout of "
            _msg += str(tout) + "s was established."
            cbdebug(_msg)
            return self.mongodb_conn

        except PymongoException as msg:
            _msg = "Unable to establish a connection with the MongoDB "
            _msg += "server on host " + self.host + " port "
            _msg += str(self.port) + "database " + str(self.database) + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 14
0
    def initialize_metric_store(self, username):
        '''
        TBD
        '''
        self.conn_check()
        username = username.replace('-', "dash")

        try:
            _collections = [ \
                            "latest_management_VM_" + username, \
                            "latest_management_HOST_" + username, \
                            "latest_runtime_os_VM_" + username, \
                            "latest_runtime_os_HOST_" + username, \
                            "latest_runtime_app_VM_" + username \
                            ]

            for _collection in _collections:
                _collection_handle = self.mongodb_conn[
                    self.database][_collection]
                #_collection_handle.drop()

            _collections = [ "trace_" + username, \
                            "management_HOST_" + username, \
                            "management_VM_" + username, \
                            "runtime_os_VM_" + username, \
                            "runtime_app_VM_" + username, \
                            "runtime_os_HOST_" + username ]

            for _collection in _collections:
                _collection_handle = self.mongodb_conn[
                    self.database][_collection]

                if int(self.version) < 3:
                    _collection_handle.ensure_index("dashboard_polled")
                    _collection_handle.ensure_index("expid")
                    _collection_handle.ensure_index("time")
                    _collection_handle.ensure_index("uuid")
                else:
                    _collection_handle.create_index("dashboard_polled")
                    _collection_handle.create_index("expid")
                    _collection_handle.create_index("time")
                    _collection_handle.create_index("uuid")

            self.disconnect()
            return True

        except PymongoException as msg:
            _msg = "Unable to initialize all documents on "
            _msg += "\" on collection \"" + _collection + "\": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 15
0
    def count_document(self, table, criteria, disconnect_finish = False) :
        table = table.replace('-',"dash")

        try :
            cursor = self.conn_check()
            statement = "select * from " + table + self.make_restrictions(criteria)
            count = cursor.execute(statement).rowcount
            cursor.close()
            if disconnect_finish :
                self.disconnect()
            return count

        except mysql.connector.Error as err :
            _msg = "Unable to count documents on the table \""
            _msg += table + ": " + str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 11)
Ejemplo n.º 16
0
    def disconnect(self) :
        try:

            if "disconnect" in dir(MysqlMgdConn.catalogs.cbtool["conn"]) :
                MysqlMgdConn.catalogs.cbtool["conn"].disconnect()
                MysqlMgdConn.catalogs.cbtool["conn"] = False
                _msg = "A connection to MySQL running on host "
                _msg += self.host + ", port " + str(self.port) + ", database"
                _msg += ' ' + str(MysqlMgdConn.catalogs.cbtool["database"]) + ", was terminated."
                cbdebug(_msg)

        except mysql.connector.Error as err :
            _msg = "Unable to terminate a connection with MySQL "
            _msg += "server on host " + self.host + " port "
            _msg += str(self.port) + "database " + str(MysqlMgdConn.catalogs.cbtool["database"]) + ": "
            _msg += str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 3)
Ejemplo n.º 17
0
    def cleanup_collection(self, table, disconnect_finish = False) :
        table = table.replace('-',"dash")

        try :
            cursor = self.conn_check()
            statement = "delete from " + table
            cursor.execute(statement) 
            cursor.close()
            MysqlMgdConn.catalogs.cbtool["conn"].commit()
            if disconnect_finish :
                self.disconnect()
            return True

        except mysql.connector.Error as err :
            _msg = "Unable to drop all documents from the table \""
            _msg += table + ": " + str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 10)
Ejemplo n.º 18
0
    def delete_document(self, table, criteria, disconnect_finish = False) :
        table = table.replace('-',"dash")

        try :
            
            cursor = self.conn_check()
            statement = "delete from " + table + self.make_restrictions(criteria)
            cursor.execute(statement) 
            cursor.close()
            MysqlMgdConn.catalogs.cbtool["conn"].commit()
            if disconnect_finish :
                self.disconnect()

        except mysql.connector.Error as err :
            _msg = "Unable to remove document from the table \""
            _msg += table + ": " + str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 9)
Ejemplo n.º 19
0
    def flush_metric_store(self, username, partial=False, criteria={}):
        '''
        TBD
        '''
        self.conn_check()

        username = username.replace('-', "dash")

        try:
            _collections = ["latest_management_VM_" + username, \
                            "latest_management_HOST_" + username, \
                            "latest_runtime_os_VM_" + username, \
                            "latest_runtime_os_HOST_" + username, \
                            "latest_runtime_app_VM_" + username, \
                            "trace_" + username, \
                            "management_HOST_" + username, \
                            "management_VM_" + username, \
                            "runtime_os_VM_" + username, \
                            "runtime_app_VM_" + username, \
                            "runtime_os_HOST_" + username, \
                            "reported_management_VM_metric_names_" + username, \
                            "reported_runtime_app_VM_metric_names_" + username, \
                            "reported_runtime_os_HOST_metric_names_" + username, \
                            "reported_runtime_os_VM_metric_names_" + username ]

            for _collection in _collections:
                _collection_handle = self.mongodb_conn[
                    self.database][_collection]
                if partial:
                    _collection_handle.remove(criteria)
                else:
                    _collection_handle.drop()

            self.disconnect()
            return True

        except PymongoException as msg:
            _msg = "Unable to initialize all documents on "
            _msg += "\" on collection \"" + _collection + "\": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 20
0
    def get_reported_objects(self, collection, disconnect_finish=False):
        '''
        TBD
        '''
        self.conn_check()

        collection = collection.replace('-', "dash")

        try:
            _result = {}
            _attributes = [
                "vm_name", "role", "ai_name", "type", "aidrs_name", "pattern"
            ]
            for _attribute in _attributes:
                _result[_attribute + 's'] = []

            _collection_handle = self.mongodb_conn[self.database][collection]
            _documents = _collection_handle.find()
            for _document in _documents:
                for _attribute in _attributes:
                    if _attribute == "vm_name":
                        _attribute_r = "name"
                    else:
                        _attribute_r = _attribute

                    if _attribute_r in _document:
                        if not _result[_attribute + 's'].count(
                                _document[_attribute_r]):
                            _result[_attribute + 's'].append(
                                _document[_attribute_r])

            if disconnect_finish:
                self.disconnect()
            return _result

        except PymongoException as msg:
            _msg = "Unable to get reported attributes on the collection \""
            _msg += collection + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 21
0
    def cleanup_collection(self, collection, disconnect_finish=False):
        '''
        TBD
        '''
        self.conn_check()

        collection = collection.replace('-', "dash")

        try:
            _collection_handle = self.mongodb_conn[self.database][collection]
            _collection_handle.drop()
            if disconnect_finish:
                self.disconnect()
            return True

        except PymongoException as msg:
            _msg = "Unable to drop all documents from the collection \""
            _msg += collection + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 22
0
    def count_document(self, collection, criteria, disconnect_finish=False):
        '''
        TBD
        '''
        self.conn_check()

        collection = collection.replace('-', "dash")

        try:
            _collection_handle = self.mongodb_conn[self.database][collection]
            _matches = _collection_handle.find(criteria)
            if disconnect_finish:
                self.disconnect()
            return _matches.count()

        except PymongoException as msg:
            _msg = "Unable to count documents on the collection \""
            _msg += collection + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 23
0
    def disconnect(self):
        '''
        TBD
        '''
        try:

            if "disconnect" in dir(self.mongodb_conn):
                self.mongodb_conn.disconnect()
                self.mongodb_conn = False
                _msg = "A connection to MongoDB running on host "
                _msg += self.host + ", port " + str(self.port) + ", database"
                _msg += ' ' + str(self.database) + ", was terminated."
                cbdebug(_msg)
                return self.mongodb_conn

        except PymongoException as msg:
            _msg = "Unable to terminate a connection with the MongoDB "
            _msg += "server on host " + self.host + " port "
            _msg += str(self.port) + "database " + str(self.database) + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 24
0
    def find_document(self, collection, criteria, allmatches = False, \
                      sortkeypairs = None, limitdocuments = 0, \
                      documentfields = None, disconnect_finish = False) :
        '''
        TBD
        '''

        self.conn_check()

        collection = collection.replace('-', "dash")

        try:

            _collection_handle = self.mongodb_conn[self.database][collection]

            if allmatches:

                _results = _collection_handle.find(criteria, \
                                                   sort = sortkeypairs, \
                                                   limit = limitdocuments, \
                                                   projection = documentfields)
            else:

                _results = _collection_handle.find_one(criteria, \
                                                       sort = sortkeypairs, \
                                                       projection = documentfields)

            if disconnect_finish:
                self.disconnect()

            return _results

        except PymongoException as msg:
            _msg = "Unable to retrieve documents from the collection \""
            _msg += collection + ": "
            _msg += str(msg) + '.'
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 1)
Ejemplo n.º 25
0
    def conn_check(self, hostov = False, dbov = False, tout = False) :
        self.conn_mutex.acquire()
        try :
            getattr(MysqlMgdConn.catalogs, "cbtool")
        except AttributeError as e :
            cbdebug("Initializing thread local connection: ")
            MysqlMgdConn.catalogs.cbtool = {}

        if "database" not in MysqlMgdConn.catalogs.cbtool :
            MysqlMgdConn.catalogs.cbtool["database"] = False
        if "conn" not in MysqlMgdConn.catalogs.cbtool :
            MysqlMgdConn.catalogs.cbtool["conn"] = False
        if not MysqlMgdConn.catalogs.cbtool["conn"] or not MysqlMgdConn.catalogs.cbtool["conn"].is_connected() :
            MysqlMgdConn.catalogs.cbtool["conn"] = False

            if hostov :
                self.host = hostov

            if dbov :
                MysqlMgdConn.catalogs.cbtool["database"] = dbov

            if tout :
                self.timeout = tout
                
            try :
                self.connect(self.timeout)
            except MetricStoreMgdConnException as obj :
                self.conn_mutex.release()
                raise MetricStoreMgdConnException(obj.msg, 2)
            except Exception as e :
                self.conn_mutex.release()
                raise(e)

        assert(MysqlMgdConn.catalogs.cbtool["conn"])
        assert(MysqlMgdConn.catalogs.cbtool["conn"].is_connected())
        cursor = MysqlMgdConn.catalogs.cbtool["conn"].cursor()
        self.conn_mutex.release()
        return cursor
Ejemplo n.º 26
0
    def flush_metric_store(self, username, partial = False, criteria = {}) :
        username = username.replace('-',"dash")

        try :
            cursor = self.conn_check()        
            _tables = ["latest_management_VM_" + username, \
                       "latest_management_HOST_" + username, \
                       "latest_runtime_os_VM_" + username, \
                       "latest_runtime_os_HOST_" + username, \
                       "latest_runtime_app_VM_" + username, \
                       "trace_" + username, \
                       "management_HOST_" + username, \
                       "management_VM_" + username, \
                       "runtime_os_VM_" + username, \
                       "runtime_app_VM_" + username, \
                       "runtime_os_HOST_" + username, \
                       "reported_management_VM_metric_names_" + username, \
                       "reported_runtime_app_VM_metric_names_" + username, \
                       "reported_runtime_os_HOST_metric_names_" + username, \
                       "reported_runtime_os_VM_metric_names_" + username ]

            for _table in _tables :
                if partial and len(criteria) :
                    statement = "delete from " + _table + self.make_restrictions(criteria)
                    cursor.execute(statement) 
                else :
                    cursor.execute("delete from " + _table)

            cursor.close()
            MysqlMgdConn.catalogs.cbtool["conn"].commit()
            self.disconnect()
            return True
        except mysql.connector.Error as err :
            self.disconnect()
            _msg = "Unable to flush metric store: " + str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 5)
Ejemplo n.º 27
0
    def get_reported_objects(self, table, disconnect_finish = False) :
        table = table.replace('-',"dash")

        try :
            cursor = self.conn_check()
            _result = {}
            _attributes = [ "vm_name", "role", "ai_name", "type", "aidrs_name", "pattern" ]
            for _attribute in _attributes :
                _result[_attribute + 's'] = []

            statement = "select id, document from " + table
            cursor.execute(statement)
            while True :
                rows = cursor.fetchmany(4)
                if not len(rows) :
                    break
                for (original_mysql_id, _document) in rows :
                    for _attribute in _attributes :
                        if _attribute == "vm_name" :
                            _attribute_r = "name"
                        else :
                            _attribute_r = _attribute
                            
                        if _attribute_r in _document :
                            if not _result[_attribute + 's'].count(_document[_attribute_r]) :
                                _result[_attribute + 's'].append(_document[_attribute_r])
            
            cursor.close()
            if disconnect_finish :
                self.disconnect()
            return _result

        except mysql.connector.Error as err :
            _msg = "Unable to get reported attributes on the table \""
            _msg += table + ": " + str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 12)
Ejemplo n.º 28
0
    def initialize_metric_store(self, username) :
        username = username.replace('-',"dash")

        try :
            cursor = self.conn_check()        

            if not MysqlMgdConn.catalogs.cbtool["database"] :
                cursor.execute("create database " + self.database)
                cursor.execute("use " + self.database)
                MysqlMgdConn.catalogs.cbtool["database"] = True 

            _latest_tables = [ \
                            "latest_management_VM_" + username, \
                            "latest_management_HOST_" + username, \
                            "latest_runtime_os_VM_" + username, \
                            "latest_runtime_os_HOST_" + username, \
                            "latest_runtime_app_VM_" + username, \
                            "reported_management_VM_metric_names_" + username, \
                            "reported_runtime_app_VM_metric_names_" + username, \
                            "reported_runtime_os_HOST_metric_names_" + username, \
                            "reported_runtime_os_VM_metric_names_" + username \
                            ]

            _indexed_tables = [ "trace_" + username, \
                            "management_HOST_" + username, \
                            "management_VM_" + username, \
                            "runtime_os_VM_" + username, \
                            "runtime_app_VM_" + username, \
                            "runtime_os_HOST_" + username ]

            cursor.execute("show tables")

            _tables_found = [] 

            for x in cursor:
              _tables_found.append(x[0])

            for _table in (_latest_tables + _indexed_tables) :
                if _table not in _tables_found :
                    statement = "create table " + _table + "(" + \
                            "id int auto_increment primary key," + \
                            "document json NOT NULL," + \
                            "`expid` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.expid')," + \
                            "`_id` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$._id')," + \
                            "`time` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.time')," + \
                            "`uuid` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.uuid')," + \
                            "`dashboard_polled` VARCHAR(255) GENERATED ALWAYS AS (`document` ->> '$.dashboard_polled')" + \
                        ")"
                    cursor.execute(statement)

                    if _table in _indexed_tables : 
                        cursor.execute("CREATE INDEX `expid_idx` ON `" + _table + "`(`expid`)")
                        cursor.execute("CREATE INDEX `time_idx` ON `" + _table + "`(`time`)")
                        cursor.execute("CREATE INDEX `uuid_idx` ON `" + _table + "`(`uuid`)")
                        cursor.execute("CREATE INDEX `dashboard_polled_idx` ON `" + _table + "`(`dashboard_polled`)")
            cursor.close()
            MysqlMgdConn.catalogs.cbtool["conn"].commit()
            self.disconnect()
            return True

        except mysql.connector.Error as err :
            self.disconnect()
            _msg = "Unable to complete database initialization: "
            _msg += str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 4)
Ejemplo n.º 29
0
    def make_restrictions(self, criteria, join = "and", level = 0) :
        full_list = ""
        restrictions = []
        for _key in criteria.keys() :
            _value = criteria[_key]
            if isinstance(_value, set) :
                _msg = "1) We cannot yet handle this criteria: " + str(criteria)
                cberr(_msg)
                raise MetricStoreMgdConnException(_msg, 41)
            elif isinstance(_value, dict) :
                for subkey in _value.keys() :
                    if subkey.lower() == "$exists" :
                        if not isinstance(_value[subkey], bool) :
                            _msg = "2) We cannot yet handle this criteria: " + str(_value)
                            cberr(_msg)
                            raise MetricStoreMgdConnException(_msg, 41)

                        if _value[subkey] :
                            restrictions.append("document->>'$." + _key + "' IS NOT NULL")
                        else :
                            restrictions.append("document->>'$." + _key + "' IS NULL")
                    else :
                        _msg = "3) We cannot yet handle this criteria: " + str(subkey)
                        cberr(_msg)
                        raise MetricStoreMgdConnException(_msg, 41)
            elif isinstance(_value, list) :
                # Handle this group below 
                continue
            else :
                _newvalue = _value
                if isinstance(_value, bytes) :
                    _newvalue = _value.decode("utf-8")
                restrictions.append("document->>'$." + _key + "' = '" + str(_newvalue) + "'")

        if len(restrictions) :
            full_list += (" " + join + " ").join(restrictions)

        for _key in criteria.keys() :
            if _key.lower() == "$or" or _key.lower() == "$and" :
                _value = criteria[_key]
                if isinstance(_value, list) :
                    subdict = {}
                    for subitem in _value :
                        if not isinstance(subitem, dict) :
                            _msg = "4) We cannot yet handle this criteria: " + str(subitem)
                            cberr(_msg)
                            raise MetricStoreMgdConnException(_msg, 41)
                        subdict.update(subitem)
                    sub_restrictions = self.make_restrictions(subdict, join = _key[1:], level = level + 1)
                    if sub_restrictions.strip() != "" :
                        full_list += " and (" + sub_restrictions + ")"
                else :
                    _msg = "5) We cannot yet handle this criteria: " + str(_value)
                    cberr(_msg)
                    raise MetricStoreMgdConnException(_msg, 41)

        if full_list.strip() != "" :
            if level == 0 :
                return " where " + full_list
            else :
                return full_list

        return ""
Ejemplo n.º 30
0
    def find_document(self, table, criteria, allmatches = False, \
                      sortkeypairs = None, limitdocuments = 0, \
                      documentfields = None, disconnect_finish = False) :

        table = table.replace('-',"dash")

        try :
            cursor = self.conn_check()

            statement = "select "

            if documentfields is not None :
                convertedfields = []
                for field in documentfields :
                    convertedfields.append("document->>'$." + field + "'")
                   
                statement += ",".join(["id"] + convertedfields)
            else :
                statement += " id,document "

            statement += " from " + table + " " + self.make_restrictions(criteria)

            if sortkeypairs :
                keylist = []
                for keypair in sortkeypairs :
                    # FIXME: I'm unsure of how to have different directional sort criteria for multiple
                    # sorted keys. Will have to look into that later, so for the time being,
                    # I'm dropping the direction.
                    keylist.append("document->>'$." + keypair[0] + "'")
                statement += " order by " + ",".join(keylist)

            if not allmatches or limitdocuments :
                if limitdocuments > 0 :
                    statement += " limit " + str(limitdocuments)
                else :
                    statement += " limit 1"

            _results = []

            # FIXME: We need to figure out how to safely allow iterators over
            # the live connection. But for now, let's just extract all the results
            result = cursor.execute(statement)
            while True :
                rows = cursor.fetchmany(4)
                if not len(rows) :
                    break
                for resultset in rows :
                    original_mysql_id = resultset[0]
                    document = False
                    if documentfields is not None :
                        document = {}
                        for idx in range(1, len(resultset)) :
                           document[documentfields[idx - 1]] = resultset[idx].decode()
                    else :
                        if isinstance(resultset[1], str) :
                            document = json.loads(resultset[1])
                        else :
                            assert(isinstance(resultset[1], dict))
                            document = resultset[1]

                    document["original_mysql_id"] = original_mysql_id
                    _results.append(document)

            cursor.close()

            if allmatches :
                return _results
            else :
                if len(_results) >= 1 :
                    return _results[0]

            return None 

        except mysql.connector.Error as err :
            _msg = "Unable to retrieve documents from the table \""
            _msg += table + ": "  + str(err)
            cberr(_msg)
            raise MetricStoreMgdConnException(str(_msg), 7)