Exemple #1
0
def saveDataLogMetrics(data):
    logger.debug("saveDataMonitorHost")

    conn = db.connectDb()
    cursor = conn.cursor()

    key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER
    source = masterdao.findSourceByKey(cursor, key)
    if not source:
        logger.warning("source not found")
        return (False, "source not found")

    monitorMeasures = data['metrics']

    for item in monitorMeasures:
        date = item[1]
        period = lmutil.calcTimeGroup(lmutil.datetimeParse(date))

        index = 0
        for metric in item:
            measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
                cursor, lmkey.DATATYPE_LOG_METRICS, index)
            index += 1
            if not measureType:
                logger.warning("measure type not found:" + str(index - 1))
                continue
            if not measureType['enable']:
                continue
            masterdao.createMeasure(cursor, date, period, measureType['id'],
                                    source['id'], metric)

    conn.commit()
    conn.close()

    return (True, None)
Exemple #2
0
def state():
    app.logger.debug("state:")
    conn = db.connectDb()
    cursor = conn.cursor()
    control = masterdao.getControlData(cursor)
    conn.close()
    return jsonify(control)
Exemple #3
0
def saveDataLogEvents(data):
    logger.debug("saveDataLogEvents")

    conn = db.connectDb()
    cursor = conn.cursor()

    key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER
    source = masterdao.findSourceByKey(cursor, key)
    if not source:
        logger.warning("source not found")
        return (False, "source not found")

    monitorMeasures = data['logEventsCount']

    for item in monitorMeasures:
        #        logger.debug(item)
        measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
            cursor, data['datatype'], item[2])
        if not measureType:
            logger.warning("measure type not found")
            continue
        masterdao.createMeasure(
            cursor, item[1],
            lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])),
            measureType['id'], source['id'], item[3])

    conn.commit()
    conn.close()

    return (True, None)
def getPostgresMetrics(host, user, password):
    connDb = psqldb.connectDb("postgres", host, user, password)
    cursor = connDb.cursor()
    conns = psqldb.getDbCountTransactions(cursor)
    locks = psqldb.getDbLocksExclusive(cursor)
    connDb.close()

    data = {'conns': conns, 'locks': locks}
    return data
Exemple #5
0
 def __init__(self):
     """
     ***********************************************************************
     
     ***********************************************************************
     """
     #self.conn = db.connectDbFullName(dbfilepath)
     self.conn = db.connectDb()
     self.cursor = self.conn.cursor()
Exemple #6
0
def measureAndPredict():
    try:
        start = datetime.datetime.now()
        end = lmutil.getLogMapperIntervalDate()
        start = end - datetime.timedelta(minutes=3)
        conn = db.connectDb()
        predictor.measureAndPredict(conn, start, end)
        conn.commit()
        conn.close()
    except Exception as exc:
        logger.exception("exception at measureAndPredict")
Exemple #7
0
def saveDataMonitor(data):
    logger.debug("saveDataMonitorHost")

    conn = db.connectDb()
    cursor = conn.cursor()

    key = data['monitorKey'] + "-" + data['sourcetype']

    source = masterdao.findSourceByKey(cursor, key)
    if not source:
        logger.warning("source not found")
        return

    masterdao.createSourceCounters(cursor, datetime.datetime.now(),
                                   source['id'], data['lmstats']['count'],
                                   data['lmstats']['bytes'],
                                   data['lmstats']['records'],
                                   data['lmstats']['fails'])

    hostId = source['hostId']
    monitorMeasures = data['measures']

    for item in monitorMeasures:
        date = item[0]
        period = lmutil.calcTimeGroup(lmutil.datetimeParse(date))

        if data['sourcetype'] == lmkey.SOURCE_TYPE_HOST:
            masterdao.createHostMeasure(
                cursor, date, period, hostId, source['id'], item[1], item[2],
                item[3], item[4], item[5], item[6], item[7], item[8], item[9],
                item[10], item[11], item[12], item[13], item[14], item[15],
                item[16], item[17], item[18], item[19], item[20])
        else:
            index = 0
            for metric in item:
                measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
                    cursor, data['datatype'], index)
                index += 1
                if not measureType:
                    logger.warning("measure type not found")
                    continue
                if not measureType['enable']:
                    continue
                masterdao.createMeasure(cursor, date, period,
                                        measureType['id'], source['id'],
                                        metric)

    conn.commit()
    conn.close()

    return (True, None)
Exemple #8
0
def setstate():
    app.logger.info("setstate")
    conn = db.connectDb()
    cursor = conn.cursor()
    control = masterdao.getControlData(cursor)

    collector_admin_state = request.args.get('collector_admin_state')
    if collector_admin_state:
        control['collector_admin_state'] = collector_admin_state

    masterdao.updateControlData(cursor, control)
    conn.commit()
    conn.close()

    return "OK:" + str(control)
Exemple #9
0
def saveDataPathMetrics(data):
    logger.debug("saveDataPathMetrics")

    conn = db.connectDb()
    cursor = conn.cursor()

    key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER
    source = masterdao.findSourceByKey(cursor, key)
    if not source:
        logger.warning("source not found")
        return (False, "source not found")

    masterdao.createSourceCounters(cursor, datetime.datetime.now(),
                                   source['id'], data['lmstats']['count'],
                                   data['lmstats']['bytes'],
                                   data['lmstats']['records'],
                                   data['lmstats']['fails'])

    hostId = source['hostId']
    monitorMeasures = data['pathMeasures']

    for item in monitorMeasures:
        key = source['name'] + "_" + str(item[2])
        pathId = masterdao.findLogPathIdByKeyMaster(cursor, key)

        if not pathId:
            logger.warning("logPath not found")
            return (False, "path not found")

        masterdao.createPathMeasure(
            cursor, item[1],
            lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])), pathId,
            hostId, item[3], item[4], item[5], item[6])

    conn.commit()
    conn.close()

    return (True, None)
Exemple #10
0
def saveDataAgent(data):
    logger.debug("saveDataAgent")

    agentkey = data['agentKey']
    name = data['agentKey']
    ip = data['agentIp']
    port = data['agentPort']
    hostName = data['host']

    conn = db.connectDb()
    cursor = conn.cursor()

    host = masterdao.findHostByKey(cursor, hostName)
    if not host:
        host = masterdao.createHost(conn, hostName, hostName)

    agent = masterdao.findAgentByKey(cursor, agentkey)

    if agent:
        return (False, "Agent already exist: " + str(agent))

    agent = masterdao.createAgent(conn, agentkey, name, host['id'], ip, port,
                                  True)
    return (True, str(agent))
    start = datetime.datetime(2018, 4, 23, 15, 33, 0)
    end = datetime.datetime(2018, 4, 23, 23, 0, 0)

    start = datetime.datetime(2018, 4, 24, 20, 0, 0)
    end = datetime.datetime(2018, 4, 25, 6, 0, 0)

    start = datetime.datetime(2018, 4, 26, 22, 0, 0)
    end = datetime.datetime(2018, 4, 27, 8, 0, 0)

    #rANGO PARA CALCULAR REF
    start = datetime.datetime(2018, 4, 22, 16, 0, 0)
    end = datetime.datetime(2018, 4, 28, 18, 0, 0)

    #==============================================================================
    #
    #==============================================================================

    connDbMaster = db.connectDb()
    cursor = connDbMaster.cursor()

    #    masterdao.updateMeasureRefFlag(cursor, start, end, True)
    #    connDbMaster.commit()
    #
    processOutliers(connDbMaster, start, end)
    #
    #    calculatePerformanceReference(connDbMaster, start, end, resetReference=True)

    connDbMaster.close()

    print("End module execution:" + str(datetime.datetime.now()))
Exemple #12
0
    #    start = datetime.datetime(2018, 3, 18, 20, 15, 0)
    #    end =   datetime.datetime(2018, 3, 19, 8, 45, 0)

    #    start = datetime.datetime(2018, 3, 26, 15, 50, 0)
    #    end =   datetime.datetime(2018, 3, 26, 22, 5, 0)

    #    start = datetime.datetime(2018, 4, 2, 20, 45, 0)
    #    end =   datetime.datetime(2018, 4, 2, 23, 0, 0)

    start = datetime.datetime(2018, 4, 22, 12, 35, 0)
    end = datetime.datetime(2018, 4, 22, 12, 45, 0)

    #==============================================================================
    # crea base de datos master con datos del agente
    #==============================================================================
    conn = db.connectDb()
    masterdao.createTablesBase(conn)

    try:
        #        collectAgentsConfig(config, conn)
        #        collectReadersTraceData(config, conn)
        #        collectLogRecordsData(config, conn, start, end, interval=5)
        collectSourcesData(config, conn, start, end, interval=5)
    except Exception as exc:
        logger.exception("Exception collecting data")

    conn.commit()
    conn.close()

    print("End module execution:" + str(datetime.datetime.now()))
Exemple #13
0
            n2 = record[0]
            c2 = record[1]
            if c1 and c2 and c1 != c2:
                if not (n2, n1) in listRc:
                    masterdao.createRemoteCall(cursor, n1, n2)
                listRc.append((n1, n2))
            n1 = record[0]
            c1 = record[1]
            
    conn.commit()


#%%

if __name__ == '__main__':
    print('Start module execution:')
    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%H:%M:%S')     

    cfg.createDefaultConfigfile()
    config=cfg.loadConfig()   
    
#==============================================================================
    
    start = datetime.datetime(2018, 4, 22, 20, 0, 0)
    end =   datetime.datetime(2018, 4, 27, 12, 0, 0) 

    connDbMaster=db.connectDb('master')    
    mapRemoteCalls(connDbMaster, start, end)   
    connDbMaster.close()
 
    print("End module execution")