def saveDataLogMetrics(data):
logger.debug("saveDataMonitorHost")
conn = db.connectDb()
cursor = conn.cursor()
key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER
source = masterdao.findSourceByKey(cursor, key)
if not source:
logger.warning("source not found")
return (False, "source not found")
monitorMeasures = data['metrics']
for item in monitorMeasures:
date = item[1]
period = lmutil.calcTimeGroup(lmutil.datetimeParse(date))
index = 0
for metric in item:
measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
cursor, lmkey.DATATYPE_LOG_METRICS, index)
index += 1
if not measureType:
logger.warning("measure type not found:" + str(index - 1))
continue
if not measureType['enable']:
continue
masterdao.createMeasure(cursor, date, period, measureType['id'],
source['id'], metric)
conn.commit()
conn.close()
return (True, None)
def state():
app.logger.debug("state:")
conn = db.connectDb()
cursor = conn.cursor()
control = masterdao.getControlData(cursor)
conn.close()
return jsonify(control)
def saveDataLogEvents(data):
logger.debug("saveDataLogEvents")
conn = db.connectDb()
cursor = conn.cursor()
key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER
source = masterdao.findSourceByKey(cursor, key)
if not source:
logger.warning("source not found")
return (False, "source not found")
monitorMeasures = data['logEventsCount']
for item in monitorMeasures:
# logger.debug(item)
measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
cursor, data['datatype'], item[2])
if not measureType:
logger.warning("measure type not found")
continue
masterdao.createMeasure(
cursor, item[1],
lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])),
measureType['id'], source['id'], item[3])
conn.commit()
conn.close()
return (True, None)
def getPostgresMetrics(host, user, password):
connDb = psqldb.connectDb("postgres", host, user, password)
cursor = connDb.cursor()
conns = psqldb.getDbCountTransactions(cursor)
locks = psqldb.getDbLocksExclusive(cursor)
connDb.close()
data = {'conns': conns, 'locks': locks}
return data
def __init__(self):
"""
***********************************************************************
***********************************************************************
"""
#self.conn = db.connectDbFullName(dbfilepath)
self.conn = db.connectDb()
self.cursor = self.conn.cursor()
def measureAndPredict():
try:
start = datetime.datetime.now()
end = lmutil.getLogMapperIntervalDate()
start = end - datetime.timedelta(minutes=3)
conn = db.connectDb()
predictor.measureAndPredict(conn, start, end)
conn.commit()
conn.close()
except Exception as exc:
logger.exception("exception at measureAndPredict")
def saveDataMonitor(data):
logger.debug("saveDataMonitorHost")
conn = db.connectDb()
cursor = conn.cursor()
key = data['monitorKey'] + "-" + data['sourcetype']
source = masterdao.findSourceByKey(cursor, key)
if not source:
logger.warning("source not found")
return
masterdao.createSourceCounters(cursor, datetime.datetime.now(),
source['id'], data['lmstats']['count'],
data['lmstats']['bytes'],
data['lmstats']['records'],
data['lmstats']['fails'])
hostId = source['hostId']
monitorMeasures = data['measures']
for item in monitorMeasures:
date = item[0]
period = lmutil.calcTimeGroup(lmutil.datetimeParse(date))
if data['sourcetype'] == lmkey.SOURCE_TYPE_HOST:
masterdao.createHostMeasure(
cursor, date, period, hostId, source['id'], item[1], item[2],
item[3], item[4], item[5], item[6], item[7], item[8], item[9],
item[10], item[11], item[12], item[13], item[14], item[15],
item[16], item[17], item[18], item[19], item[20])
else:
index = 0
for metric in item:
measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
cursor, data['datatype'], index)
index += 1
if not measureType:
logger.warning("measure type not found")
continue
if not measureType['enable']:
continue
masterdao.createMeasure(cursor, date, period,
measureType['id'], source['id'],
metric)
conn.commit()
conn.close()
return (True, None)
def setstate():
app.logger.info("setstate")
conn = db.connectDb()
cursor = conn.cursor()
control = masterdao.getControlData(cursor)
collector_admin_state = request.args.get('collector_admin_state')
if collector_admin_state:
control['collector_admin_state'] = collector_admin_state
masterdao.updateControlData(cursor, control)
conn.commit()
conn.close()
return "OK:" + str(control)
def saveDataPathMetrics(data):
logger.debug("saveDataPathMetrics")
conn = db.connectDb()
cursor = conn.cursor()
key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER
source = masterdao.findSourceByKey(cursor, key)
if not source:
logger.warning("source not found")
return (False, "source not found")
masterdao.createSourceCounters(cursor, datetime.datetime.now(),
source['id'], data['lmstats']['count'],
data['lmstats']['bytes'],
data['lmstats']['records'],
data['lmstats']['fails'])
hostId = source['hostId']
monitorMeasures = data['pathMeasures']
for item in monitorMeasures:
key = source['name'] + "_" + str(item[2])
pathId = masterdao.findLogPathIdByKeyMaster(cursor, key)
if not pathId:
logger.warning("logPath not found")
return (False, "path not found")
masterdao.createPathMeasure(
cursor, item[1],
lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])), pathId,
hostId, item[3], item[4], item[5], item[6])
conn.commit()
conn.close()
return (True, None)
def saveDataAgent(data):
logger.debug("saveDataAgent")
agentkey = data['agentKey']
name = data['agentKey']
ip = data['agentIp']
port = data['agentPort']
hostName = data['host']
conn = db.connectDb()
cursor = conn.cursor()
host = masterdao.findHostByKey(cursor, hostName)
if not host:
host = masterdao.createHost(conn, hostName, hostName)
agent = masterdao.findAgentByKey(cursor, agentkey)
if agent:
return (False, "Agent already exist: " + str(agent))
agent = masterdao.createAgent(conn, agentkey, name, host['id'], ip, port,
True)
return (True, str(agent))
start = datetime.datetime(2018, 4, 23, 15, 33, 0)
end = datetime.datetime(2018, 4, 23, 23, 0, 0)
start = datetime.datetime(2018, 4, 24, 20, 0, 0)
end = datetime.datetime(2018, 4, 25, 6, 0, 0)
start = datetime.datetime(2018, 4, 26, 22, 0, 0)
end = datetime.datetime(2018, 4, 27, 8, 0, 0)
#rANGO PARA CALCULAR REF
start = datetime.datetime(2018, 4, 22, 16, 0, 0)
end = datetime.datetime(2018, 4, 28, 18, 0, 0)
#==============================================================================
#
#==============================================================================
connDbMaster = db.connectDb()
cursor = connDbMaster.cursor()
# masterdao.updateMeasureRefFlag(cursor, start, end, True)
# connDbMaster.commit()
#
processOutliers(connDbMaster, start, end)
#
# calculatePerformanceReference(connDbMaster, start, end, resetReference=True)
connDbMaster.close()
print("End module execution:" + str(datetime.datetime.now()))
# start = datetime.datetime(2018, 3, 18, 20, 15, 0)
# end = datetime.datetime(2018, 3, 19, 8, 45, 0)
# start = datetime.datetime(2018, 3, 26, 15, 50, 0)
# end = datetime.datetime(2018, 3, 26, 22, 5, 0)
# start = datetime.datetime(2018, 4, 2, 20, 45, 0)
# end = datetime.datetime(2018, 4, 2, 23, 0, 0)
start = datetime.datetime(2018, 4, 22, 12, 35, 0)
end = datetime.datetime(2018, 4, 22, 12, 45, 0)
#==============================================================================
# crea base de datos master con datos del agente
#==============================================================================
conn = db.connectDb()
masterdao.createTablesBase(conn)
try:
# collectAgentsConfig(config, conn)
# collectReadersTraceData(config, conn)
# collectLogRecordsData(config, conn, start, end, interval=5)
collectSourcesData(config, conn, start, end, interval=5)
except Exception as exc:
logger.exception("Exception collecting data")
conn.commit()
conn.close()
print("End module execution:" + str(datetime.datetime.now()))
n2 = record[0]
c2 = record[1]
if c1 and c2 and c1 != c2:
if not (n2, n1) in listRc:
masterdao.createRemoteCall(cursor, n1, n2)
listRc.append((n1, n2))
n1 = record[0]
c1 = record[1]
conn.commit()
#%%
if __name__ == '__main__':
print('Start module execution:')
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%H:%M:%S')
cfg.createDefaultConfigfile()
config=cfg.loadConfig()
#==============================================================================
start = datetime.datetime(2018, 4, 22, 20, 0, 0)
end = datetime.datetime(2018, 4, 27, 12, 0, 0)
connDbMaster=db.connectDb('master')
mapRemoteCalls(connDbMaster, start, end)
connDbMaster.close()
print("End module execution")