def saveDataLogMetrics(data):
logger.debug("saveDataMonitorHost")
conn = db.connectDb()
cursor = conn.cursor()
key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER
source = masterdao.findSourceByKey(cursor, key)
if not source:
logger.warning("source not found")
return (False, "source not found")
monitorMeasures = data['metrics']
for item in monitorMeasures:
date = item[1]
period = lmutil.calcTimeGroup(lmutil.datetimeParse(date))
index = 0
for metric in item:
measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
cursor, lmkey.DATATYPE_LOG_METRICS, index)
index += 1
if not measureType:
logger.warning("measure type not found:" + str(index - 1))
continue
if not measureType['enable']:
continue
masterdao.createMeasure(cursor, date, period, measureType['id'],
source['id'], metric)
conn.commit()
conn.close()
return (True, None)
def saveDataLogEvents(data):
logger.debug("saveDataLogEvents")
conn = db.connectDb()
cursor = conn.cursor()
key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER
source = masterdao.findSourceByKey(cursor, key)
if not source:
logger.warning("source not found")
return (False, "source not found")
monitorMeasures = data['logEventsCount']
for item in monitorMeasures:
# logger.debug(item)
measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
cursor, data['datatype'], item[2])
if not measureType:
logger.warning("measure type not found")
continue
masterdao.createMeasure(
cursor, item[1],
lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])),
measureType['id'], source['id'], item[3])
conn.commit()
conn.close()
return (True, None)
def saveDataMonitor(data):
logger.debug("saveDataMonitorHost")
conn = db.connectDb()
cursor = conn.cursor()
key = data['monitorKey'] + "-" + data['sourcetype']
source = masterdao.findSourceByKey(cursor, key)
if not source:
logger.warning("source not found")
return
masterdao.createSourceCounters(cursor, datetime.datetime.now(),
source['id'], data['lmstats']['count'],
data['lmstats']['bytes'],
data['lmstats']['records'],
data['lmstats']['fails'])
hostId = source['hostId']
monitorMeasures = data['measures']
for item in monitorMeasures:
date = item[0]
period = lmutil.calcTimeGroup(lmutil.datetimeParse(date))
if data['sourcetype'] == lmkey.SOURCE_TYPE_HOST:
masterdao.createHostMeasure(
cursor, date, period, hostId, source['id'], item[1], item[2],
item[3], item[4], item[5], item[6], item[7], item[8], item[9],
item[10], item[11], item[12], item[13], item[14], item[15],
item[16], item[17], item[18], item[19], item[20])
else:
index = 0
for metric in item:
measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
cursor, data['datatype'], index)
index += 1
if not measureType:
logger.warning("measure type not found")
continue
if not measureType['enable']:
continue
masterdao.createMeasure(cursor, date, period,
measureType['id'], source['id'],
metric)
conn.commit()
conn.close()
return (True, None)
def collectDataMonitor(config, conn, report, source, start, end):
logger.debug('collectDataMonitor:' + str(source))
cursor = conn.cursor()
data = report.getMonMeasures(source['name'], start, end)
data = json.loads(data)
if source['name'] != data['monitorKey']:
logger.warn("Data error:" + str(data))
return
source = masterdao.findSourceByKey(cursor, source['key'])
if not source:
logger.warning("source not found")
return
monitorMeasures = data['measures']
hostId = source['hostId']
sourcetype = source['type']
for item in monitorMeasures:
date = item[0]
period = lmutil.calcTimeGroup(lmutil.datetimeParse(date))
if sourcetype == lmkey.SOURCE_TYPE_HOST:
masterdao.createHostMeasure(
cursor, date, period, hostId, source['id'], item[1], item[2],
item[3], item[4], item[5], item[6], item[7], item[8], item[9],
item[10], item[11], item[12], item[13], item[14], item[15],
item[16], item[17], item[18], item[19], item[20])
else:
index = 0
for metric in item:
measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
cursor, data['datatype'], index)
index += 1
if not measureType:
logger.warning("measure type not found")
continue
if not measureType['enable']:
continue
masterdao.createMeasure(cursor, date, period,
measureType['id'], source['id'],
metric)
conn.commit()
def saveDataPathMetrics(data):
logger.debug("saveDataPathMetrics")
conn = db.connectDb()
cursor = conn.cursor()
key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER
source = masterdao.findSourceByKey(cursor, key)
if not source:
logger.warning("source not found")
return (False, "source not found")
masterdao.createSourceCounters(cursor, datetime.datetime.now(),
source['id'], data['lmstats']['count'],
data['lmstats']['bytes'],
data['lmstats']['records'],
data['lmstats']['fails'])
hostId = source['hostId']
monitorMeasures = data['pathMeasures']
for item in monitorMeasures:
key = source['name'] + "_" + str(item[2])
pathId = masterdao.findLogPathIdByKeyMaster(cursor, key)
if not pathId:
logger.warning("logPath not found")
return (False, "path not found")
masterdao.createPathMeasure(
cursor, item[1],
lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])), pathId,
hostId, item[3], item[4], item[5], item[6])
conn.commit()
conn.close()
return (True, None)
def collectDataConfig(config, conn, report):
logger.debug('collectDataConfig:')
cursor = conn.cursor()
#==========================================================================
#
#==========================================================================
data = report.getAgent()
data = json.loads(data)
agentkey = data['agentKey']
name = data['agentKey']
ip = data['agentIp']
port = data['agentPort']
hostName = data['host']
host = masterdao.findHostByKey(cursor, hostName)
if not host:
host = masterdao.createHost(conn, hostName, hostName)
agent = masterdao.findAgentByKey(cursor, agentkey)
if not agent:
agent = masterdao.createAgent(conn, agentkey, name, host['id'], ip,
port, True)
#%%
#==========================================================================
#
#==========================================================================
data = report.getReaders()
data = json.loads(data)
readers = data['readers']
for item in readers:
key = item['component']
name = item['component']
component = masterdao.findComponentByKey(cursor, key)
if not component:
component = masterdao.createComponent(conn, key, name, host['id'])
sourcetype = lmkey.SOURCE_TYPE_READER
key = item['readerKey'] + "-" + sourcetype
enable = item['enable']
source = masterdao.findSourceByKey(cursor, key)
if not source:
source = masterdao.createSource(conn, key, item['readerKey'],
agent['id'], sourcetype, enable)
for category in cat.getValuesLogEventCategories():
name = category.name
measType = lmkey.DATATYPE_LOG_EVENTS
mtype = masterdao.findMeaureType(cursor, name, measType)
if not mtype:
mtype = masterdao.createMeasureType(conn, name, measType,
lmkey.MEASURE_CAT_EVENT,
True, category.value)
measureSource = masterdao.findMeaureSource(cursor, component['id'],
mtype['id'])
if not measureSource:
masterdao.createMeasureSource(conn, component['id'],
mtype['id'], source['id'], True)
data = report.getLogMetricsColnames(item['readerKey'])
data = json.loads(data)
for col in data['colnames']:
name = col['name']
index = col['idx']
measType = data['datatype']
category = lmkey.MEASURE_CAT_METRIC
mtype = masterdao.findMeaureType(cursor, name, measType)
if not mtype:
mtype = masterdao.createMeasureType(conn, name, measType,
category, True, index)
measureSource = masterdao.findMeaureSource(cursor, component['id'],
mtype['id'])
if not measureSource:
masterdao.createMeasureSource(conn, component['id'],
mtype['id'], source['id'], True)
#==========================================================================
#
#==========================================================================
data = report.getMonitors()
data = json.loads(data)
monitors = data['monitors']
for item in monitors:
for componentKey in item['components'].split(','):
component = masterdao.findComponentByKey(cursor, componentKey)
if not component:
component = masterdao.createComponent(conn, componentKey,
componentKey, host['id'])
sourcetype = item['type']
key = item['monitorKey'] + "-" + sourcetype
enable = item['enable']
source = masterdao.findSourceByKey(cursor, key)
if not source:
source = masterdao.createSource(conn, key, item['monitorKey'],
agent['id'], sourcetype,
enable)
conn.commit()
data = report.getLogMonMeasuresColnames(item['monitorKey'])
data = json.loads(data)
for col in data['colnames']:
name = col['name']
index = col['idx']
measType = data['datatype']
category = lmkey.MEASURE_CAT_METRIC
mtype = masterdao.findMeaureType(cursor, name, measType)
if not mtype:
mtype = masterdao.createMeasureType(
conn, name, measType, category, True, index)
measureSource = masterdao.findMeaureSource(
cursor, component['id'], mtype['id'])
if not measureSource:
masterdao.createMeasureSource(conn, component['id'],
mtype['id'], source['id'],
True)
conn.commit()
cursor.close()
def collectDataReader(config, conn, report, readerKey, hostId, componentId,
start, end):
logger.debug('collectDataReader:' + str(readerKey))
#TODO VERIFICAR SI ESTO FUNCIONA BIEN
end = lmutil.getBeforeMapperIntervalDate(end)
cursor = conn.cursor()
data = report.getPathMeasures(readerKey, start, end)
data = json.loads(data)
pathMeasures = data['pathMeasures']
#['READER_devicetypetok_20', '2017-09-26', 102, 172.4, 204.1164045613189, 0.229, 418.314, 34.5, 54.02, 239.0, 172.4]
for item in pathMeasures:
key = readerKey + "_" + str(item[2])
pathId = masterdao.findLogPathIdByKeyMaster(cursor, key)
if not pathId:
logger.warning("logPath not found")
continue
masterdao.createPathMeasure(
cursor, item[1],
lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])), pathId,
hostId, item[3], item[4], item[5], item[6])
conn.commit()
data = report.getLogEventsCount(readerKey, start, end)
data = json.loads(data)
eventsCount = data['logEventsCount']
source = masterdao.findSourceByKey(cursor, readerKey)
if not source:
logger.warning("source not found")
return
#TODO SE DEBE MANEJAR LA TABLA lmp_issuesT, POR AHORA SOLO lmp_logNodesT
for item in eventsCount:
logger.debug(item)
measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
cursor, data['datatype'], item[2])
if not measureType:
logger.warning("measure type not found")
continue
masterdao.createMeasure(
cursor, item[1],
lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])),
measureType['id'], source['id'], item[3])
conn.commit()
data = report.getLogMetrics(readerKey, start, end)
data = json.loads(data)
metrics = data['metrics']
#TODO SE DEBE MANEJAR LA TABLA lmp_issuesT, POR AHORA SOLO lmp_logNodesT
for item in metrics:
logger.debug(item)
date = item[1]
period = lmutil.calcTimeGroup(lmutil.datetimeParse(item[1]))
index = 0
for metric in item:
measureType = masterdao.findMeaureTypeByDataTypeAndIndex(
cursor, data['datatype'], index)
index += 1
if not measureType:
logger.warning("measure type not found")
continue
if not measureType['enable']:
continue
masterdao.createMeasure(cursor, date, period, measureType['id'],
source['id'], metric)
conn.commit()