def saveDataLogMetrics(data): logger.debug("saveDataMonitorHost") conn = db.connectDb() cursor = conn.cursor() key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER source = masterdao.findSourceByKey(cursor, key) if not source: logger.warning("source not found") return (False, "source not found") monitorMeasures = data['metrics'] for item in monitorMeasures: date = item[1] period = lmutil.calcTimeGroup(lmutil.datetimeParse(date)) index = 0 for metric in item: measureType = masterdao.findMeaureTypeByDataTypeAndIndex( cursor, lmkey.DATATYPE_LOG_METRICS, index) index += 1 if not measureType: logger.warning("measure type not found:" + str(index - 1)) continue if not measureType['enable']: continue masterdao.createMeasure(cursor, date, period, measureType['id'], source['id'], metric) conn.commit() conn.close() return (True, None)
def saveDataLogEvents(data): logger.debug("saveDataLogEvents") conn = db.connectDb() cursor = conn.cursor() key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER source = masterdao.findSourceByKey(cursor, key) if not source: logger.warning("source not found") return (False, "source not found") monitorMeasures = data['logEventsCount'] for item in monitorMeasures: # logger.debug(item) measureType = masterdao.findMeaureTypeByDataTypeAndIndex( cursor, data['datatype'], item[2]) if not measureType: logger.warning("measure type not found") continue masterdao.createMeasure( cursor, item[1], lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])), measureType['id'], source['id'], item[3]) conn.commit() conn.close() return (True, None)
def saveDataMonitor(data): logger.debug("saveDataMonitorHost") conn = db.connectDb() cursor = conn.cursor() key = data['monitorKey'] + "-" + data['sourcetype'] source = masterdao.findSourceByKey(cursor, key) if not source: logger.warning("source not found") return masterdao.createSourceCounters(cursor, datetime.datetime.now(), source['id'], data['lmstats']['count'], data['lmstats']['bytes'], data['lmstats']['records'], data['lmstats']['fails']) hostId = source['hostId'] monitorMeasures = data['measures'] for item in monitorMeasures: date = item[0] period = lmutil.calcTimeGroup(lmutil.datetimeParse(date)) if data['sourcetype'] == lmkey.SOURCE_TYPE_HOST: masterdao.createHostMeasure( cursor, date, period, hostId, source['id'], item[1], item[2], item[3], item[4], item[5], item[6], item[7], item[8], item[9], item[10], item[11], item[12], item[13], item[14], item[15], item[16], item[17], item[18], item[19], item[20]) else: index = 0 for metric in item: measureType = masterdao.findMeaureTypeByDataTypeAndIndex( cursor, data['datatype'], index) index += 1 if not measureType: logger.warning("measure type not found") continue if not measureType['enable']: continue masterdao.createMeasure(cursor, date, period, measureType['id'], source['id'], metric) conn.commit() conn.close() return (True, None)
def collectDataMonitor(config, conn, report, source, start, end): logger.debug('collectDataMonitor:' + str(source)) cursor = conn.cursor() data = report.getMonMeasures(source['name'], start, end) data = json.loads(data) if source['name'] != data['monitorKey']: logger.warn("Data error:" + str(data)) return source = masterdao.findSourceByKey(cursor, source['key']) if not source: logger.warning("source not found") return monitorMeasures = data['measures'] hostId = source['hostId'] sourcetype = source['type'] for item in monitorMeasures: date = item[0] period = lmutil.calcTimeGroup(lmutil.datetimeParse(date)) if sourcetype == lmkey.SOURCE_TYPE_HOST: masterdao.createHostMeasure( cursor, date, period, hostId, source['id'], item[1], item[2], item[3], item[4], item[5], item[6], item[7], item[8], item[9], item[10], item[11], item[12], item[13], item[14], item[15], item[16], item[17], item[18], item[19], item[20]) else: index = 0 for metric in item: measureType = masterdao.findMeaureTypeByDataTypeAndIndex( cursor, data['datatype'], index) index += 1 if not measureType: logger.warning("measure type not found") continue if not measureType['enable']: continue masterdao.createMeasure(cursor, date, period, measureType['id'], source['id'], metric) conn.commit()
def saveDataPathMetrics(data): logger.debug("saveDataPathMetrics") conn = db.connectDb() cursor = conn.cursor() key = data['readerKey'] + "-" + lmkey.SOURCE_TYPE_READER source = masterdao.findSourceByKey(cursor, key) if not source: logger.warning("source not found") return (False, "source not found") masterdao.createSourceCounters(cursor, datetime.datetime.now(), source['id'], data['lmstats']['count'], data['lmstats']['bytes'], data['lmstats']['records'], data['lmstats']['fails']) hostId = source['hostId'] monitorMeasures = data['pathMeasures'] for item in monitorMeasures: key = source['name'] + "_" + str(item[2]) pathId = masterdao.findLogPathIdByKeyMaster(cursor, key) if not pathId: logger.warning("logPath not found") return (False, "path not found") masterdao.createPathMeasure( cursor, item[1], lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])), pathId, hostId, item[3], item[4], item[5], item[6]) conn.commit() conn.close() return (True, None)
def collectDataConfig(config, conn, report): logger.debug('collectDataConfig:') cursor = conn.cursor() #========================================================================== # #========================================================================== data = report.getAgent() data = json.loads(data) agentkey = data['agentKey'] name = data['agentKey'] ip = data['agentIp'] port = data['agentPort'] hostName = data['host'] host = masterdao.findHostByKey(cursor, hostName) if not host: host = masterdao.createHost(conn, hostName, hostName) agent = masterdao.findAgentByKey(cursor, agentkey) if not agent: agent = masterdao.createAgent(conn, agentkey, name, host['id'], ip, port, True) #%% #========================================================================== # #========================================================================== data = report.getReaders() data = json.loads(data) readers = data['readers'] for item in readers: key = item['component'] name = item['component'] component = masterdao.findComponentByKey(cursor, key) if not component: component = masterdao.createComponent(conn, key, name, host['id']) sourcetype = lmkey.SOURCE_TYPE_READER key = item['readerKey'] + "-" + sourcetype enable = item['enable'] source = masterdao.findSourceByKey(cursor, key) if not source: source = masterdao.createSource(conn, key, item['readerKey'], agent['id'], sourcetype, enable) for category in cat.getValuesLogEventCategories(): name = category.name measType = lmkey.DATATYPE_LOG_EVENTS mtype = masterdao.findMeaureType(cursor, name, measType) if not mtype: mtype = masterdao.createMeasureType(conn, name, measType, lmkey.MEASURE_CAT_EVENT, True, category.value) measureSource = masterdao.findMeaureSource(cursor, component['id'], mtype['id']) if not measureSource: masterdao.createMeasureSource(conn, component['id'], mtype['id'], source['id'], True) data = report.getLogMetricsColnames(item['readerKey']) data = json.loads(data) for col in data['colnames']: name = col['name'] index = col['idx'] measType = data['datatype'] category = lmkey.MEASURE_CAT_METRIC mtype = masterdao.findMeaureType(cursor, name, measType) if not mtype: mtype = masterdao.createMeasureType(conn, name, measType, category, True, index) measureSource = masterdao.findMeaureSource(cursor, component['id'], mtype['id']) if not measureSource: masterdao.createMeasureSource(conn, component['id'], mtype['id'], source['id'], True) #========================================================================== # #========================================================================== data = report.getMonitors() data = json.loads(data) monitors = data['monitors'] for item in monitors: for componentKey in item['components'].split(','): component = masterdao.findComponentByKey(cursor, componentKey) if not component: component = masterdao.createComponent(conn, componentKey, componentKey, host['id']) sourcetype = item['type'] key = item['monitorKey'] + "-" + sourcetype enable = item['enable'] source = masterdao.findSourceByKey(cursor, key) if not source: source = masterdao.createSource(conn, key, item['monitorKey'], agent['id'], sourcetype, enable) conn.commit() data = report.getLogMonMeasuresColnames(item['monitorKey']) data = json.loads(data) for col in data['colnames']: name = col['name'] index = col['idx'] measType = data['datatype'] category = lmkey.MEASURE_CAT_METRIC mtype = masterdao.findMeaureType(cursor, name, measType) if not mtype: mtype = masterdao.createMeasureType( conn, name, measType, category, True, index) measureSource = masterdao.findMeaureSource( cursor, component['id'], mtype['id']) if not measureSource: masterdao.createMeasureSource(conn, component['id'], mtype['id'], source['id'], True) conn.commit() cursor.close()
def collectDataReader(config, conn, report, readerKey, hostId, componentId, start, end): logger.debug('collectDataReader:' + str(readerKey)) #TODO VERIFICAR SI ESTO FUNCIONA BIEN end = lmutil.getBeforeMapperIntervalDate(end) cursor = conn.cursor() data = report.getPathMeasures(readerKey, start, end) data = json.loads(data) pathMeasures = data['pathMeasures'] #['READER_devicetypetok_20', '2017-09-26', 102, 172.4, 204.1164045613189, 0.229, 418.314, 34.5, 54.02, 239.0, 172.4] for item in pathMeasures: key = readerKey + "_" + str(item[2]) pathId = masterdao.findLogPathIdByKeyMaster(cursor, key) if not pathId: logger.warning("logPath not found") continue masterdao.createPathMeasure( cursor, item[1], lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])), pathId, hostId, item[3], item[4], item[5], item[6]) conn.commit() data = report.getLogEventsCount(readerKey, start, end) data = json.loads(data) eventsCount = data['logEventsCount'] source = masterdao.findSourceByKey(cursor, readerKey) if not source: logger.warning("source not found") return #TODO SE DEBE MANEJAR LA TABLA lmp_issuesT, POR AHORA SOLO lmp_logNodesT for item in eventsCount: logger.debug(item) measureType = masterdao.findMeaureTypeByDataTypeAndIndex( cursor, data['datatype'], item[2]) if not measureType: logger.warning("measure type not found") continue masterdao.createMeasure( cursor, item[1], lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])), measureType['id'], source['id'], item[3]) conn.commit() data = report.getLogMetrics(readerKey, start, end) data = json.loads(data) metrics = data['metrics'] #TODO SE DEBE MANEJAR LA TABLA lmp_issuesT, POR AHORA SOLO lmp_logNodesT for item in metrics: logger.debug(item) date = item[1] period = lmutil.calcTimeGroup(lmutil.datetimeParse(item[1])) index = 0 for metric in item: measureType = masterdao.findMeaureTypeByDataTypeAndIndex( cursor, data['datatype'], index) index += 1 if not measureType: logger.warning("measure type not found") continue if not measureType['enable']: continue masterdao.createMeasure(cursor, date, period, measureType['id'], source['id'], metric) conn.commit()