def measureAndPredict(): try: start = datetime.datetime.now() end = lmutil.getLogMapperIntervalDate() start = end - datetime.timedelta(minutes=3) conn = db.connectDb() predictor.measureAndPredict(conn, start, end) conn.commit() conn.close() except Exception as exc: logger.exception("exception at measureAndPredict")
def process(self): logger.debug("process2") self.createMonitorDb() self.setState(th.ThreadState.PROCESSING) while not self._stopRun: self.loopCount += 1 bytesProcessed = self.monitor() if bytesProcessed > 0: self.recordsProcessed += 1 self.bytesProcessed += bytesProcessed time.sleep(self.monitorConfig.interval) #================================================================== # Preprocess data. Summarize data #================================================================== now = datetime.datetime.now() if now > self.nextSaveDataDate and self.enableDataSend: self.nextSaveDataDate = lmutil.getNextLogMapperIntervalDate( now) start = lmutil.getBeforeMapperIntervalDate(now) end = lmutil.getLogMapperIntervalDate(now) j = get.getJsonMonitorMeasures(self.monitorConfig, start, end) j['lmstats'] = self.getCounters() logger.debug("send:" + str(start) + " - " + str(end)) sender = ds.DataSenderThread(self.name, self.monitorConfig.masterHost, self.monitorConfig.masterPort, lmkey.DATATYPE_MONITOR_POSTGRES, j) sender.start() #================================================================== # check day change, reset memory database #================================================================== if self.currentDate != now.date(): dayBefore = str(self.currentDate) logger.info('Day end. reset data: ' + dayBefore) self.currentDate = datetime.date.today() newFile = self.monitorConfig.dbMonitorPath + "." + dayBefore db.copyDbFile(self.monitorConfig.dbMonitorPath, newFile) db.deleteDbFile(self.monitorConfig.dbMonitorPath) self.createMonitorDb() self.setState(th.ThreadState.ENDING)
def process(self): #====================================================================== # Runtime search of the parser class #====================================================================== module = __import__('parsers.' + self.readerConfig.moduleName, fromlist=['']) class_ = getattr(module, self.readerConfig.className) parser = class_(self.readerConfig.hostname, self.readerConfig.component) if not parser: self.setStateError("Parser class not found[" + self.name + "]" + self.readerConfig.className) return #====================================================================== # Check logger file exist. If not wait #====================================================================== while (not self._stopRun) and (not os.path.isfile( self.readerConfig.sourcefilepath)): logger.debug("File not exist. Wait: " + self.readerConfig.sourcefilepath) time.sleep(10) connDbMemory = db.connectDbMemory() cursor = connDbMemory.cursor() #====================================================================== # Create events database if not exist # If exist get last index #====================================================================== #db.deleteDbFile(self.readerConfig.dbEventsPath) connDbEvents = db.connectDb(self.readerConfig.dbEventsPath) cursorDbEvents = connDbEvents.cursor() eventdao.createTablesEvents(cursorDbEvents) connDbEvents.commit() db.copyDbData(connDbEvents, connDbMemory) connDbMemory.commit() cursorDbEvents.close() connDbEvents.close() #====================================================================== # Create base tables if not exist # If exist load data into db in memory # truncate events table in memory. Set id sequence #====================================================================== connDbBase = db.connectDb(self.readerConfig.dbBasePath) cursorDbBase = connDbBase.cursor() lmdao.createTablesBase(cursorDbBase) connDbBase.commit() db.copyDbData(connDbBase, connDbMemory) connDbMemory.commit() cursorDbBase.close() connDbBase.close() #====================================================================== # Database for preprocess data #====================================================================== #database for measures and events connDbData = db.connectDb(self.readerConfig.dbDataPath) cursorData = connDbData.cursor() lmdao.createTablesData(cursorData) connDbData.commit() #====================================================================== # Set control variables #====================================================================== self.setState(th.ThreadState.PROCESSING) #====================================================================== # Main loop #====================================================================== while not self._stopRun: for line in self.follow(self.readerConfig.sourcefilepath): self.loopCount += 1 self.bytesProcessed += len(line) #============================================================== # If parser was OK, fill other data in object logEvent. #============================================================== logEvent = parser.parse(line) if logEvent == None: continue self.recordsProcessed += 1 if self.operationMode == th.OperationMode.TEST: logger.debug(str(logEvent)) continue #============================================================== # Save logmapper interval measures #============================================================== now = datetime.datetime.now() if now > self.nextSaveDataDate: self.nextSaveDataDate = lmutil.getNextLogMapperIntervalDate( now) start = lmutil.getBeforeMapperIntervalDate(now) end = lmutil.getLogMapperIntervalDate(now) measures = get.getPathMeasures(cursor, start, end) for m in measures: logger.debug(">>measure:" + str(m)) lmdao.insertPathMeasure(cursorData, date=m[0], path_id=m[1], count=m[2], duration_avg=m[3], duration_std=m[4], duration_max=m[5]) logevents = get.getLogEventsCount(cursor, start, end) for e in logevents: lmdao.insertLogEvent(cursorData, date=e[0], eventtype=e[2], count=e[3]) logMetrics = get.getLogMetrics(cursor, start, end) for m in logMetrics: lmdao.insertLogMetrics(cursorData, m[0], m[1], m[2], m[3], m[4], m[5], m[6]) connDbData.commit() if self.enableDataSend: j = get.getJsonPathMeasures(self.readerConfig, start, end) j['lmstats'] = self.getCounters() logger.debug("send1:" + str(start) + " - " + str(end)) sender = ds.DataSenderThread( self.name, self.readerConfig.masterHost, self.readerConfig.masterPort, lmkey.DATATYPE_PATH_METRICS, j) sender.start() j = get.getJsonLogEventsCount(self.readerConfig, start, end) logger.debug("send2:" + str(start) + " - " + str(end)) sender2 = ds.DataSenderThread( self.name, self.readerConfig.masterHost, self.readerConfig.masterPort, lmkey.DATATYPE_LOG_EVENTS, j) sender2.start() j = get.getJsonLogMetrics(self.readerConfig, start, end) logger.debug("send3:" + str(start) + " - " + str(end)) sender3 = ds.DataSenderThread( self.name, self.readerConfig.masterHost, self.readerConfig.masterPort, lmkey.DATATYPE_LOG_METRICS, j) sender3.start() #============================================================== # check day change, reset memory database #============================================================== if self.currentDate != now.date(): dayBefore = str(self.currentDate) logger.info('Day end. reset data: ' + dayBefore) self.currentDate = datetime.date.today() #Save base db #db.deleteDbFile(self.readerConfig.dbBasePath) #connDbBase=db.connectDb(self.readerConfig.dbBasePath) #db.copyDbDataSelection(connDbMemory, connDbBase, ['lmp_logPathsT', 'lmp_logNodesT']) #connDbBase.close() #newFile = self.readerConfig.dbEventsPath + "." + dayBefore #connDbEvents = db.connectDb(self.readerConfig.dbEventsPath) #db.copyDbDataSelection(connDbMemory, connDbEvents, ['lmp_logEventsT']) #connDbEvents.close() #db.copyDbFile(self.readerConfig.dbEventsPath, newFile) #db.deleteDbFile(self.readerConfig.dbEventsPath) #connDbEvents=db.connectDb(self.readerConfig.dbEventsPath) #cursorDbEvents = connDbEvents.cursor() #eventdao.createTablesEvents(cursorDbEvents) #connDbEvents.close() #eventdao.truncateTableLogEvents(connDbMemory, currentEventIdUpdated) #connDbMemory.commit() #============================================================== # clean events #============================================================== if self.recordsProcessed % TRUNCATE_COUNT == 0: eventdao.truncateTableLogEvents(connDbMemory) connDbMemory.commit() self.lastTruncateDate = now #============================================================== # Find logNode, if not exist, it is created # If exist, increment counter # Vars created: logNodeId, logNodeCategory #============================================================== row = lmdao.findLogNodeIdByKey(cursor, logEvent.key) if not row: logger.debug("insertLogNode" + str(logEvent)) lmdao.insertLogNode(cursor, logEvent) connDbMemory.commit() logNodeId = cursor.lastrowid logNodeCategory = logEvent.eventCategory.value else: lmdao.updateLogNodeCount(cursor, row[0], row[1] + 1) connDbMemory.commit() logNodeId = row[0] logNodeCategory = row[2] # logger.debug("LogNode Found:"+str(logNodeId)+logNodeCategory) #============================================================== # Find logThreads, if not exist, it is created # Vars: logThreadId, logThreadCount, lastLogEventId, path1LogNodeId, path1Exectime #============================================================== threadKey = logEvent.host + ":" + logEvent.component + ":" + logEvent.threadKey row = lmdao.findLogThreadByKey(cursor, threadKey) if row: threadCreation = datetime.datetime.strptime( row[5], '%Y-%m-%d %H:%M:%S.%f') timeEvent = datetime.datetime.strptime( logEvent.timeexec, '%Y-%m-%d %H:%M:%S.%f') duration = timeEvent - threadCreation #datetime.timedelta if duration.seconds > 600: lmdao.setLogThreadInvalid(cursor, threadKey, row[0]) row = None else: logThreadId = row[0] logThreadCount = row[1] lastLogEventId = row[2] path1LogNodeId = row[3] path1Exectime = row[4] if not row: lmdao.insertLogThread(cursor, threadKey, logEvent.timeexec) connDbMemory.commit() logThreadId = cursor.lastrowid logThreadCount = 1 lastLogEventId = None path1LogNodeId = None path1Exectime = None #============================================================== # create logEvent # vars: logEventId #============================================================== eventdao.insertLogEvent(cursor, logThreadId, logNodeId, logEvent.timeexec, logEvent.remoteCallKey, logEvent.userKey, lastLogEventId, logNodeCategory) connDbMemory.commit() logEventId = cursor.lastrowid now = datetime.datetime.now() timelog = datetime.datetime.strptime(logEvent.timeexec, '%Y-%m-%d %H:%M:%S.%f') duration = now - timelog #datetime.timedelta # logger.debug("logEventId="+str(logEventId)+", delay="+str(duration.seconds)) if duration.seconds > 5: logger.warning(self.name + ": logEventId=" + str(logEventId) + ", delay=" + str(duration.seconds)) #============================================================== # update thread, check relation, search path #============================================================== if logNodeCategory == LogEventCategories.TRACE_MAIN_NODE.value: if path1LogNodeId: row = lmdao.findPathByNodes(cursor, path1LogNodeId, logNodeId) if row: pathId = row[0] timeEvent = datetime.datetime.strptime( logEvent.timeexec, '%Y-%m-%d %H:%M:%S.%f') timeEventBefore = datetime.datetime.strptime( path1Exectime, '%Y-%m-%d %H:%M:%S.%f' ) #2017-04-06 15:39:28.890 duration = timeEvent - timeEventBefore #datetime.timedelta desviation = None if row[2]: #if duration_avg desviation = duration normalized (x - AVG)/STD desviation = (duration.total_seconds() - row[2]) / row[3] eventdao.updateLogEventDurationAndPathId( cursor, logEventId, duration.total_seconds(), pathId, desviation) connDbMemory.commit() self.countPathsFound += 1 # logger.debug("Update eventLog " + str(logEventId)+" with "+str(pathId)) else: logger.debug("Path not found:" + str(path1LogNodeId) + "->" + str(logNodeId)) path1LogNodeId = logNodeId path1Exectime = logEvent.timeexec # elif logNodeCategory == lmdao.NODE_CATEGORY_ISSUE: # eventdao.updateLogEventIssueLogNodeId(cursor, logEventId, logNodeId) # connDbMemory.commit() logThreadCount += 1 lastLogEventId = logEventId finishDate = logEvent.timeexec lmdao.updateLogThread(cursor, logThreadId, logThreadCount, finishDate, lastLogEventId, path1LogNodeId, path1Exectime) connDbMemory.commit() self.setState(th.ThreadState.ENDING) connDbMemory.commit() connDbData.close() #Save base db db.deleteDbFile(self.readerConfig.dbBasePath) connDbBase = db.connectDb(self.readerConfig.dbBasePath) db.copyDbDataSelection(connDbMemory, connDbBase, [ 'lmp_logPathsT', 'lmp_logNodesT', 'lmp_logThreadsT', 'lmp_logDetailedPathsT' ]) connDbBase.close() #Save events db if self.enableSaveEvents: #db.deleteDbFile(self.readerConfig.dbEventsPath) connDbEvents = db.connectDb(self.readerConfig.dbEventsPath) db.copyDbDataSelection(connDbMemory, connDbEvents, ['lmp_logEventsT']) connDbEvents.close() connDbMemory.close()