def _determineProcessStatus(self, procs): """ Determine the up/down/restarted status of processes. @parameter procs: array of pid, (name_with_args) info @type procs: list @parameter deviceStats: @type procs: """ beforePids = set(self._deviceStats.pids) afterPidToProcessStats = {} for pid, name_with_args in procs: log.debug("pid: %s --- name_with_args: %s" % (pid, name_with_args)) for pStats in self._deviceStats.processStats: if pStats._config.name is not None: if pStats.matches(name_with_args): log.debug("Found process %s belonging to %s", name_with_args, pStats._config) afterPidToProcessStats[pid] = pStats break afterPids = set(afterPidToProcessStats) afterByConfig = reverseDict(afterPidToProcessStats) restarted = {} (deadPids, restartedPids, newPids) = determineProcessState( reverseDict(self._deviceStats._pidToProcess), afterByConfig) restarted = {} for restartedPid in restartedPids: ZenProcessTask.RESTARTED += 1 procStats = afterPidToProcessStats[restartedPid] pConfig = procStats._config # only if configured to alert on restarts... if pConfig.restart: restarted[procStats] = pConfig # populate missing (the process set contains 0 processes...) missing = [] for procStat in self._deviceStats.processStats: if procStat not in afterByConfig: missing.append(procStat._config) # For historical reasons, return the beforeByConfig beforeByConfig = reverseDict(self._deviceStats._pidToProcess) return (afterByConfig, afterPidToProcessStats, beforeByConfig, newPids, restarted, deadPids, missing)
def _determineProcessStatus(self, procs): """ Determine the up/down/restarted status of processes. @parameter procs: array of pid, (name_with_args) info @type procs: list @parameter deviceStats: @type procs: """ beforePids = set(self._deviceStats.pids) afterPidToProcessStats = {} for pid, name_with_args in procs: log.debug("pid: %s --- name_with_args: %s" % (pid, name_with_args)) for pStats in self._deviceStats.processStats: if pStats._config.name is not None: if pStats.matches(name_with_args): log.debug("Found process %s belonging to %s", name_with_args, pStats._config) afterPidToProcessStats[pid] = pStats break afterPids = set(afterPidToProcessStats) afterByConfig = reverseDict(afterPidToProcessStats) restarted = {} (deadPids, restartedPids, newPids) = determineProcessState(reverseDict(self._deviceStats._pidToProcess), afterByConfig) restarted = {} for restartedPid in restartedPids: ZenProcessTask.RESTARTED += 1 procStats = afterPidToProcessStats[restartedPid] pConfig = procStats._config # only if configured to alert on restarts... if pConfig.restart: restarted[procStats] = pConfig # populate missing (the process set contains 0 processes...) missing = [] for procStat in self._deviceStats.processStats: if procStat not in afterByConfig: missing.append(procStat._config) # For historical reasons, return the beforeByConfig beforeByConfig = reverseDict(self._deviceStats._pidToProcess) return (afterByConfig, afterPidToProcessStats, beforeByConfig, newPids, restarted, deadPids, missing)
def processResults(self, cmd, results): matcher = OSProcessDataMatcher( includeRegex = cmd.includeRegex, excludeRegex = cmd.excludeRegex, replaceRegex = cmd.replaceRegex, replacement = cmd.replacement, primaryUrlPath = cmd.primaryUrlPath, generatedId = cmd.generatedId) def matches(processMetrics): pid, rss, cpu, cmdAndArgs = processMetrics return matcher.matches(cmdAndArgs) lines = cmd.result.output.splitlines()[1:] metrics = map(self._extractProcessMetrics, lines) matchingMetrics = filter(matches, metrics) # We can not take into account processes that have already been # matched by other process class if hasattr(cmd, 'already_matched_cmdAndArgs'): if cmd.already_matched_cmdAndArgs: matchingMetrics = [ m for m in matchingMetrics if m[3] not in cmd.already_matched_cmdAndArgs ] cmd.already_matched_cmdAndArgs.extend([ m[3] for m in matchingMetrics ]) else: cmd.already_matched_cmdAndArgs = [ m[3] for m in matchingMetrics ] pids, rss, cpu = self._combineProcessMetrics(matchingMetrics) processSet = cmd.displayName # report any processes that are missing, and post perf data missingeventSent = False for dp in cmd.points: # cmd.points = list of tuples ... each tuple contains one of the following: # dictionary, count # dictionary, cpu # dictionary, mem if pids: if 'cpu' in dp.id: results.values.append( (dp, cpu) ) if 'mem' in dp.id: results.values.append( (dp, rss) ) if 'count' in dp.id: results.values.append( (dp, len(pids)) ) else: if 'count' in dp.id: results.values.append((dp,0)) failSeverity = dp.data['failSeverity'] # alert on missing (the process set contains 0 processes...) summary = 'Process set contains 0 running processes: %s' % processSet message = "%s\n Using regex \'%s\' \n All Processes have stopped since the last model occurred. Last Modification time (%s)" \ % (summary, cmd.includeRegex, cmd.deviceConfig.lastmodeltime) if missingeventSent != summary: self.sendEvent(results, device=cmd.deviceConfig.device, summary=summary, message=message, component=processSet, eventKey=cmd.generatedId, severity=failSeverity) log.warning("(%s) %s" % (cmd.deviceConfig.device, message)) missingeventSent = summary # When not instantiated for each call fixes missing messages missingeventSent = False # Report process changes # Note that we can't tell the difference between a # reconfiguration from zenhub and process that goes away. device = cmd.deviceConfig.device # Retrieve the current processes and corresponding pids afterPidsProcesses = {} if pids: afterPidsProcesses = pids afterPids = afterPidsProcesses.keys() afterProcessSetPIDs = {} afterProcessSetPIDs[processSet] = afterPids # Globals.MostRecentMonitoredTimePids is a global that simply keeps the most recent data ... used to retrieve the "before" at monitoring time if Globals.MostRecentMonitoredTimePids.get(device, None): beforePidsProcesses = Globals.MostRecentMonitoredTimePids[device].get(processSet, None) else: beforePidsProcesses = Globals.MostRecentMonitoredTimePids[device] = {} # the first time this runs ... there is no "before" # this occurs when beforePidsProcesses is an empty dict # we need to save off the current processes and continue til the next monitoring time when "before" and "after" will be present if beforePidsProcesses is None: log.debug("No existing 'before' process information for process set: %s ... skipping" % processSet) Globals.MostRecentMonitoredTimePids[device][processSet] = afterPidsProcesses return beforePids = beforePidsProcesses.keys() beforeProcessSetPIDs = {} beforeProcessSetPIDs[processSet] = beforePids processState = determineProcessState(beforeProcessSetPIDs, afterProcessSetPIDs) (deadPids, restartedPids, newPids) = processState # only if configured to alert on restarts... alertOnRestart = dp.data['alertOnRestart'] if alertOnRestart and restartedPids: droppedPids = [] for pid in beforeProcessSetPIDs[processSet]: if pid not in afterProcessSetPIDs[processSet]: droppedPids.append(pid) summary = 'Process(es) restarted in process set: %s' % processSet message = '%s\n Using regex \'%s\' Discarded dead pid(s) %s Using new pid(s) %s'\ % (summary, cmd.includeRegex, droppedPids, afterProcessSetPIDs[processSet]) self.sendEvent(results, device=cmd.deviceConfig.device, summary=summary, message=message, component=processSet, eventKey=cmd.generatedId, severity=cmd.severity) log.info("(%s) %s" % (cmd.deviceConfig.device, message)) # report alive processes for alivePid in afterProcessSetPIDs[processSet]: if alivePid in restartedPids: continue summary = "Process up: %s" % processSet message = '%s\n Using regex \'%s\' with pid\'s %s ' % (summary, cmd.includeRegex, alivePid) self.sendEvent(results, device=cmd.deviceConfig.device, summary=summary, message=message, component=processSet, eventKey=cmd.generatedId, severity=Event.Clear) log.debug("(%s) %s" % (cmd.deviceConfig.device, message)) for newPid in newPids: log.debug("found new process: %s (pid: %d) on %s" % (afterPidsProcesses[newPid], newPid, cmd.deviceConfig.device)) Globals.MostRecentMonitoredTimePids[device][processSet] = afterPidsProcesses
def processResults(self, cmd, results): matcher = OSProcessDataMatcher(includeRegex=cmd.includeRegex, excludeRegex=cmd.excludeRegex, replaceRegex=cmd.replaceRegex, replacement=cmd.replacement, primaryUrlPath=cmd.primaryUrlPath, generatedId=cmd.generatedId) def matches(processMetrics): pid, rss, cpu, cmdAndArgs = processMetrics return matcher.matches(cmdAndArgs) lines = cmd.result.output.splitlines()[1:] metrics = map(self._extractProcessMetrics, lines) matchingMetrics = filter(matches, metrics) # We can not take into account processes that have already been # matched by other process class if hasattr(cmd, 'already_matched_cmdAndArgs'): if cmd.already_matched_cmdAndArgs: matchingMetrics = [ m for m in matchingMetrics if m[3] not in cmd.already_matched_cmdAndArgs ] cmd.already_matched_cmdAndArgs.extend( [m[3] for m in matchingMetrics]) else: cmd.already_matched_cmdAndArgs = [ m[3] for m in matchingMetrics ] pids, rss, cpu = self._combineProcessMetrics(matchingMetrics) processSet = cmd.displayName # report any processes that are missing, and post perf data missingeventSent = False for dp in cmd.points: # cmd.points = list of tuples ... each tuple contains one of the following: # dictionary, count # dictionary, cpu # dictionary, mem if pids: if 'cpu' in dp.id: results.values.append((dp, cpu)) if 'mem' in dp.id: results.values.append((dp, rss)) if 'count' in dp.id: results.values.append((dp, len(pids))) else: if 'count' in dp.id: results.values.append((dp, 0)) failSeverity = dp.data['failSeverity'] # alert on missing (the process set contains 0 processes...) summary = 'Process set contains 0 running processes: %s' % processSet message = "%s\n Using regex \'%s\' \n All Processes have stopped since the last model occurred. Last Modification time (%s)" \ % (summary, cmd.includeRegex, cmd.deviceConfig.lastmodeltime) if missingeventSent != summary: self.sendEvent(results, device=cmd.deviceConfig.device, summary=summary, message=message, component=processSet, eventKey=cmd.generatedId, severity=failSeverity) log.warning("(%s) %s" % (cmd.deviceConfig.device, message)) missingeventSent = summary # When not instantiated for each call fixes missing messages missingeventSent = False # Report process changes # Note that we can't tell the difference between a # reconfiguration from zenhub and process that goes away. device = cmd.deviceConfig.device # Retrieve the current processes and corresponding pids afterPidsProcesses = {} if pids: afterPidsProcesses = pids afterPids = afterPidsProcesses.keys() afterProcessSetPIDs = {} afterProcessSetPIDs[processSet] = afterPids # Globals.MostRecentMonitoredTimePids is a global that simply keeps the most recent data ... used to retrieve the "before" at monitoring time if Globals.MostRecentMonitoredTimePids.get(device, None): beforePidsProcesses = Globals.MostRecentMonitoredTimePids[ device].get(processSet, None) else: beforePidsProcesses = Globals.MostRecentMonitoredTimePids[ device] = {} # the first time this runs ... there is no "before" # this occurs when beforePidsProcesses is an empty dict # we need to save off the current processes and continue til the next monitoring time when "before" and "after" will be present if beforePidsProcesses is None: log.debug( "No existing 'before' process information for process set: %s ... skipping" % processSet) Globals.MostRecentMonitoredTimePids[device][ processSet] = afterPidsProcesses return beforePids = beforePidsProcesses.keys() beforeProcessSetPIDs = {} beforeProcessSetPIDs[processSet] = beforePids processState = determineProcessState(beforeProcessSetPIDs, afterProcessSetPIDs) (deadPids, restartedPids, newPids) = processState # only if configured to alert on restarts... alertOnRestart = dp.data['alertOnRestart'] if alertOnRestart and restartedPids: droppedPids = [] for pid in beforeProcessSetPIDs[processSet]: if pid not in afterProcessSetPIDs[processSet]: droppedPids.append(pid) summary = 'Process(es) restarted in process set: %s' % processSet message = '%s\n Using regex \'%s\' Discarded dead pid(s) %s Using new pid(s) %s'\ % (summary, cmd.includeRegex, droppedPids, afterProcessSetPIDs[processSet]) self.sendEvent(results, device=cmd.deviceConfig.device, summary=summary, message=message, component=processSet, eventKey=cmd.generatedId, severity=cmd.severity) log.info("(%s) %s" % (cmd.deviceConfig.device, message)) # report alive processes for alivePid in afterProcessSetPIDs[processSet]: if alivePid in restartedPids: continue summary = "Process up: %s" % processSet message = '%s\n Using regex \'%s\' with pid\'s %s ' % ( summary, cmd.includeRegex, alivePid) self.sendEvent(results, device=cmd.deviceConfig.device, summary=summary, message=message, component=processSet, eventKey=cmd.generatedId, severity=Event.Clear) log.debug("(%s) %s" % (cmd.deviceConfig.device, message)) for newPid in newPids: log.debug( "found new process: %s (pid: %d) on %s" % (afterPidsProcesses[newPid], newPid, cmd.deviceConfig.device)) Globals.MostRecentMonitoredTimePids[device][ processSet] = afterPidsProcesses