def _determineProcessStatus(self, procs):
"""
Determine the up/down/restarted status of processes.
@parameter procs: array of pid, (name_with_args) info
@type procs: list
@parameter deviceStats:
@type procs:
"""
beforePids = set(self._deviceStats.pids)
afterPidToProcessStats = {}
for pid, name_with_args in procs:
log.debug("pid: %s --- name_with_args: %s" % (pid, name_with_args))
for pStats in self._deviceStats.processStats:
if pStats._config.name is not None:
if pStats.matches(name_with_args):
log.debug("Found process %s belonging to %s",
name_with_args, pStats._config)
afterPidToProcessStats[pid] = pStats
break
afterPids = set(afterPidToProcessStats)
afterByConfig = reverseDict(afterPidToProcessStats)
restarted = {}
(deadPids, restartedPids, newPids) = determineProcessState(
reverseDict(self._deviceStats._pidToProcess), afterByConfig)
restarted = {}
for restartedPid in restartedPids:
ZenProcessTask.RESTARTED += 1
procStats = afterPidToProcessStats[restartedPid]
pConfig = procStats._config
# only if configured to alert on restarts...
if pConfig.restart:
restarted[procStats] = pConfig
# populate missing (the process set contains 0 processes...)
missing = []
for procStat in self._deviceStats.processStats:
if procStat not in afterByConfig:
missing.append(procStat._config)
# For historical reasons, return the beforeByConfig
beforeByConfig = reverseDict(self._deviceStats._pidToProcess)
return (afterByConfig, afterPidToProcessStats, beforeByConfig, newPids,
restarted, deadPids, missing)
def _determineProcessStatus(self, procs):
"""
Determine the up/down/restarted status of processes.
@parameter procs: array of pid, (name_with_args) info
@type procs: list
@parameter deviceStats:
@type procs:
"""
beforePids = set(self._deviceStats.pids)
afterPidToProcessStats = {}
for pid, name_with_args in procs:
log.debug("pid: %s --- name_with_args: %s" % (pid, name_with_args))
for pStats in self._deviceStats.processStats:
if pStats._config.name is not None:
if pStats.matches(name_with_args):
log.debug("Found process %s belonging to %s", name_with_args, pStats._config)
afterPidToProcessStats[pid] = pStats
break
afterPids = set(afterPidToProcessStats)
afterByConfig = reverseDict(afterPidToProcessStats)
restarted = {}
(deadPids, restartedPids, newPids) = determineProcessState(reverseDict(self._deviceStats._pidToProcess), afterByConfig)
restarted = {}
for restartedPid in restartedPids:
ZenProcessTask.RESTARTED += 1
procStats = afterPidToProcessStats[restartedPid]
pConfig = procStats._config
# only if configured to alert on restarts...
if pConfig.restart:
restarted[procStats] = pConfig
# populate missing (the process set contains 0 processes...)
missing = []
for procStat in self._deviceStats.processStats:
if procStat not in afterByConfig:
missing.append(procStat._config)
# For historical reasons, return the beforeByConfig
beforeByConfig = reverseDict(self._deviceStats._pidToProcess)
return (afterByConfig, afterPidToProcessStats,
beforeByConfig, newPids, restarted, deadPids, missing)
def processResults(self, cmd, results):
matcher = OSProcessDataMatcher(
includeRegex = cmd.includeRegex,
excludeRegex = cmd.excludeRegex,
replaceRegex = cmd.replaceRegex,
replacement = cmd.replacement,
primaryUrlPath = cmd.primaryUrlPath,
generatedId = cmd.generatedId)
def matches(processMetrics):
pid, rss, cpu, cmdAndArgs = processMetrics
return matcher.matches(cmdAndArgs)
lines = cmd.result.output.splitlines()[1:]
metrics = map(self._extractProcessMetrics, lines)
matchingMetrics = filter(matches, metrics)
# We can not take into account processes that have already been
# matched by other process class
if hasattr(cmd, 'already_matched_cmdAndArgs'):
if cmd.already_matched_cmdAndArgs:
matchingMetrics = [ m for m in matchingMetrics if m[3] not in cmd.already_matched_cmdAndArgs ]
cmd.already_matched_cmdAndArgs.extend([ m[3] for m in matchingMetrics ])
else:
cmd.already_matched_cmdAndArgs = [ m[3] for m in matchingMetrics ]
pids, rss, cpu = self._combineProcessMetrics(matchingMetrics)
processSet = cmd.displayName
# report any processes that are missing, and post perf data
missingeventSent = False
for dp in cmd.points:
# cmd.points = list of tuples ... each tuple contains one of the following:
# dictionary, count
# dictionary, cpu
# dictionary, mem
if pids:
if 'cpu' in dp.id:
results.values.append( (dp, cpu) )
if 'mem' in dp.id:
results.values.append( (dp, rss) )
if 'count' in dp.id:
results.values.append( (dp, len(pids)) )
else:
if 'count' in dp.id:
results.values.append((dp,0))
failSeverity = dp.data['failSeverity']
# alert on missing (the process set contains 0 processes...)
summary = 'Process set contains 0 running processes: %s' % processSet
message = "%s\n Using regex \'%s\' \n All Processes have stopped since the last model occurred. Last Modification time (%s)" \
% (summary, cmd.includeRegex, cmd.deviceConfig.lastmodeltime)
if missingeventSent != summary:
self.sendEvent(results,
device=cmd.deviceConfig.device,
summary=summary,
message=message,
component=processSet,
eventKey=cmd.generatedId,
severity=failSeverity)
log.warning("(%s) %s" % (cmd.deviceConfig.device, message))
missingeventSent = summary
# When not instantiated for each call fixes missing messages
missingeventSent = False
# Report process changes
# Note that we can't tell the difference between a
# reconfiguration from zenhub and process that goes away.
device = cmd.deviceConfig.device
# Retrieve the current processes and corresponding pids
afterPidsProcesses = {}
if pids:
afterPidsProcesses = pids
afterPids = afterPidsProcesses.keys()
afterProcessSetPIDs = {}
afterProcessSetPIDs[processSet] = afterPids
# Globals.MostRecentMonitoredTimePids is a global that simply keeps the most recent data ... used to retrieve the "before" at monitoring time
if Globals.MostRecentMonitoredTimePids.get(device, None):
beforePidsProcesses = Globals.MostRecentMonitoredTimePids[device].get(processSet, None)
else:
beforePidsProcesses = Globals.MostRecentMonitoredTimePids[device] = {}
# the first time this runs ... there is no "before"
# this occurs when beforePidsProcesses is an empty dict
# we need to save off the current processes and continue til the next monitoring time when "before" and "after" will be present
if beforePidsProcesses is None:
log.debug("No existing 'before' process information for process set: %s ... skipping" % processSet)
Globals.MostRecentMonitoredTimePids[device][processSet] = afterPidsProcesses
return
beforePids = beforePidsProcesses.keys()
beforeProcessSetPIDs = {}
beforeProcessSetPIDs[processSet] = beforePids
processState = determineProcessState(beforeProcessSetPIDs, afterProcessSetPIDs)
(deadPids, restartedPids, newPids) = processState
# only if configured to alert on restarts...
alertOnRestart = dp.data['alertOnRestart']
if alertOnRestart and restartedPids:
droppedPids = []
for pid in beforeProcessSetPIDs[processSet]:
if pid not in afterProcessSetPIDs[processSet]:
droppedPids.append(pid)
summary = 'Process(es) restarted in process set: %s' % processSet
message = '%s\n Using regex \'%s\' Discarded dead pid(s) %s Using new pid(s) %s'\
% (summary, cmd.includeRegex, droppedPids, afterProcessSetPIDs[processSet])
self.sendEvent(results,
device=cmd.deviceConfig.device,
summary=summary,
message=message,
component=processSet,
eventKey=cmd.generatedId,
severity=cmd.severity)
log.info("(%s) %s" % (cmd.deviceConfig.device, message))
# report alive processes
for alivePid in afterProcessSetPIDs[processSet]:
if alivePid in restartedPids:
continue
summary = "Process up: %s" % processSet
message = '%s\n Using regex \'%s\' with pid\'s %s ' % (summary, cmd.includeRegex, alivePid)
self.sendEvent(results,
device=cmd.deviceConfig.device,
summary=summary,
message=message,
component=processSet,
eventKey=cmd.generatedId,
severity=Event.Clear)
log.debug("(%s) %s" % (cmd.deviceConfig.device, message))
for newPid in newPids:
log.debug("found new process: %s (pid: %d) on %s" % (afterPidsProcesses[newPid], newPid, cmd.deviceConfig.device))
Globals.MostRecentMonitoredTimePids[device][processSet] = afterPidsProcesses
def processResults(self, cmd, results):
matcher = OSProcessDataMatcher(includeRegex=cmd.includeRegex,
excludeRegex=cmd.excludeRegex,
replaceRegex=cmd.replaceRegex,
replacement=cmd.replacement,
primaryUrlPath=cmd.primaryUrlPath,
generatedId=cmd.generatedId)
def matches(processMetrics):
pid, rss, cpu, cmdAndArgs = processMetrics
return matcher.matches(cmdAndArgs)
lines = cmd.result.output.splitlines()[1:]
metrics = map(self._extractProcessMetrics, lines)
matchingMetrics = filter(matches, metrics)
# We can not take into account processes that have already been
# matched by other process class
if hasattr(cmd, 'already_matched_cmdAndArgs'):
if cmd.already_matched_cmdAndArgs:
matchingMetrics = [
m for m in matchingMetrics
if m[3] not in cmd.already_matched_cmdAndArgs
]
cmd.already_matched_cmdAndArgs.extend(
[m[3] for m in matchingMetrics])
else:
cmd.already_matched_cmdAndArgs = [
m[3] for m in matchingMetrics
]
pids, rss, cpu = self._combineProcessMetrics(matchingMetrics)
processSet = cmd.displayName
# report any processes that are missing, and post perf data
missingeventSent = False
for dp in cmd.points:
# cmd.points = list of tuples ... each tuple contains one of the following:
# dictionary, count
# dictionary, cpu
# dictionary, mem
if pids:
if 'cpu' in dp.id:
results.values.append((dp, cpu))
if 'mem' in dp.id:
results.values.append((dp, rss))
if 'count' in dp.id:
results.values.append((dp, len(pids)))
else:
if 'count' in dp.id:
results.values.append((dp, 0))
failSeverity = dp.data['failSeverity']
# alert on missing (the process set contains 0 processes...)
summary = 'Process set contains 0 running processes: %s' % processSet
message = "%s\n Using regex \'%s\' \n All Processes have stopped since the last model occurred. Last Modification time (%s)" \
% (summary, cmd.includeRegex, cmd.deviceConfig.lastmodeltime)
if missingeventSent != summary:
self.sendEvent(results,
device=cmd.deviceConfig.device,
summary=summary,
message=message,
component=processSet,
eventKey=cmd.generatedId,
severity=failSeverity)
log.warning("(%s) %s" % (cmd.deviceConfig.device, message))
missingeventSent = summary
# When not instantiated for each call fixes missing messages
missingeventSent = False
# Report process changes
# Note that we can't tell the difference between a
# reconfiguration from zenhub and process that goes away.
device = cmd.deviceConfig.device
# Retrieve the current processes and corresponding pids
afterPidsProcesses = {}
if pids:
afterPidsProcesses = pids
afterPids = afterPidsProcesses.keys()
afterProcessSetPIDs = {}
afterProcessSetPIDs[processSet] = afterPids
# Globals.MostRecentMonitoredTimePids is a global that simply keeps the most recent data ... used to retrieve the "before" at monitoring time
if Globals.MostRecentMonitoredTimePids.get(device, None):
beforePidsProcesses = Globals.MostRecentMonitoredTimePids[
device].get(processSet, None)
else:
beforePidsProcesses = Globals.MostRecentMonitoredTimePids[
device] = {}
# the first time this runs ... there is no "before"
# this occurs when beforePidsProcesses is an empty dict
# we need to save off the current processes and continue til the next monitoring time when "before" and "after" will be present
if beforePidsProcesses is None:
log.debug(
"No existing 'before' process information for process set: %s ... skipping"
% processSet)
Globals.MostRecentMonitoredTimePids[device][
processSet] = afterPidsProcesses
return
beforePids = beforePidsProcesses.keys()
beforeProcessSetPIDs = {}
beforeProcessSetPIDs[processSet] = beforePids
processState = determineProcessState(beforeProcessSetPIDs,
afterProcessSetPIDs)
(deadPids, restartedPids, newPids) = processState
# only if configured to alert on restarts...
alertOnRestart = dp.data['alertOnRestart']
if alertOnRestart and restartedPids:
droppedPids = []
for pid in beforeProcessSetPIDs[processSet]:
if pid not in afterProcessSetPIDs[processSet]:
droppedPids.append(pid)
summary = 'Process(es) restarted in process set: %s' % processSet
message = '%s\n Using regex \'%s\' Discarded dead pid(s) %s Using new pid(s) %s'\
% (summary, cmd.includeRegex, droppedPids, afterProcessSetPIDs[processSet])
self.sendEvent(results,
device=cmd.deviceConfig.device,
summary=summary,
message=message,
component=processSet,
eventKey=cmd.generatedId,
severity=cmd.severity)
log.info("(%s) %s" % (cmd.deviceConfig.device, message))
# report alive processes
for alivePid in afterProcessSetPIDs[processSet]:
if alivePid in restartedPids:
continue
summary = "Process up: %s" % processSet
message = '%s\n Using regex \'%s\' with pid\'s %s ' % (
summary, cmd.includeRegex, alivePid)
self.sendEvent(results,
device=cmd.deviceConfig.device,
summary=summary,
message=message,
component=processSet,
eventKey=cmd.generatedId,
severity=Event.Clear)
log.debug("(%s) %s" % (cmd.deviceConfig.device, message))
for newPid in newPids:
log.debug(
"found new process: %s (pid: %d) on %s" %
(afterPidsProcesses[newPid], newPid, cmd.deviceConfig.device))
Globals.MostRecentMonitoredTimePids[device][
processSet] = afterPidsProcesses