def __init__(self, username, workdir, workflowId, daxen, eventStream, bufferSize): logger.debug ("workflow-monitor: <init>") self.username = username self.workdir = workdir self.scanner = EventScanner (workdir) self.detailScanner = EventDetailScanner () self.workflowId = workflowId self.daxen = daxen self.maxTimestamp = 0 self.status = None self.totalEvents = 0 self.cycleEventsA = 0 # all self.cycleEventsT = 0 # total self.isRunning = False self.isComplete = False self.examineWorkflow () self.bufferSize = 0 if self.isRunning else bufferSize self.eventStream = eventStream.getEventContext (bufferSize)
class WorkflowMonitor (Processor): def __init__(self, username, workdir, workflowId, daxen, eventStream, bufferSize): logger.debug ("workflow-monitor: <init>") self.username = username self.workdir = workdir self.scanner = EventScanner (workdir) self.detailScanner = EventDetailScanner () self.workflowId = workflowId self.daxen = daxen self.maxTimestamp = 0 self.status = None self.totalEvents = 0 self.cycleEventsA = 0 # all self.cycleEventsT = 0 # total self.isRunning = False self.isComplete = False self.examineWorkflow () self.bufferSize = 0 if self.isRunning else bufferSize self.eventStream = eventStream.getEventContext (bufferSize) def examineWorkflow (self): jobstatelog = os.path.join (self.workdir, 'jobstate.log') sched_id = 0 def process (line): finishedTag = 'DAGMAN_FINISHED' schedIdTag = 'DAGMAN STARTED' index = line.find (schedIdTag) if index > -1: sched_id = line.split (' ')[-2] index = line.find (finishedTag) if index > -1: self.isComplete = True text = GraysonUtil.readFile (jobstatelog, process) output = [] executor = Executor ({ 'condorHome' : os.environ ['CONDOR_HOME'], 'sched_id' : sched_id }) executor.execute (command = "${condorHome}/bin/condor_q ${sched_id} -format '%s' JobStatus", pipe = True, processor = lambda n : output.append (n)) self.isRunning = ''.join (output) == WorkflowStatus.CONDOR_JOB_STATUS__RUNNING logger.debug ("WorkflowMonitor - isRunning=%s, isComplete=%s", self.isRunning, self.isComplete) def accepts (self, daxName): accepts = False if len(self.daxen) == 0: # no filters specified, accept everything. accepts = True else: if daxName: for key in self.daxen: # starts with the filter and either (a) rest is '.dax' or (b) key does not contain '.' ... i.e. is not an indexed dax. if daxName.find (key) == 0 and ( (len(daxName) == (len(key) + 4)) or (daxName.find('.') == -1) ): accepts = True break #logger.debug ("dax: %s, filter: %s, accepts: %s", daxName, self.daxen, accepts) return accepts def processEvent (self, event): self.totalEvents += 1 self.cycleEventsT += 1 if self.accepts (event.dax_file) or event.name.find ("subdax_") == 0: if event.state in WorkflowStatus.STATUS_MAP: logger.debug (" -------(sendevent: %s)", event.name) self.sendEvent (event) self.cycleEventsA += 1 def runCycle (self): maxTimestampIn = self.maxTimestamp self.cycleEventsA = 0 self.cycleEventsT = 0 self.status, self.maxTimestamp = self.scanner.getEvents (processor = self, since = self.maxTimestamp, daxen = self.daxen) if logger.isEnabledFor (logging.DEBUG): logger.debug ("monitor cycle: flow[%s] maxTimestamp[in->%s, out->%s] events[cycle-accepted->%s, cycle-all->%s, total->%s] daxen:%s wf[%s]", self.workflowId, maxTimestampIn, self.maxTimestamp, self.cycleEventsA, self.cycleEventsT, self.totalEvents, self.daxen, self.workflowId) return self.status def finish (self): logger.debug ("workflow-monitor.finish: send end event. workflow(%s)", self.workflowId) self.eventStream.sendEndEvent (self.username, self.workflowId) def sendEvent (self, event): status = self.translateExitCode (event) logdir = os.path.abspath (event.work_dir) self.sendTranslatedEvent (event, status, logdir) def sendTranslatedEvent (self, event, status, logdir): self.eventStream.sendJobStatusEvent (username = self.username, flowId = self.workflowId, jobid = event.name, status = self.translateExitCode (event), logdir = logdir, evt_time = event.timestamp, aux = self.detailScanner.detectEventDetails (event, logdir, status)) def translateExitCode (self, event): return WorkflowStatus.STATUS_MAP [event.state] if event.state in WorkflowStatus.STATUS_MAP else None