def __init__(self, username, workdir, workflowId, daxen, eventStream, bufferSize):
logger.debug ("workflow-monitor: <init>")
self.username = username
self.workdir = workdir
self.scanner = EventScanner (workdir)
self.detailScanner = EventDetailScanner ()
self.workflowId = workflowId
self.daxen = daxen
self.maxTimestamp = 0
self.status = None
self.totalEvents = 0
self.cycleEventsA = 0 # all
self.cycleEventsT = 0 # total
self.isRunning = False
self.isComplete = False
self.examineWorkflow ()
self.bufferSize = 0 if self.isRunning else bufferSize
self.eventStream = eventStream.getEventContext (bufferSize)
class WorkflowMonitor (Processor):
def __init__(self, username, workdir, workflowId, daxen, eventStream, bufferSize):
logger.debug ("workflow-monitor: <init>")
self.username = username
self.workdir = workdir
self.scanner = EventScanner (workdir)
self.detailScanner = EventDetailScanner ()
self.workflowId = workflowId
self.daxen = daxen
self.maxTimestamp = 0
self.status = None
self.totalEvents = 0
self.cycleEventsA = 0 # all
self.cycleEventsT = 0 # total
self.isRunning = False
self.isComplete = False
self.examineWorkflow ()
self.bufferSize = 0 if self.isRunning else bufferSize
self.eventStream = eventStream.getEventContext (bufferSize)
def examineWorkflow (self):
jobstatelog = os.path.join (self.workdir, 'jobstate.log')
sched_id = 0
def process (line):
finishedTag = 'DAGMAN_FINISHED'
schedIdTag = 'DAGMAN STARTED'
index = line.find (schedIdTag)
if index > -1:
sched_id = line.split (' ')[-2]
index = line.find (finishedTag)
if index > -1:
self.isComplete = True
text = GraysonUtil.readFile (jobstatelog, process)
output = []
executor = Executor ({
'condorHome' : os.environ ['CONDOR_HOME'],
'sched_id' : sched_id
})
executor.execute (command = "${condorHome}/bin/condor_q ${sched_id} -format '%s' JobStatus",
pipe = True,
processor = lambda n : output.append (n))
self.isRunning = ''.join (output) == WorkflowStatus.CONDOR_JOB_STATUS__RUNNING
logger.debug ("WorkflowMonitor - isRunning=%s, isComplete=%s", self.isRunning, self.isComplete)
def accepts (self, daxName):
accepts = False
if len(self.daxen) == 0: # no filters specified, accept everything.
accepts = True
else:
if daxName:
for key in self.daxen:
# starts with the filter and either (a) rest is '.dax' or (b) key does not contain '.' ... i.e. is not an indexed dax.
if daxName.find (key) == 0 and ( (len(daxName) == (len(key) + 4)) or (daxName.find('.') == -1) ):
accepts = True
break
#logger.debug ("dax: %s, filter: %s, accepts: %s", daxName, self.daxen, accepts)
return accepts
def processEvent (self, event):
self.totalEvents += 1
self.cycleEventsT += 1
if self.accepts (event.dax_file) or event.name.find ("subdax_") == 0:
if event.state in WorkflowStatus.STATUS_MAP:
logger.debug (" -------(sendevent: %s)", event.name)
self.sendEvent (event)
self.cycleEventsA += 1
def runCycle (self):
maxTimestampIn = self.maxTimestamp
self.cycleEventsA = 0
self.cycleEventsT = 0
self.status, self.maxTimestamp = self.scanner.getEvents (processor = self,
since = self.maxTimestamp,
daxen = self.daxen)
if logger.isEnabledFor (logging.DEBUG):
logger.debug ("monitor cycle: flow[%s] maxTimestamp[in->%s, out->%s] events[cycle-accepted->%s, cycle-all->%s, total->%s] daxen:%s wf[%s]",
self.workflowId, maxTimestampIn, self.maxTimestamp,
self.cycleEventsA, self.cycleEventsT, self.totalEvents, self.daxen, self.workflowId)
return self.status
def finish (self):
logger.debug ("workflow-monitor.finish: send end event. workflow(%s)", self.workflowId)
self.eventStream.sendEndEvent (self.username, self.workflowId)
def sendEvent (self, event):
status = self.translateExitCode (event)
logdir = os.path.abspath (event.work_dir)
self.sendTranslatedEvent (event, status, logdir)
def sendTranslatedEvent (self, event, status, logdir):
self.eventStream.sendJobStatusEvent (username = self.username,
flowId = self.workflowId,
jobid = event.name,
status = self.translateExitCode (event),
logdir = logdir,
evt_time = event.timestamp,
aux = self.detailScanner.detectEventDetails (event, logdir, status))
def translateExitCode (self, event):
return WorkflowStatus.STATUS_MAP [event.state] if event.state in WorkflowStatus.STATUS_MAP else None