def __init__(self, username, graphPrefix, workflowRoot=".", amqpSettings=None): self.username = username self.graphPathPrefix = graphPrefix self.eventStream = EventStream (amqpSettings, workflowRoot).getEventContext () self.amqpSettings = amqpSettings self.workflowRoot = workflowRoot self.workflowMonitorDatabase = WorkflowMonitorDatabase () logger.debug ("workflowmonitorcompilerplugin:amqpsettings: %s", amqpSettings)
def ExecuteWorkflow (user, archive, archivePath, workflowRoot=".", amqpSettings=None): logger.info ("executeworkflow:amqpsettings: %s", amqpSettings) try: basename = os.path.basename (archive) unpack_dir = GraysonWebUtil.form_workflow_path (user, GraysonWebConst.UNPACK_EXT % basename) # If we don't open up permissions, using x509userproxy doesn't work. i.e. users can't run as themselves and write files here, for example. chmodCommand = "chmod -R 777 %s" % unpack_dir logger.debug ("setting output directory permissions: %s" % chmodCommand) os.system (chmodCommand) log_file = os.path.join (unpack_dir, "log.txt") executionMonitor = WorkflowMonitorCompilerPlugin (user.username, unpack_dir, workflowRoot, amqpSettings) logger.debug (""" ======================================================================================= == C O M P I L E (%s) ======================================================================================= == user.username: (%s) == outputDir : (%s) == logFile : (%s) == amqp : (%s) =======================================================================================""", archivePath, user.username, unpack_dir, log_file, amqpSettings) GraysonCompiler.compile (models = [ archivePath ], outputdir = unpack_dir, logDir = unpack_dir, appHome = unpack_dir, toLogFile = "log.txt", execute = True, logLevel = "debug", plugin = executionMonitor) except ValueError, e: traceback.print_exc () eventStream = EventStream (amqpSettings) eventStream.sendCompilationMessagesEvent (username = user.username, flowId = archive, log = log_file)
class WorkflowMonitorCompilerPlugin (object): def __init__(self, username, graphPrefix, workflowRoot=".", amqpSettings=None): self.username = username self.graphPathPrefix = graphPrefix self.eventStream = EventStream (amqpSettings, workflowRoot).getEventContext () self.amqpSettings = amqpSettings self.workflowRoot = workflowRoot self.workflowMonitorDatabase = WorkflowMonitorDatabase () logger.debug ("workflowmonitorcompilerplugin:amqpsettings: %s", amqpSettings) def notifyShellEvent (self, line, outputWorkflowPath): logger.info ("|| %s", line) jobid = None status = None workdirMarker = "pegasus-status -l " if workdirMarker in line: ''' this is a grid job ''' workDir = line [ line.rfind (workdirMarker) + len (workdirMarker) : ] workDir = workDir.rstrip () logger.info ("starting grid monitor workflowId: %s, username: %s, workDir: %s", outputWorkflowPath, self.username, workDir) idparts = workDir.split (os.sep) if len(idparts) >= 7: workflowName = idparts [len(idparts)-2] workflowId = os.path.join (idparts [len(idparts)-6], workflowName).replace ('.dax', '') self.workflowMonitorDatabase.subscribeToWorkflow ( self.workflowRoot, { "username" : self.username, "workflowId" : workflowId, "workdir" : workDir, "daxen" : [], "buffer" : 0 }) elif "Executing JOB" in line: jobid = StrUtil.between (line, "::", ":") if " " in jobid: jobid = StrUtil.before (jobid, " ") status = "running" logger.info ('--grayson:shell-compilerplugin: got executing %s %s', jobid, status) elif "Returned with" in line: jobid = StrUtil.between (line, " JOB ", "_") exitcode = "" try: statustext = StrUtil.afterlast (line, "Returned with ").rstrip () status = int (statustext) logger.info ('--grayson:shell-compilerplugin: got status %s %s %s', jobid, statustext, status) except: logger.error ("cant parse exit code") traceback.print_exc () logger.info (' -- got returned with %s %s', jobid, status) if jobid and status != None: logger.info ("--grayson:shell-compilerplugin: notifying got returned with %s %s", jobid, status) self.eventStream.sendJobStatusEvent (self.username, outputWorkflowPath, jobid, status) def notifyWorkflowCreation (self, workflowGraph, outputWorkflowPath, topWorkflow=False): if topWorkflow: logger.info ("--grayson:shell-compilerplugin: notifying workflow creation: %s", workflowGraph) graphPath = "%s.graphml" % workflowGraph graphPath = os.path.join (self.graphPathPrefix, graphPath) self.eventStream.sendWorkflowEvent (self.username, outputWorkflowPath, graphPath, outputWorkflowPath) else: logger.info ("outputWorkflowPath: %s graph: %s", outputWorkflowPath, workflowGraph) self.eventStream.sendSubworkflowEvent (self.username, outputWorkflowPath, workflowGraph)