def __init__(self, username, graphPrefix, workflowRoot=".", amqpSettings=None):
self.username = username
self.graphPathPrefix = graphPrefix
self.eventStream = EventStream (amqpSettings, workflowRoot).getEventContext ()
self.amqpSettings = amqpSettings
self.workflowRoot = workflowRoot
self.workflowMonitorDatabase = WorkflowMonitorDatabase ()
logger.debug ("workflowmonitorcompilerplugin:amqpsettings: %s", amqpSettings)
def ExecuteWorkflow (user, archive, archivePath, workflowRoot=".", amqpSettings=None):
logger.info ("executeworkflow:amqpsettings: %s", amqpSettings)
try:
basename = os.path.basename (archive)
unpack_dir = GraysonWebUtil.form_workflow_path (user, GraysonWebConst.UNPACK_EXT % basename)
# If we don't open up permissions, using x509userproxy doesn't work. i.e. users can't run as themselves and write files here, for example.
chmodCommand = "chmod -R 777 %s" % unpack_dir
logger.debug ("setting output directory permissions: %s" % chmodCommand)
os.system (chmodCommand)
log_file = os.path.join (unpack_dir, "log.txt")
executionMonitor = WorkflowMonitorCompilerPlugin (user.username,
unpack_dir,
workflowRoot,
amqpSettings)
logger.debug ("""
=======================================================================================
== C O M P I L E (%s)
=======================================================================================
== user.username: (%s)
== outputDir : (%s)
== logFile : (%s)
== amqp : (%s)
=======================================================================================""",
archivePath,
user.username,
unpack_dir,
log_file,
amqpSettings)
GraysonCompiler.compile (models = [ archivePath ],
outputdir = unpack_dir,
logDir = unpack_dir,
appHome = unpack_dir,
toLogFile = "log.txt",
execute = True,
logLevel = "debug",
plugin = executionMonitor)
except ValueError, e:
traceback.print_exc ()
eventStream = EventStream (amqpSettings)
eventStream.sendCompilationMessagesEvent (username = user.username,
flowId = archive,
log = log_file)
class WorkflowMonitorCompilerPlugin (object):
def __init__(self, username, graphPrefix, workflowRoot=".", amqpSettings=None):
self.username = username
self.graphPathPrefix = graphPrefix
self.eventStream = EventStream (amqpSettings, workflowRoot).getEventContext ()
self.amqpSettings = amqpSettings
self.workflowRoot = workflowRoot
self.workflowMonitorDatabase = WorkflowMonitorDatabase ()
logger.debug ("workflowmonitorcompilerplugin:amqpsettings: %s", amqpSettings)
def notifyShellEvent (self, line, outputWorkflowPath):
logger.info ("|| %s", line)
jobid = None
status = None
workdirMarker = "pegasus-status -l "
if workdirMarker in line:
''' this is a grid job '''
workDir = line [ line.rfind (workdirMarker) + len (workdirMarker) : ]
workDir = workDir.rstrip ()
logger.info ("starting grid monitor workflowId: %s, username: %s, workDir: %s", outputWorkflowPath, self.username, workDir)
idparts = workDir.split (os.sep)
if len(idparts) >= 7:
workflowName = idparts [len(idparts)-2]
workflowId = os.path.join (idparts [len(idparts)-6],
workflowName).replace ('.dax', '')
self.workflowMonitorDatabase.subscribeToWorkflow (
self.workflowRoot,
{
"username" : self.username,
"workflowId" : workflowId,
"workdir" : workDir,
"daxen" : [],
"buffer" : 0
})
elif "Executing JOB" in line:
jobid = StrUtil.between (line, "::", ":")
if " " in jobid:
jobid = StrUtil.before (jobid, " ")
status = "running"
logger.info ('--grayson:shell-compilerplugin: got executing %s %s', jobid, status)
elif "Returned with" in line:
jobid = StrUtil.between (line, " JOB ", "_")
exitcode = ""
try:
statustext = StrUtil.afterlast (line, "Returned with ").rstrip ()
status = int (statustext)
logger.info ('--grayson:shell-compilerplugin: got status %s %s %s', jobid, statustext, status)
except:
logger.error ("cant parse exit code")
traceback.print_exc ()
logger.info (' -- got returned with %s %s', jobid, status)
if jobid and status != None:
logger.info ("--grayson:shell-compilerplugin: notifying got returned with %s %s", jobid, status)
self.eventStream.sendJobStatusEvent (self.username, outputWorkflowPath, jobid, status)
def notifyWorkflowCreation (self, workflowGraph, outputWorkflowPath, topWorkflow=False):
if topWorkflow:
logger.info ("--grayson:shell-compilerplugin: notifying workflow creation: %s", workflowGraph)
graphPath = "%s.graphml" % workflowGraph
graphPath = os.path.join (self.graphPathPrefix, graphPath)
self.eventStream.sendWorkflowEvent (self.username, outputWorkflowPath, graphPath, outputWorkflowPath)
else:
logger.info ("outputWorkflowPath: %s graph: %s", outputWorkflowPath, workflowGraph)
self.eventStream.sendSubworkflowEvent (self.username, outputWorkflowPath, workflowGraph)
