def __init__(self, username, graphPrefix, workflowRoot=".", amqpSettings=None):
self.username = username
self.graphPathPrefix = graphPrefix
self.eventStream = EventStream (amqpSettings, workflowRoot).getEventContext ()
self.amqpSettings = amqpSettings
self.workflowRoot = workflowRoot
self.workflowMonitorDatabase = WorkflowMonitorDatabase ()
logger.debug ("workflowmonitorcompilerplugin:amqpsettings: %s", amqpSettings)
def get_flow_events (request):
workdir = request.REQUEST ["workdir"]
workflowId = request.REQUEST ["workflowid"]
runId = request.REQUEST ["runid"]
dax = request.REQUEST ["dax"] if "dax" in request.REQUEST else None
if not dax:
dax = os.path.basename (workflowId)
logger.debug ("dax: %s", dax)
workflowName = os.path.basename (workflowId).replace (".dax", "")
process_username = ViewUtil.get_os_username ()
workdirPath = GraysonUtil.form_workdir_path (workdir, process_username, workflowName, runId)
user = ViewUtil.get_user (request)
workdirPath = ViewUtil.form_workflow_path (user, workdirPath)
logger.debug ("launching monitor: user: %s, workdir: %s, workflowId: %s, runId: %s, dax: %s",
user.username, workdirPath, workflowId, runId, dax)
workflowMonitorDatabase = WorkflowMonitorDatabase ()
WorkflowMonitor.ensureRunning (workflowRoot = settings.GRAYSONWEB_WORKFLOW_ROOT,
amqpSettings = settings.AMQP_SETTINGS,
eventBufferSize = settings.EVENT_BUFFER_SIZE)
workflowMonitorDatabase.subscribeToWorkflow (
settings.GRAYSONWEB_WORKFLOW_ROOT,
{
"username" : user.username,
"workflowId" : workflowId,
"workdir" : workdirPath,
"daxen" : dax.split (','),
"buffer" : 0
})
return ViewUtil.get_json_response ({ "status" : "ok" })
class WorkflowMonitorCompilerPlugin (object):
def __init__(self, username, graphPrefix, workflowRoot=".", amqpSettings=None):
self.username = username
self.graphPathPrefix = graphPrefix
self.eventStream = EventStream (amqpSettings, workflowRoot).getEventContext ()
self.amqpSettings = amqpSettings
self.workflowRoot = workflowRoot
self.workflowMonitorDatabase = WorkflowMonitorDatabase ()
logger.debug ("workflowmonitorcompilerplugin:amqpsettings: %s", amqpSettings)
def notifyShellEvent (self, line, outputWorkflowPath):
logger.info ("|| %s", line)
jobid = None
status = None
workdirMarker = "pegasus-status -l "
if workdirMarker in line:
''' this is a grid job '''
workDir = line [ line.rfind (workdirMarker) + len (workdirMarker) : ]
workDir = workDir.rstrip ()
logger.info ("starting grid monitor workflowId: %s, username: %s, workDir: %s", outputWorkflowPath, self.username, workDir)
idparts = workDir.split (os.sep)
if len(idparts) >= 7:
workflowName = idparts [len(idparts)-2]
workflowId = os.path.join (idparts [len(idparts)-6],
workflowName).replace ('.dax', '')
self.workflowMonitorDatabase.subscribeToWorkflow (
self.workflowRoot,
{
"username" : self.username,
"workflowId" : workflowId,
"workdir" : workDir,
"daxen" : [],
"buffer" : 0
})
elif "Executing JOB" in line:
jobid = StrUtil.between (line, "::", ":")
if " " in jobid:
jobid = StrUtil.before (jobid, " ")
status = "running"
logger.info ('--grayson:shell-compilerplugin: got executing %s %s', jobid, status)
elif "Returned with" in line:
jobid = StrUtil.between (line, " JOB ", "_")
exitcode = ""
try:
statustext = StrUtil.afterlast (line, "Returned with ").rstrip ()
status = int (statustext)
logger.info ('--grayson:shell-compilerplugin: got status %s %s %s', jobid, statustext, status)
except:
logger.error ("cant parse exit code")
traceback.print_exc ()
logger.info (' -- got returned with %s %s', jobid, status)
if jobid and status != None:
logger.info ("--grayson:shell-compilerplugin: notifying got returned with %s %s", jobid, status)
self.eventStream.sendJobStatusEvent (self.username, outputWorkflowPath, jobid, status)
def notifyWorkflowCreation (self, workflowGraph, outputWorkflowPath, topWorkflow=False):
if topWorkflow:
logger.info ("--grayson:shell-compilerplugin: notifying workflow creation: %s", workflowGraph)
graphPath = "%s.graphml" % workflowGraph
graphPath = os.path.join (self.graphPathPrefix, graphPath)
self.eventStream.sendWorkflowEvent (self.username, outputWorkflowPath, graphPath, outputWorkflowPath)
else:
logger.info ("outputWorkflowPath: %s graph: %s", outputWorkflowPath, workflowGraph)
self.eventStream.sendSubworkflowEvent (self.username, outputWorkflowPath, workflowGraph)