def request_index_old(request, request_id):
"""
Detail page on a specific request (i.e. DAGManJobId) in a way that resembles
the OPUS/NHPPS OSF blackboard:
Dataset, Owner, DAGManJobId, [Nodei JobState, Nodei ExitCode, ]
"""
# Load the template.
t = loader.get_template('monitor/request.html')
# Get the bb entries.
(dataset, user, requestId, stages) = blackboard.getOSFEntry(request_id)
# Render the template and exit.
c = Context({'dataset': dataset,
'user': user,
'requestId': requestId,
'stages': stages})
return(HttpResponse(t.render(c)))
def owlapi_jobs_get_dag(self, dagId):
"""
Return the list of all Blackboard entries associated to the given DAG id
`dagId`. The DAG id can be either a global Id or a local id. Global ids
are safer as they ensure that only jobs submitted from the same host as
the input DAG are returned.
The results are sorted by ascending local Job id.
It is important to realize that further checks in the calling code are
needed as (even when using global DAG ids) since Condor stored only the
ClusterId of the parent dagman job, there is a degeneracy in the ClassAd
as well as database content. In particular, different DAGs submitted on
different hosts sharing the same Blackboard database could end up having
the same ClusterId. While this case is solved by using global ids,
another situation is not: reinstalling Condor resets the job id counter
and therefore can cause different DAGs to have the same ClusterId and
the same submit host. In that case, eiter wipe the Blackboard database
after reinstalling Condor or filter the results of this API call by
time.
Usage
jobs_get_dag(dagId)
Return
[{GlobalJobId, DAGManJobId, Dataset, Owner, JobStartDate,
DAGNodeName, DAGParentNodeNames, ExitCode, JobState, JobDuration}]
"""
fields = ('GlobalJobId',
'DAGManJobId',
'Dataset',
'Owner',
'JobStartDate',
'DAGNodeName',
'DAGParentNodeNames',
'ExitCode',
'JobState',
'JobDuration')
res = blackboard.getOSFEntry(str(dagId))
if(not res or len(res) != 4):
return([])
return([dict([(f, getattr(e, f, None)) for f in fields]) \
for e in res[3]])
def owlapi_jobs_get_dag(self, dagId):
"""
Return the list of all Blackboard entries associated to the given DAG id
`dagId`. The DAG id can be either a global Id or a local id. Global ids
are safer as they ensure that only jobs submitted from the same host as
the input DAG are returned.
The results are sorted by ascending local Job id.
It is important to realize that further checks in the calling code are
needed as (even when using global DAG ids) since Condor stored only the
ClusterId of the parent dagman job, there is a degeneracy in the ClassAd
as well as database content. In particular, different DAGs submitted on
different hosts sharing the same Blackboard database could end up having
the same ClusterId. While this case is solved by using global ids,
another situation is not: reinstalling Condor resets the job id counter
and therefore can cause different DAGs to have the same ClusterId and
the same submit host. In that case, eiter wipe the Blackboard database
after reinstalling Condor or filter the results of this API call by
time.
Usage
jobs_get_dag(dagId)
Return
[{GlobalJobId, DAGManJobId, Dataset, Owner, JobStartDate,
DAGNodeName, DAGParentNodeNames, ExitCode, JobState, JobDuration}]
"""
fields = ('GlobalJobId', 'DAGManJobId', 'Dataset', 'Owner',
'JobStartDate', 'DAGNodeName', 'DAGParentNodeNames',
'ExitCode', 'JobState', 'JobDuration')
res = blackboard.getOSFEntry(str(dagId))
if (not res or len(res) != 4):
return ([])
return([dict([(f, getattr(e, f, None)) for f in fields]) \
for e in res[3]])
def request_index(request, request_id):
"""
Detail page on a specific request (i.e. DAGManJobId@<submit host>) by
creating an SVG representation of the executing DAG.
"""
# Load the template.
t = loader.get_template('monitor/request_svg.html')
# Get the bb entries.
dagManJobId, host = request_id.split('@', 1)
(dataset, user, requestId, stages) = blackboard.getOSFEntry(dagManJobId)
# Filter out all the spurious ones.
# FIXME: Maybe do this a bit better, huh?
hh = host + '#'
stages = [s for s in stages if s.GlobalJobId.startswith(hh)]
# Create the DOT graph.
dot = pydot.Dot(type='digraph')
# Keep track of node names vs node ids.
nodes = {} # {node name: [node id1, node id2, ...]}
for stage in stages:
nodeId = '%d.%d' % (stage.ClusterId, stage.ProcId)
# Remember that if a condor job is on hold, it will have an ExitCode =
# None and a JobState = Exited.
if(stage.JobState != 'Exited'):
nodeLabel = '%s - %s' % (stage.DAGNodeName,
stage.JobState)
elif(stage.ExitCode == None):
# The corresponding condor job was put on hold.
nodeLabel = '%s - %s' % (stage.DAGNodeName,
'On Hold')
else:
nodeLabel = '%s - %s (%d)' % (stage.DAGNodeName,
stage.JobState,
stage.ExitCode)
if(stage.JobState == 'Exited' and stage.ExitCode != 0):
dot.add_node(pydot.Node(nodeId, shape='ellipse', label=nodeLabel,
style="filled", fillcolor="red"))
else:
dot.add_node(pydot.Node(nodeId, shape='ellipse', label=nodeLabel))
if(nodes.has_key(stage.DAGNodeName)):
nodes[stage.DAGNodeName].append(nodeId)
else:
nodes[stage.DAGNodeName] = [nodeId, ]
# Now add the edges of the graph.
for stage in stages:
if(stage.DAGParentNodeNames):
nodeId = '%d.%d' % (stage.ClusterId, stage.ProcId)
parentNames = stage.DAGParentNodeNames.split(',')
for parentName in parentNames:
if(not nodes.has_key(parentName)):
continue
# We have three cases:
# 1. node.Instances == parent.Instances: 1 to 1
# 2. node.Instances > parent.Instances: scatter
# 3. node.Instances < parent.Instances: gather
# We can treat 2. and 3. the same way. !. is different.
if(len(nodes[stage.DAGNodeName]) == len(nodes[parentName])):
# Case 1.
#
# FIXME: here we assume that ProcIds are the same for parent and child.
#
pClusterId = int(nodes[parentName][0].split('.')[0])
pId = '%d.%d' % (pClusterId, stage.ProcId)
dot.add_edge(pydot.Edge(pId, nodeId))
else:
# Cases 2. and 3.
for parentId in nodes[parentName]:
dot.add_edge(pydot.Edge(parentId, nodeId))
# Create the SVG graph.
# This does a system call, which I would rather avoid.
# rawSvg = dot.create(format='svg')
# This works standalone but not in the web environment... dunno why :-(
# cLength = ctypes.c_int(1)
# cSvg = ctypes.pointer(ctypes.create_string_buffer(1))
# dotString = dot.to_string()
#
# gvc = libgvc.gvContext()
# g = libgraph.agmemread(dotString)
#
# libgvc.gvLayout(gvc, g, "dot")
# libgvc.gvRenderData(gvc, g, "svg",
# ctypes.byref(cSvg), ctypes.byref(cLength))
#
# libgvc.gvFreeLayout(gvc, g)
# libgraph.agclose(g)
# libgvc.gvFreeContext(gvc)
#
# rawSvg = ctypes.string_at(cSvg)
# This works, as long as I convert the unicode to string.
dotString = dot.to_string()
rawSvg = grapher.create(str(dotString), 'dot', 'svg')
# Remove the header.
svg = rawSvg.split('\n')[6:]
# Render the template and exit.
c = Context({'dataset': dataset,
'user': user,
'requestId': requestId,
'svg': '\n'.join(svg)})
return(HttpResponse(t.render(c), mimetype='application/xhtml+xml'))