def getGraphStatus(self):
if self.status not in (SessionStates.RUNNING, SessionStates.FINISHED):
raise InvalidSessionState("The session is currently not running, cannot get graph status")
# We shouldn't traverse the full graph because there might be nodes
# attached to our DROPs that are actually part of other DM (and have been
# wired together by the DIM after deploying each individual graph on
# each of the DMs).
# We recognize such nodes because they are actually not an instance of
# AbstractDROP (they are DropProxy instances).
#
# The same trick is used in luigi_int.RunDROPTask.requires
statusDict = collections.defaultdict(dict)
for drop, downStreamDrops in droputils.breadFirstTraverse(self._roots):
downStreamDrops[:] = [dsDrop for dsDrop in downStreamDrops if isinstance(dsDrop, AbstractDROP)]
if isinstance(drop, AppDROP):
statusDict[drop.oid]['execStatus'] = drop.execStatus
statusDict[drop.oid]['status'] = drop.status
return statusDict
def linkGraphParts(self, lhOID, rhOID, linkType, force=False):
"""
Links together two DROP specifications (i.e., those pointed by
`lhOID` and `rhOID`) using `linkType`. The DROP specifications
must both already be part of one of the graph specs contained in this
session; otherwise an exception will be raised.
"""
if self.status != SessionStates.BUILDING:
raise InvalidSessionState("Can't link DROPs anymore since this session isn't in the BUILDING state")
# Look for the two DROPs in all our graph parts and reporting
# missing DROPs
lhDropSpec = self._graph.get(lhOID, None)
rhDropSpec = self._graph.get(rhOID, None)
missingOids = []
if lhDropSpec is None: missingOids.append(lhOID)
if rhDropSpec is None: missingOids.append(rhOID)
if missingOids:
oids = 'OID' if len(missingOids) == 1 else 'OIDs'
raise InvalidGraphException('No DROP found for %s %r' % (oids, missingOids))
graph_loader.addLink(linkType, lhDropSpec, rhOID, force=force)
def addGraphSpec(self, graphSpec):
"""
Adds the graph specification given in `graphSpec` to the
graph specification currently held by this session. A graphSpec is a
list of dictionaries, each of which contains the information of one
DROP. Each DROP specification is checked to see it contains
all the necessary details to construct a proper DROP. If one
DROP specification is found to be inconsistent the whole operation
fill wail.
Adding graph specs to the session is only allowed while the session is
in the PRISTINE or BUILDING status; otherwise an exception will be
raised.
If the `graphSpec` being added contains DROPs that have already
been added to the session an exception will be raised. DROPs are
uniquely identified by their OID at this point.
"""
status = self.status
if status not in (SessionStates.PRISTINE, SessionStates.BUILDING):
raise InvalidSessionState("Can't add graphs to this session since it isn't in the PRISTINE or BUILDING status: %d" % (status))
self.status = SessionStates.BUILDING
# This will check the consistency of each dropSpec
graphSpecDict = graph_loader.loadDropSpecs(graphSpec)
# Check for duplicates
duplicates = set(graphSpecDict) & set(self._graph)
if duplicates:
raise InvalidGraphException('Trying to add drops with OIDs that already exist: %r' % (duplicates,))
self._graph.update(graphSpecDict)
logger.debug("Added a graph definition with %d DROPs", len(graphSpecDict))
def getGraphStatus(self, session_id):
self.check_session_id(session_id)
if self._session_status_reqno < run_step:
raise InvalidSessionState("Requesting status of graph that is not running yet")
while True:
l = self._status_file.readline()
if not l:
self._status_file.close()
self._status = SessionStates.FINISHED
return self._last_graph_status
content = json.loads(l)
this_session_id = content['ssid']
if this_session_id != session_id:
continue
graph_status = content['gs']
self._last_graph_status = graph_status
logger.info("Serving graph status")
return graph_status
def deploy(self, completedDrops=[], foreach=None):
"""
Creates the DROPs represented by all the graph specs contained in
this session, effectively deploying them.
When this method has finished executing a Pyro Daemon will also be
up and running, servicing requests to access to all the DROPs
belonging to this session
"""
status = self.status
if status != SessionStates.BUILDING:
raise InvalidSessionState("Can't deploy this session in its current status: %d" % (status))
self.status = SessionStates.DEPLOYING
# Create the real DROPs from the graph specs
logger.info("Creating DROPs for session %s", self._sessionId)
self._roots = graph_loader.createGraphFromDropSpecList(self._graph.values())
logger.info("%d drops successfully created", len(self._graph))
for drop,_ in droputils.breadFirstTraverse(self._roots):
# Register them
self._drops[drop.uid] = drop
# Register them with the error handler
if self._error_status_listener:
drop.subscribe(self._error_status_listener, eventType='status')
logger.info("Stored all drops, proceeding with further customization")
# Start the luigi task that will make sure the graph is executed
# If we're not using luigi we still
if self._enable_luigi:
logger.debug("Starting Luigi FinishGraphExecution task for session %s", self._sessionId)
task = luigi_int.FinishGraphExecution(self._sessionId, self._roots)
sch = scheduler.CentralPlannerScheduler()
w = worker.Worker(scheduler=sch)
w.add(task)
workerT = threading.Thread(None, self._run, args=[w])
workerT.daemon = True
workerT.start()
else:
leaves = droputils.getLeafNodes(self._roots)
logger.info("Adding completion listener to leaf drops")
listener = LeavesCompletionListener(leaves, self)
for leaf in leaves:
if isinstance(leaf, AppDROP):
leaf.subscribe(listener, 'producerFinished')
else:
leaf.subscribe(listener, 'dropCompleted')
logger.info("Listener added to leaf drops")
# We move to COMPLETED the DROPs that we were requested to
# InputFiredAppDROP are here considered as having to be executed and
# not directly moved to COMPLETED.
#
# This is done in a separate iteration at the very end because all drops
# to make sure all event listeners are ready
self.trigger_drops(completedDrops)
# Foreach
if foreach:
logger.info("Invoking 'foreach' on each drop")
for drop,_ in droputils.breadFirstTraverse(self._roots):
foreach(drop)
logger.info("'foreach' invoked for each drop")
# Append proxies
logger.info("Creating %d drop proxies", len(self._proxyinfo))
for nm, host, port, local_uid, relname, remote_uid in self._proxyinfo:
proxy = DropProxy(nm, host, port, self._sessionId, remote_uid)
method = getattr(self._drops[local_uid], relname)
method(proxy, False)
self.status = SessionStates.RUNNING
logger.info("Session %s is now RUNNING", self._sessionId)