def testGetEndNodes(self):
"""
Checks that the getLeafNodes works correctly
"""
a, _, _, _, _, f, _, _, _, j = self._createGraph()
endNodes = droputils.getLeafNodes(a)
self.assertSetEqual(set([j, f]), set(endNodes))
def testGetEndNodes(self):
"""
Checks that the getLeafNodes works correctly
"""
a, _, _, _, _, f, _, _, _, j = self._createGraph()
endNodes = droputils.getLeafNodes(a)
self.assertSetEqual(set([j, f]), set(endNodes))
pass
def __init__(self, *args, **kwargs):
super(FinishGraphExecution, self).__init__(*args, **kwargs)
self._req = None
if isinstance(self.pgCreator, basestring):
parts = self.pgCreator.split('.')
module = importlib.import_module('.'.join(parts[:-1]))
pgCreatorFn = getattr(module, parts[-1])
roots = pgCreatorFn()
else:
roots = self.pgCreator
self._roots = droputils.listify(roots)
self._leaves = droputils.getLeafNodes(self._roots)
self._completed = False
def __init__(self, *args, **kwargs):
super(FinishGraphExecution, self).__init__(*args, **kwargs)
self._req = None
if isinstance(self.pgCreator, six.string_types):
parts = self.pgCreator.split('.')
module = importlib.import_module('.'.join(parts[:-1]))
pgCreatorFn = getattr(module, parts[-1])
roots = pgCreatorFn()
else:
roots = self.pgCreator
self._roots = droputils.listify(roots)
self._leaves = droputils.getLeafNodes(self._roots)
self._completed = False
def deploy(self, completedDrops=[], foreach=None):
"""
Creates the DROPs represented by all the graph specs contained in
this session, effectively deploying them.
When this method has finished executing a Pyro Daemon will also be
up and running, servicing requests to access to all the DROPs
belonging to this session
"""
status = self.status
if status != SessionStates.BUILDING:
raise InvalidSessionState("Can't deploy this session in its current status: %d" % (status))
self.status = SessionStates.DEPLOYING
# Create the real DROPs from the graph specs
logger.info("Creating DROPs for session %s", self._sessionId)
self._roots = graph_loader.createGraphFromDropSpecList(self._graph.values())
logger.info("%d drops successfully created", len(self._graph))
for drop,_ in droputils.breadFirstTraverse(self._roots):
# Register them
self._drops[drop.uid] = drop
# Register them with the error handler
if self._error_status_listener:
drop.subscribe(self._error_status_listener, eventType='status')
logger.info("Stored all drops, proceeding with further customization")
# Start the luigi task that will make sure the graph is executed
# If we're not using luigi we still
if self._enable_luigi:
logger.debug("Starting Luigi FinishGraphExecution task for session %s", self._sessionId)
task = luigi_int.FinishGraphExecution(self._sessionId, self._roots)
sch = scheduler.CentralPlannerScheduler()
w = worker.Worker(scheduler=sch)
w.add(task)
workerT = threading.Thread(None, self._run, args=[w])
workerT.daemon = True
workerT.start()
else:
leaves = droputils.getLeafNodes(self._roots)
logger.info("Adding completion listener to leaf drops")
listener = LeavesCompletionListener(leaves, self)
for leaf in leaves:
if isinstance(leaf, AppDROP):
leaf.subscribe(listener, 'producerFinished')
else:
leaf.subscribe(listener, 'dropCompleted')
logger.info("Listener added to leaf drops")
# We move to COMPLETED the DROPs that we were requested to
# InputFiredAppDROP are here considered as having to be executed and
# not directly moved to COMPLETED.
#
# This is done in a separate iteration at the very end because all drops
# to make sure all event listeners are ready
self.trigger_drops(completedDrops)
# Foreach
if foreach:
logger.info("Invoking 'foreach' on each drop")
for drop,_ in droputils.breadFirstTraverse(self._roots):
foreach(drop)
logger.info("'foreach' invoked for each drop")
# Append proxies
logger.info("Creating %d drop proxies", len(self._proxyinfo))
for nm, host, port, local_uid, relname, remote_uid in self._proxyinfo:
proxy = DropProxy(nm, host, port, self._sessionId, remote_uid)
method = getattr(self._drops[local_uid], relname)
method(proxy, False)
self.status = SessionStates.RUNNING
logger.info("Session %s is now RUNNING", self._sessionId)
def deploy(self, completedDrops=[]):
"""
Creates the DROPs represented by all the graph specs contained in
this session, effectively deploying them.
When this method has finished executing a Pyro Daemon will also be
up and running, servicing requests to access to all the DROPs
belonging to this session
"""
status = self.status
if status != SessionStates.BUILDING:
raise Exception("Can't deploy this session in its current status: %d" % (status))
self.status = SessionStates.DEPLOYING
# Create the Pyro daemon that will serve the DROP proxies and start it
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Starting Pyro4 Daemon for session %s" % (self._sessionId))
self._daemon = Pyro4.Daemon(host=self._host)
self._daemonT = threading.Thread(target = lambda: self._daemon.requestLoop(), name="Session %s Pyro Daemon" % (self._sessionId))
self._daemonT.daemon = True
self._daemonT.start()
# Create the real DROPs from the graph specs
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Creating DROPs for session %s" % (self._sessionId))
self._roots = graph_loader.createGraphFromDropSpecList(self._graph.values())
# Register them
droputils.breadFirstTraverse(self._roots, self._registerDrop)
# Register them with the error handler
# TODO: We should probably merge all these breadFirstTraverse calls into
# a single one to avoid so much iteration through the drops
if self._error_status_listener:
def register_error_status_listener(drop):
drop.subscribe(self._error_status_listener, eventType='status')
droputils.breadFirstTraverse(self._roots, register_error_status_listener)
# We move to COMPLETED the DROPs that we were requested to
# InputFiredAppDROP are here considered as having to be executed and
# not directly moved to COMPLETED.
# TODO: We should possibly unify this initial triggering into a more
# solid concept that encompasses these two and other types of DROPs
def triggerDrop(drop):
if drop.uid in completedDrops:
if isinstance(drop, InputFiredAppDROP):
t = threading.Thread(target=lambda:drop.execute())
t.daemon = True
t.start()
else:
drop.setCompleted()
droputils.breadFirstTraverse(self._roots, triggerDrop)
# Start the luigi task that will make sure the graph is executed
# If we're not using luigi we still
if self._enable_luigi:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Starting Luigi FinishGraphExecution task for session %s" % (self._sessionId))
task = luigi_int.FinishGraphExecution(self._sessionId, self._roots)
sch = scheduler.CentralPlannerScheduler()
w = worker.Worker(scheduler=sch)
w.add(task)
workerT = threading.Thread(None, self._run, args=[w])
workerT.daemon = True
workerT.start()
else:
leaves = droputils.getLeafNodes(self._roots)
logger.debug("Adding completion listener to leaf drops %r", leaves)
listener = LeavesCompletionListener(leaves, self)
for leaf in leaves:
leaf.subscribe(listener, 'dropCompleted')
leaf.subscribe(listener, 'producerFinished')
self.status = SessionStates.RUNNING
if logger.isEnabledFor(logging.INFO):
logger.info("Session %s is now RUNNING" % (self._sessionId))