def test_containerDrop(self):
dropSpecList = [{"oid":"A", "type":"plain", "storage":"memory"},
{"oid":"B", "type":"container", "children":["A"]}]
a = graph_loader.createGraphFromDropSpecList(dropSpecList)[0]
self.assertIsInstance(a, InMemoryDROP)
self.assertEquals("A", a.oid)
self.assertEquals("A", a.uid)
self.assertIsNotNone(a.parent)
b = a.parent
self.assertIsInstance(b, ContainerDROP)
self.assertEquals("B", b.oid)
self.assertEquals("B", b.uid)
# A directory container
dropSpecList = [{"oid":"A", "type":"plain", "storage":"file", "dirname":"."},
{"oid":"B", "type":"container", "container":"dfms.drop.DirectoryContainer", "children":["A"], "dirname":"."}]
a = graph_loader.createGraphFromDropSpecList(dropSpecList)[0]
b = a.parent
self.assertIsInstance(b, DirectoryContainer)
def test_consumer(self):
dropSpecList = [{"oid":"A", "type":"plain", "storage":"memory", "consumers":["B"]},
{"oid":"B", "type":"app", "app":"test.test_graph_loader.DummyApp"}]
a = graph_loader.createGraphFromDropSpecList(dropSpecList)[0]
self.assertIsInstance(a, InMemoryDROP)
self.assertEquals("A", a.oid)
self.assertEquals("A", a.uid)
self.assertEquals(1, len(a.consumers))
b = a.consumers[0]
self.assertIsInstance(b, DummyApp)
self.assertEquals("B", b.oid)
self.assertEquals("B", b.uid)
self.assertEquals(a, b.inputs[0])
def test_containerDrop(self):
dropSpecList = [{
"oid": "A",
"type": "plain",
"storage": "memory"
}, {
"oid": "B",
"type": "container",
"children": ["A"]
}]
a = graph_loader.createGraphFromDropSpecList(dropSpecList)[0]
self.assertIsInstance(a, InMemoryDROP)
self.assertEqual("A", a.oid)
self.assertEqual("A", a.uid)
self.assertIsNotNone(a.parent)
b = a.parent
self.assertIsInstance(b, ContainerDROP)
self.assertEqual("B", b.oid)
self.assertEqual("B", b.uid)
# A directory container
dropSpecList = [{
"oid": "A",
"type": "plain",
"storage": "file",
"dirname": "."
}, {
"oid": "B",
"type": "container",
"container": "dfms.drop.DirectoryContainer",
"children": ["A"],
"dirname": "."
}]
a = graph_loader.createGraphFromDropSpecList(dropSpecList)[0]
b = a.parent
self.assertIsInstance(b, DirectoryContainer)
def test_consumer(self):
dropSpecList = [{
"oid": "A",
"type": "plain",
"storage": "memory",
"consumers": ["B"]
}, {
"oid": "B",
"type": "app",
"app": "test.test_graph_loader.DummyApp"
}]
a = graph_loader.createGraphFromDropSpecList(dropSpecList)[0]
self.assertIsInstance(a, InMemoryDROP)
self.assertEqual("A", a.oid)
self.assertEqual("A", a.uid)
self.assertEqual(1, len(a.consumers))
b = a.consumers[0]
self.assertIsInstance(b, DummyApp)
self.assertEqual("B", b.oid)
self.assertEqual("B", b.uid)
self.assertEqual(a, b.inputs[0])
def deploy(self, completedDrops=[], foreach=None):
"""
Creates the DROPs represented by all the graph specs contained in
this session, effectively deploying them.
When this method has finished executing a Pyro Daemon will also be
up and running, servicing requests to access to all the DROPs
belonging to this session
"""
status = self.status
if status != SessionStates.BUILDING:
raise InvalidSessionState("Can't deploy this session in its current status: %d" % (status))
self.status = SessionStates.DEPLOYING
# Create the real DROPs from the graph specs
logger.info("Creating DROPs for session %s", self._sessionId)
self._roots = graph_loader.createGraphFromDropSpecList(self._graph.values())
logger.info("%d drops successfully created", len(self._graph))
for drop,_ in droputils.breadFirstTraverse(self._roots):
# Register them
self._drops[drop.uid] = drop
# Register them with the error handler
if self._error_status_listener:
drop.subscribe(self._error_status_listener, eventType='status')
logger.info("Stored all drops, proceeding with further customization")
# Start the luigi task that will make sure the graph is executed
# If we're not using luigi we still
if self._enable_luigi:
logger.debug("Starting Luigi FinishGraphExecution task for session %s", self._sessionId)
task = luigi_int.FinishGraphExecution(self._sessionId, self._roots)
sch = scheduler.CentralPlannerScheduler()
w = worker.Worker(scheduler=sch)
w.add(task)
workerT = threading.Thread(None, self._run, args=[w])
workerT.daemon = True
workerT.start()
else:
leaves = droputils.getLeafNodes(self._roots)
logger.info("Adding completion listener to leaf drops")
listener = LeavesCompletionListener(leaves, self)
for leaf in leaves:
if isinstance(leaf, AppDROP):
leaf.subscribe(listener, 'producerFinished')
else:
leaf.subscribe(listener, 'dropCompleted')
logger.info("Listener added to leaf drops")
# We move to COMPLETED the DROPs that we were requested to
# InputFiredAppDROP are here considered as having to be executed and
# not directly moved to COMPLETED.
#
# This is done in a separate iteration at the very end because all drops
# to make sure all event listeners are ready
self.trigger_drops(completedDrops)
# Foreach
if foreach:
logger.info("Invoking 'foreach' on each drop")
for drop,_ in droputils.breadFirstTraverse(self._roots):
foreach(drop)
logger.info("'foreach' invoked for each drop")
# Append proxies
logger.info("Creating %d drop proxies", len(self._proxyinfo))
for nm, host, port, local_uid, relname, remote_uid in self._proxyinfo:
proxy = DropProxy(nm, host, port, self._sessionId, remote_uid)
method = getattr(self._drops[local_uid], relname)
method(proxy, False)
self.status = SessionStates.RUNNING
logger.info("Session %s is now RUNNING", self._sessionId)
def _test_graphFromFile(self, f, socketListeners=1):
f = pkg_resources.resource_stream("test", "graphs/%s" % (f)) # @UndefinedVariable
self._test_graph(graph_loader.createGraphFromDropSpecList(json.load(f)), socketListeners)
def deploy(self, completedDrops=[]):
"""
Creates the DROPs represented by all the graph specs contained in
this session, effectively deploying them.
When this method has finished executing a Pyro Daemon will also be
up and running, servicing requests to access to all the DROPs
belonging to this session
"""
status = self.status
if status != SessionStates.BUILDING:
raise Exception("Can't deploy this session in its current status: %d" % (status))
self.status = SessionStates.DEPLOYING
# Create the Pyro daemon that will serve the DROP proxies and start it
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Starting Pyro4 Daemon for session %s" % (self._sessionId))
self._daemon = Pyro4.Daemon(host=self._host)
self._daemonT = threading.Thread(target = lambda: self._daemon.requestLoop(), name="Session %s Pyro Daemon" % (self._sessionId))
self._daemonT.daemon = True
self._daemonT.start()
# Create the real DROPs from the graph specs
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Creating DROPs for session %s" % (self._sessionId))
self._roots = graph_loader.createGraphFromDropSpecList(self._graph.values())
# Register them
droputils.breadFirstTraverse(self._roots, self._registerDrop)
# Register them with the error handler
# TODO: We should probably merge all these breadFirstTraverse calls into
# a single one to avoid so much iteration through the drops
if self._error_status_listener:
def register_error_status_listener(drop):
drop.subscribe(self._error_status_listener, eventType='status')
droputils.breadFirstTraverse(self._roots, register_error_status_listener)
# We move to COMPLETED the DROPs that we were requested to
# InputFiredAppDROP are here considered as having to be executed and
# not directly moved to COMPLETED.
# TODO: We should possibly unify this initial triggering into a more
# solid concept that encompasses these two and other types of DROPs
def triggerDrop(drop):
if drop.uid in completedDrops:
if isinstance(drop, InputFiredAppDROP):
t = threading.Thread(target=lambda:drop.execute())
t.daemon = True
t.start()
else:
drop.setCompleted()
droputils.breadFirstTraverse(self._roots, triggerDrop)
# Start the luigi task that will make sure the graph is executed
# If we're not using luigi we still
if self._enable_luigi:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Starting Luigi FinishGraphExecution task for session %s" % (self._sessionId))
task = luigi_int.FinishGraphExecution(self._sessionId, self._roots)
sch = scheduler.CentralPlannerScheduler()
w = worker.Worker(scheduler=sch)
w.add(task)
workerT = threading.Thread(None, self._run, args=[w])
workerT.daemon = True
workerT.start()
else:
leaves = droputils.getLeafNodes(self._roots)
logger.debug("Adding completion listener to leaf drops %r", leaves)
listener = LeavesCompletionListener(leaves, self)
for leaf in leaves:
leaf.subscribe(listener, 'dropCompleted')
leaf.subscribe(listener, 'producerFinished')
self.status = SessionStates.RUNNING
if logger.isEnabledFor(logging.INFO):
logger.info("Session %s is now RUNNING" % (self._sessionId))
def _test_graphFromFile(self, f, socketListeners=1):
with pkg_resources.resource_stream("test", "graphs/%s" %
(f)) as f: # @UndefinedVariable
self._test_graph(
graph_loader.createGraphFromDropSpecList(
json.load(codecs.getreader('utf-8')(f))), socketListeners)
def test_singleMemoryDrop(self):
dropSpecList = [{"oid": "A", "type": "plain", "storage": "memory"}]
a = graph_loader.createGraphFromDropSpecList(dropSpecList)[0]
self.assertIsInstance(a, InMemoryDROP)
self.assertEqual("A", a.oid)
self.assertEqual("A", a.uid)
def test_singleMemoryDrop(self):
dropSpecList = [{"oid":"A", "type":"plain", "storage":"memory"}]
a = graph_loader.createGraphFromDropSpecList(dropSpecList)[0]
self.assertIsInstance(a, InMemoryDROP)
self.assertEquals("A", a.oid)
self.assertEquals("A", a.uid)