def test_many_relationships(self):
"""
A test in which a drop is related to many other drops that live in a
separate DM (and thus requires many Pyro connections).
Drop A is accessed by many applications (B1, B2, .., BN), which should
not exhaust resources on DM #1 (in particular, the pyro thread pool).
We collapse all into C so we can monitor only its status to know that
the execution is over.
DM #1 DM #2
======= ====================
| | | |--> B1 --| |
| | | |--> B2 --| |
| A --|----|-|--> B3 --|--> C |
| | | |.........| |
| | | |--> BN --| |
======= ====================
"""
dm1 = NodeManager(useDLM=False)
dm2 = NodeManager(useDLM=False)
sessionId = 's1'
N = 100
g1 = [{"oid":"A", "type":"plain", "storage": "memory"}]
g2 = [{"oid":"C", "type":"plain", "storage": "memory"}]
for i in xrange(N):
b_oid = "B%d" % (i,)
# SleepAndCopyApp effectively opens the input drop
g2.append({"oid":b_oid, "type":"app", "app":"test.graphsRepository.SleepAndCopyApp", "outputs":["C"], "sleepTime": 0})
uris1 = dm1.quickDeploy(sessionId, g1)
uris2 = dm2.quickDeploy(sessionId, g2)
self.assertEquals(1, len(uris1))
self.assertEquals(1+N, len(uris2))
# We externally wire the Proxy objects to establish the inter-DM
# relationships. Make sure we release the proxies
with Pyro4.Proxy(uris1['A']) as a:
for i in xrange(N):
with Pyro4.Proxy(uris2['B%d' % (i,)]) as b:
b.addInput(a, False)
a.addConsumer(b, False)
# Run! The sole fact that this doesn't throw exceptions is already
# a good proof that everything is working as expected
c = Pyro4.Proxy(uris2['C'])
with droputils.EvtConsumerProxyCtx(self, c, 5):
a.write('a')
a.setCompleted()
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)
def test_runGraphOneDOPerDOM(self):
"""
A test that creates three DROPs in two different DMs, wire two of
them together externally (i.e., using their proxies), and runs the graph.
For this the graphs that are fed into the DMs must *not* express the
inter-DM relationships. The graph looks like:
DM #1 DM #2
======= =============
| A --|----|-> B --> C |
======= =============
"""
dm1 = NodeManager(useDLM=False)
dm2 = NodeManager(useDLM=False)
sessionId = 's1'
g1 = [{"oid":"A", "type":"plain", "storage": "memory"}]
g2 = [{"oid":"B", "type":"app", "app":"dfms.apps.crc.CRCApp"},
{"oid":"C", "type":"plain", "storage": "memory", "producers":["B"]}]
uris1 = dm1.quickDeploy(sessionId, g1)
uris2 = dm2.quickDeploy(sessionId, g2)
self.assertEquals(1, len(uris1))
self.assertEquals(2, len(uris2))
# We externally wire the Proxy objects now
a = Pyro4.Proxy(uris1['A'])
b = Pyro4.Proxy(uris2['B'])
c = Pyro4.Proxy(uris2['C'])
a.addConsumer(b)
# Run! We wait until c is completed
with droputils.EvtConsumerProxyCtx(self, c, 1):
a.write('a')
a.setCompleted()
for drop in a, b, c:
self.assertEquals(DROPStates.COMPLETED, drop.status)
self.assertEquals(a.checksum, int(droputils.allDropContents(c)))
for dropProxy in a,b,c:
dropProxy._pyroRelease()
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)
def test_runWithFourDMs(self):
"""
A test that creates several DROPs in two different DMs and runs
the graph. The graph looks like this
DM #2
+--------------------------+
| |--> C --| |
+---|--> B --|--> D --|--> F --|--|
| | |--> E --| | |
DM #1 | +--------------------------+ | DM #4
+-----+ | | +---------------------+
| | | |--|--> L --| |
| A --|--+ | |--> N --> O |
| | | |--|--> M --| |
+-----+ | DM #3 | +---------------------+
| +--------------------------+ |
| | |--> H --| | |
+---|--> G --|--> I --|--> K --|--|
| |--> J --| |
+--------------------------+
B, F, G, K and N are AppDOs; the rest are plain in-memory DROPs
"""
dm1 = NodeManager(useDLM=False)
dm2 = NodeManager(useDLM=False)
dm3 = NodeManager(useDLM=False)
dm4 = NodeManager(useDLM=False)
sessionId = 's1'
g1 = [memory('A', expectedSize=1)]
g2 = [sleepAndCopy('B', outputs=['C','D','E'], sleepTime=0),
memory('C'),
memory('D'),
memory('E'),
sleepAndCopy('F', inputs=['C','D','E'], sleepTime=0)]
g3 = [sleepAndCopy('G', outputs=['H','I','J'], sleepTime=0),
memory('H'),
memory('I'),
memory('J'),
sleepAndCopy('K', inputs=['H','I','J'], sleepTime=0)]
g4 = [memory('L'),
memory('M'),
sleepAndCopy('N', inputs=['L','M'], outputs=['O'], sleepTime=0),
memory('O')]
uris1 = dm1.quickDeploy(sessionId, g1)
uris2 = dm2.quickDeploy(sessionId, g2)
uris3 = dm3.quickDeploy(sessionId, g3)
uris4 = dm4.quickDeploy(sessionId, g4)
self.assertEquals(1, len(uris1))
self.assertEquals(5, len(uris2))
self.assertEquals(5, len(uris3))
self.assertEquals(4, len(uris4))
allUris = {}
allUris.update(uris1)
allUris.update(uris2)
allUris.update(uris3)
allUris.update(uris4)
# We externally wire the Proxy objects to establish the inter-DM
# relationships. Intra-DM relationships are already established
proxies = {}
for uid,uri in allUris.viewitems():
proxies[uid] = Pyro4.Proxy(uri)
a = proxies['A']
b = proxies['B']
f = proxies['F']
g = proxies['G']
k = proxies['K']
l = proxies['L']
m = proxies['M']
o = proxies['O']
a.addConsumer(b)
a.addConsumer(g)
f.addOutput(l)
k.addOutput(m)
# Run! This should trigger the full execution of the graph
with droputils.EvtConsumerProxyCtx(self, o, 1):
a.write('a')
for dropProxy in proxies.viewvalues():
self.assertEquals(DROPStates.COMPLETED, dropProxy.status, "Status of '%s' is not COMPLETED: %d" % (dropProxy.uid, dropProxy.status))
dropProxy._pyroRelease()
for dm in [dm1, dm2, dm3, dm4]:
dm.destroySession(sessionId)
def test_runGraphSeveralDropsPerDM(self):
"""
A test that creates several DROPs in two different DMs and runs
the graph. The graph looks like this
DM #1 DM #2
=================== ================
| A --> C --> D --|----|-| |
| | | |--> E --> F |
| B --------------|----|-| |
=================== ================
:see: `self.test_runGraphSingleDOPerDOM`
"""
dm1 = NodeManager(useDLM=False)
dm2 = NodeManager(useDLM=False)
sessionId = 's1'
g1 = [{"oid":"A", "type":"plain", "storage": "memory", "consumers":["C"]},
{"oid":"B", "type":"plain", "storage": "memory"},
{"oid":"C", "type":"app", "app":"dfms.apps.crc.CRCApp"},
{"oid":"D", "type":"plain", "storage": "memory", "producers": ["C"]}]
g2 = [{"oid":"E", "type":"app", "app":"test.test_drop.SumupContainerChecksum"},
{"oid":"F", "type":"plain", "storage": "memory", "producers":["E"]}]
uris1 = dm1.quickDeploy(sessionId, g1)
uris2 = dm2.quickDeploy(sessionId, g2)
self.assertEquals(4, len(uris1))
self.assertEquals(2, len(uris2))
# We externally wire the Proxy objects to establish the inter-DM
# relationships
a = Pyro4.Proxy(uris1['A'])
b = Pyro4.Proxy(uris1['B'])
c = Pyro4.Proxy(uris1['C'])
d = Pyro4.Proxy(uris1['D'])
e = Pyro4.Proxy(uris2['E'])
f = Pyro4.Proxy(uris2['F'])
for drop,uid in [(a,'A'),(b,'B'),(c,'C'),(d,'D'),(e,'E'),(f,'F')]:
self.assertEquals(uid, drop.uid, "Proxy is not the DROP we think should be (assumed: %s/ actual: %s)" % (uid, drop.uid))
e.addInput(d)
e.addInput(b)
# Run! The sole fact that this doesn't throw exceptions is already
# a good proof that everything is working as expected
with droputils.EvtConsumerProxyCtx(self, f, 5):
a.write('a')
a.setCompleted()
b.write('a')
b.setCompleted()
for drop in a,b,c,d,e,f:
self.assertEquals(DROPStates.COMPLETED, drop.status, "DROP %s is not COMPLETED" % (drop.uid))
self.assertEquals(a.checksum, int(droputils.allDropContents(d)))
self.assertEquals(b.checksum + d.checksum, int(droputils.allDropContents(f)))
for dropProxy in a,b,c,d,e,f:
dropProxy._pyroRelease()
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)