def test_app_multiple_outputs(self):
"""
A small method that tests that the AppDROPs writing to two
different DROPs outputs works
The graph constructed by this example looks as follow:
|--> E
A --> B --> C --> D --|
|--> F
Here B and D are an AppDROPs, with D writing to two DROPs
outputs (E and F) and reading from C. C, in turn, is written by B, which
in turns reads the data from A
"""
# This is used as "B"
class NumberWriterApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in range(howMany):
output.write(str(i) + " ")
# This is used as "D"
class OddAndEvenContainerApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + six.b(" "))
# Create DROPs
a = InMemoryDROP('oid:A', 'uid:A')
b = NumberWriterApp('oid:B', 'uid:B')
c = InMemoryDROP('oid:A', 'uid:A')
d = OddAndEvenContainerApp('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
f = InMemoryDROP('oid:F', 'uid:F')
# Wire them together
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
d.addOutput(f)
# Start the execution
with DROPWaiterCtx(self, [e,f]):
a.write('20')
a.setCompleted()
# Check the final results are correct
for drop in [a,b,c,d,e]:
self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
self.assertEqual(six.b("0 2 4 6 8 10 12 14 16 18"), droputils.allDropContents(e).strip())
self.assertEqual(six.b("1 3 5 7 9 11 13 15 17 19"), droputils.allDropContents(f).strip())
def test_app_multiple_outputs(self):
"""
A small method that tests that the AppDROPs writing to two
different DROPs outputs works
The graph constructed by this example looks as follow:
|--> E
A --> B --> C --> D --|
|--> F
Here B and D are an AppDROPs, with D writing to two DROPs
outputs (E and F) and reading from C. C, in turn, is written by B, which
in turns reads the data from A
"""
# This is used as "B"
class NumberWriterApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in xrange(howMany):
output.write(str(i) + " ")
# This is used as "D"
class OddAndEvenContainerApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + " ")
# Create DROPs
a = InMemoryDROP("oid:A", "uid:A")
b = NumberWriterApp("oid:B", "uid:B")
c = InMemoryDROP("oid:A", "uid:A")
d = OddAndEvenContainerApp("oid:D", "uid:D")
e = InMemoryDROP("oid:E", "uid:E")
f = InMemoryDROP("oid:F", "uid:F")
# Wire them together
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
d.addOutput(f)
# Start the execution
with DROPWaiterCtx(self, [e, f]):
a.write("20")
a.setCompleted()
# Check the final results are correct
for drop in [a, b, c, d, e]:
self.assertEquals(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
self.assertEquals("0 2 4 6 8 10 12 14 16 18", droputils.allDropContents(e).strip())
self.assertEquals("1 3 5 7 9 11 13 15 17 19", droputils.allDropContents(f).strip())
def assertMsgIsCorrect(msg, command):
a = DockerApp('a', 'a', image='ubuntu:14.04', command=command)
b = FileDROP('b','b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.execute()
self.assertEquals(msg, droputils.allDropContents(b))
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = BytesIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = BytesIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
def test_speadApp(self):
port = 1111
itemId = 0x2000
thread_pool = spead2.ThreadPool()
self._stream = spead2.send.UdpStream(
thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7))
a = SpeadReceiverApp('a', 'a', port=port, itemId=itemId)
b = InMemoryDROP('b', 'b')
a.addOutput(b)
size = 1024
threading.Thread(target=lambda: a.execute()).start()
time.sleep(1)
msg = os.urandom(size)
with DROPWaiterCtx(self, b, timeout=1):
ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48))
item = ig.add_item(itemId,
'main_data',
'a char array',
shape=(size, ),
format=[('c', 8)])
item.value = msg
self._stream.send_heap(ig.get_heap())
self._stream.send_heap(ig.get_end())
for drop in a, b:
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual(size, b.size)
self.assertEqual(msg, droputils.allDropContents(b))
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = StringIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
def test_speadApp(self):
port = 1111
itemId = 0x2000
thread_pool = spead2.ThreadPool()
self._stream = spead2.send.UdpStream(thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7))
a = SpeadReceiverApp('a','a',port=port, itemId=itemId)
b = InMemoryDROP('b','b')
a.addOutput(b)
size = 1024
threading.Thread(target=lambda: a.execute()).start()
time.sleep(1)
msg = os.urandom(size)
with DROPWaiterCtx(self, b, timeout=1):
ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48))
item = ig.add_item(itemId, 'main_data', 'a char array', shape=(size,), format=[('c',8)])
item.value = msg
self._stream.send_heap(ig.get_heap())
self._stream.send_heap(ig.get_end())
for drop in a,b:
self.assertEquals(DROPStates.COMPLETED, drop.status)
self.assertEquals(size, b.size)
self.assertEquals(msg, droputils.allDropContents(b))
def assert_message_is_correct(message, command):
a = BashShellApp('a', 'a', command=command)
b = FileDROP('b', 'b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.execute()
self.assertEquals(message, droputils.allDropContents(b))
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + " ")
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = StringIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
def assertMsgIsCorrect(msg, command):
a = DockerApp('a', 'a', image='ubuntu:14.04', command=command)
b = FileDROP('b', 'b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.execute()
self.assertEqual(six.b(msg), droputils.allDropContents(b))
def assert_message_is_correct(message, command):
a = BashShellApp('a', 'a', command=command)
b = FileDROP('b', 'b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.async_execute()
self.assertEqual(six.b(message), droputils.allDropContents(b))
def test_dropWroteFromOutside(self):
"""
A different scenario to those tested above, in which the data
represented by the DROP isn't actually written *through* the
DROP. Still, the DROP needs to be moved to COMPLETED once
the data is written, and reading from it should still yield a correct
result
"""
# Write, but not through the DROP
a = FileDROP('A', 'A')
filename = a.path
msg = 'a message'
with open(filename, 'w') as f:
f.write(msg)
a.setCompleted()
# Read from the DROP
self.assertEqual(six.b(msg), droputils.allDropContents(a))
self.assertIsNone(a.checksum)
self.assertIsNone(a.size)
# We can manually set the size because the DROP wasn't able to calculate
# it itself; if we couldn't an exception would be thrown
a.size = len(msg)
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + six.b(" "))
def test_dropWroteFromOutside(self):
"""
A different scenario to those tested above, in which the data
represented by the DROP isn't actually written *through* the
DROP. Still, the DROP needs to be moved to COMPLETED once
the data is written, and reading from it should still yield a correct
result
"""
# Write, but not through the DROP
a = FileDROP("A", "A")
filename = a.path
msg = "a message"
with open(filename, "w") as f:
f.write(msg)
a.setCompleted()
# Read from the DROP
self.assertEquals(msg, droputils.allDropContents(a))
self.assertIsNone(a.checksum)
self.assertIsNone(a.size)
# We can manually set the size because the DROP wasn't able to calculate
# it itself; if we couldn't an exception would be thrown
a.size = len(msg)
def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastChar = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastChar = data[-1]
outputDrop.write(self._lastChar)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP("a", "a")
b = LastCharWriterApp("b", "b")
c = SumupContainerChecksum("c", "c")
d = InMemoryDROP("d", "d")
e = InMemoryDROP("e", "e")
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, lambda: a.addConsumer(b))
self.assertRaises(Exception, lambda: a.addStreamingConsumer(c))
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastChar):
self.assertEquals(aStatus, a.status)
self.assertEquals(dStatus, d.status)
self.assertEquals(eStatus, e.status)
self.assertEquals(lastChar, b._lastChar)
checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write("abcde")
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "e")
a.write("fghij")
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "j")
a.write("k")
with DROPWaiterCtx(self, [d, e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, "k")
self.assertEquals("ejk", droputils.allDropContents(d))
def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastByte = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastByte = six.indexbytes(data, -1)
outputDrop.write(self._lastByte)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP('a', 'a')
b = LastCharWriterApp('b', 'b')
c = SumupContainerChecksum('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, a.addConsumer, b)
self.assertRaises(Exception, a.addStreamingConsumer, c)
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastByte):
self.assertEqual(aStatus, a.status)
self.assertEqual(dStatus, d.status)
self.assertEqual(eStatus, e.status)
if lastByte is not None:
self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte))
checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write('abcde')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e')
a.write('fghij')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j')
a.write('k')
with DROPWaiterCtx(self, [d,e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k')
self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
def test_NullDROP(self):
"""
Check that the NullDROP is usable for testing
"""
a = NullDROP("A", "A", expectedSize=5)
a.write("1234")
a.write("5")
allContents = droputils.allDropContents(a)
self.assertFalse(allContents)
def _ngas_and_fs_io(self, command):
a = NgasDROP('a', 'a') # not a filesystem-related DROP, we can reference its URL in the command-line
b = DockerApp('b', 'b', image="ubuntu:14.04", command=command)
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
with DROPWaiterCtx(self, b, 100):
a.setCompleted()
self.assertEquals(a.dataURL, droputils.allDropContents(c))
def test_NullDROP(self):
"""
Check that the NullDROP is usable for testing
"""
a = NullDROP('A', 'A', expectedSize=5)
a.write("1234")
a.write("5")
allContents = droputils.allDropContents(a)
self.assertFalse(allContents)
def _ngas_and_fs_io(self, command):
a = NgasDROP(
'a', 'a'
) # not a filesystem-related DROP, we can reference its URL in the command-line
b = DockerApp('b', 'b', image="ubuntu:14.04", command=command)
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
with DROPWaiterCtx(self, b, 100):
a.setCompleted()
self.assertEqual(six.b(a.dataURL), droputils.allDropContents(c))
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allbytes = droputils.allDropContents(drop)
buf = bytearray()
for c in allbytes:
if c == six.b(' ') or c == six.b('\n'):
output.write(buf[::-1])
output.write(c)
buf = bytearray()
else:
buf.append(c)
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = StringIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
buf = ""
for c in line:
if c == " " or c == "\n":
output.write(buf[::-1])
output.write(c)
buf = ""
else:
buf += c
def test_two_simultaneous_pipes(self):
"""
A more complicated test where three bash applications run at the same
time. The first streams its output to the second one, while the second
one streams *its* output to the third one.
------------- -------------- ------------- -------------- ------------- ----------
| BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F |
| echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | |
-----*------- -------------- ----*--*----- -------------- -----*------- ----------
| | | |
\-------------|named-pipe|----------\ \-----------|named-pipe|-----------/
BashApp A writes "5 4 3 2 1" (each on a new line), which is read
by "cat" (BashApp C). The printed results (a copy of the original) are
streamed through D and read by "sort" (BashApp E), which writes the
output to F.
"""
output_fname = tempfile.mktemp()
a = StreamingOutputBashApp('a',
'a',
command=r"echo -en '5\n4\n3\n2\n1'")
b = InMemoryDROP('b', 'b')
c = StreamingInputOutputBashApp('c', 'c', command="cat")
d = InMemoryDROP('d', 'd')
e = StreamingInputBashApp('e', 'e', command="sort -n > %o0")
f = FileDROP('f', 'f', filepath=output_fname)
a.addOutput(b)
b.addStreamingConsumer(c)
c.addOutput(d)
d.addStreamingConsumer(e)
e.addOutput(f)
# Let's fire the app
with DROPWaiterCtx(self, f, 2):
a.async_execute()
# The application executed, finished, and its output was recorded
for drop in (a, b, c, d, e, f):
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual([1, 2, 3, 4, 5], [
int(x)
for x in droputils.allDropContents(f).strip().split(six.b('\n'))
])
# Clean up and go
os.remove(output_fname)
def test_socket_listener(self):
'''
A simple test to check that SocketListenerApps are indeed working as
expected; that is, they write the data they receive into their output,
and finish when the connection is closed from the client side
The data flow diagram looks like this:
A --> B --> C --> D
'''
host = 'localhost'
port = 9933
data = os.urandom(1025)
a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
b = InMemoryDROP('oid:B', 'uid:B')
c = SumupContainerChecksum('oid:C', 'uid:C')
d = InMemoryDROP('oid:D', 'uid:D')
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
# Create the socket, write, and close the connection, allowing
# A to move to COMPLETED
with DROPWaiterCtx(self, d, 3): # That's plenty of time
a.async_execute()
utils.write_to(host, port, data, 1)
for drop in [a,b,c,d]:
self.assertEqual(DROPStates.COMPLETED, drop.status)
# Our expectations are fulfilled!
bContents = droputils.allDropContents(b)
dContents = int(droputils.allDropContents(d))
self.assertEqual(data, bContents)
self.assertEqual(crc32(data, 0), dContents)
def test_socket_listener(self):
'''
A simple test to check that SocketListenerApps are indeed working as
expected; that is, they write the data they receive into their output,
and finish when the connection is closed from the client side
The data flow diagram looks like this:
A --> B --> C --> D
'''
host = 'localhost'
port = 9933
data = 'shine on you crazy diamond'
a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
b = InMemoryDROP('oid:B', 'uid:B')
c = SumupContainerChecksum('oid:C', 'uid:C')
d = InMemoryDROP('oid:D', 'uid:D')
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
# Create the socket, write, and close the connection, allowing
# A to move to COMPLETED
with DROPWaiterCtx(self, d, 3): # That's plenty of time
threading.Thread(target=lambda a: a.execute(), args=(a,)).start()
utils.writeToRemotePort(host, port, data, 1)
for drop in [a,b,c,d]:
self.assertEquals(DROPStates.COMPLETED, drop.status)
# Our expectations are fulfilled!
bContents = droputils.allDropContents(b)
dContents = int(droputils.allDropContents(d))
self.assertEquals(data, bContents)
self.assertEquals(crc32(data, 0), dContents)
def test_clientServer(self):
"""
A client-server duo. The server outputs the data it receives to its
output DROP, which in turn is the data held in its input DROP. The graph
looks like this:
A --|--> B(client) --|--> D
|--> C(server) --|
C is a server application which B connects to. Therefore C must be
started before B, so B knows C's IP address and connects successfully.
Although the real writing is done by C, B in this example is also
treated as a publisher of D. This way D waits for both applications to
finish before proceeding.
"""
try:
AutoVersionClient().close()
except DockerException:
warnings.warn(
"Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b',
'b',
image='ubuntu:14.04',
command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
c = DockerApp('c',
'c',
image='ubuntu:14.04',
command='nc -l 8000 > %o0')
d = FileDROP('d', 'd')
b.addInput(a)
b.addOutput(d)
c.addInput(a)
c.addOutput(d)
# Let 'b' handle its interest in c
b.handleInterest(c)
data = os.urandom(10)
with DROPWaiterCtx(self, d, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(d))
def test_runGraphOneDOPerDOM(self):
"""
A test that creates three DROPs in two different DMs and runs the graph.
For this the graphs that are fed into the DMs must *not* express the
inter-DM relationships, although they are still passed down
separately. The graph looks like:
DM #1 DM #2
======= =============
| A --|----|-> B --> C |
======= =============
"""
dm1, dm2 = [self._start_dm() for _ in range(2)]
sessionId = 's1'
g1 = [{"oid": "A", "type": "plain", "storage": "memory"}]
g2 = [{
"oid": "B",
"type": "app",
"app": "dfms.apps.crc.CRCApp"
}, {
"oid": "C",
"type": "plain",
"storage": "memory",
"producers": ["B"]
}]
rels = [DROPRel('B', DROPLinkType.CONSUMER, 'A')]
quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels})
quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels})
self.assertEqual(1, len(dm1._sessions[sessionId].drops))
self.assertEqual(2, len(dm2._sessions[sessionId].drops))
# Run! We wait until c is completed
a = dm1._sessions[sessionId].drops['A']
b, c = [dm2._sessions[sessionId].drops[x] for x in ('B', 'C')]
with droputils.DROPWaiterCtx(self, c, 1):
a.write('a')
a.setCompleted()
for drop in a, b, c:
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual(a.checksum, int(droputils.allDropContents(c)))
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)
def test_single_pipe(self):
"""
A simple test where two bash apps are connected to each other in a
streaming fashion. The data flows through a pipe which is created by
the framework. The data drop in between acts only as a intermediator
to establish the underlying communication channel.
------------- -------------- ------------- ----------
| BashApp A | --> | InMemory B | --> | BashApp C | --> | File D |
| echo | | "/a/pipe" | | dc | | |
-----*------- -------------- ------*------ ----------
| |
\-------------|named-pipe|------------/
BashApp A writes "5 4 3 2 1" (each on a new line), which is read by cat
and redirected to D.
"""
output_fname = tempfile.mktemp()
a = StreamingOutputBashApp('a',
'a',
command=r"echo -en '5\n4\n3\n2\n1'")
b = InMemoryDROP('b', 'b')
c = StreamingInputBashApp('c', 'c', command="cat > %o0")
d = FileDROP('d', 'd', filepath=output_fname)
a.addOutput(b)
c.addStreamingInput(b)
c.addOutput(d)
# Let's fire the app
with DROPWaiterCtx(self, d, 2):
a.async_execute()
# The application executed, finished, and its output was recorded
for drop in (a, b, c, d):
self.assertEqual(DROPStates.COMPLETED, drop.status,
"Drop %r not COMPLETED: %d" % (drop, drop.status))
self.assertEqual(
[5, 4, 3, 2, 1],
[int(x) for x in droputils.allDropContents(d).split(six.b('\n'))])
# Clean up and go
os.remove(output_fname)
def test_runGraphOneDOPerDOM(self):
"""
A test that creates three DROPs in two different DMs, wire two of
them together externally (i.e., using their proxies), and runs the graph.
For this the graphs that are fed into the DMs must *not* express the
inter-DM relationships. The graph looks like:
DM #1 DM #2
======= =============
| A --|----|-> B --> C |
======= =============
"""
dm1 = NodeManager(useDLM=False)
dm2 = NodeManager(useDLM=False)
sessionId = 's1'
g1 = [{"oid":"A", "type":"plain", "storage": "memory"}]
g2 = [{"oid":"B", "type":"app", "app":"dfms.apps.crc.CRCApp"},
{"oid":"C", "type":"plain", "storage": "memory", "producers":["B"]}]
uris1 = dm1.quickDeploy(sessionId, g1)
uris2 = dm2.quickDeploy(sessionId, g2)
self.assertEquals(1, len(uris1))
self.assertEquals(2, len(uris2))
# We externally wire the Proxy objects now
a = Pyro4.Proxy(uris1['A'])
b = Pyro4.Proxy(uris2['B'])
c = Pyro4.Proxy(uris2['C'])
a.addConsumer(b)
# Run! We wait until c is completed
with droputils.EvtConsumerProxyCtx(self, c, 1):
a.write('a')
a.setCompleted()
for drop in a, b, c:
self.assertEquals(DROPStates.COMPLETED, drop.status)
self.assertEquals(a.checksum, int(droputils.allDropContents(c)))
for dropProxy in a,b,c:
dropProxy._pyroRelease()
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)
def test_echo(self):
a = FileDROP('a', 'a')
b = BashShellApp('b', 'b', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEquals(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEquals(uid, os.stat(c.path).st_uid)
def test_echo(self):
a = FileDROP('a', 'a')
b = BashShellApp('b', 'b', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = os.urandom(10)
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEqual(uid, os.stat(c.path).st_uid)
def test_clientServer(self):
"""
A client-server duo. The server outputs the data it receives to its
output DROP, which in turn is the data held in its input DROP. The graph
looks like this:
A --|--> B(client) --|--> D
|--> C(server) --|
C is a server application which B connects to. Therefore C must be
started before B, so B knows C's IP address and connects successfully.
Although the real writing is done by C, B in this example is also
treated as a publisher of D. This way D waits for both applications to
finish before proceeding.
"""
try:
AutoVersionClient()
except DockerException:
warnings.warn("Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
c = DockerApp('c', 'c', image='ubuntu:14.04', command='nc -l 8000 > %o0')
d = FileDROP('d', 'd')
b.addInput(a)
b.addOutput(d)
c.addInput(a)
c.addOutput(d)
# Let 'b' handle its interest in c
b.handleInterest(c)
data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
with DROPWaiterCtx(self, d, 100):
a.write(data)
a.setCompleted()
self.assertEquals(data, droputils.allDropContents(d))
def _test_write_withDropType(self, dropType):
"""
Test an AbstractDROP and a simple AppDROP (for checksum calculation)
"""
a = dropType("oid:A", "uid:A", expectedSize=self._test_drop_sz * ONE_MB)
b = SumupContainerChecksum("oid:B", "uid:B")
c = InMemoryDROP("oid:C", "uid:C")
b.addInput(a)
b.addOutput(c)
test_crc = 0
with DROPWaiterCtx(self, c):
for _ in range(self._test_num_blocks):
a.write(self._test_block)
test_crc = crc32(self._test_block, test_crc)
# Read the checksum from c
cChecksum = int(droputils.allDropContents(c))
self.assertNotEquals(a.checksum, 0)
self.assertEquals(a.checksum, test_crc)
self.assertEquals(cChecksum, test_crc)
def _test_write_withDropType(self, dropType):
"""
Test an AbstractDROP and a simple AppDROP (for checksum calculation)
"""
a = dropType('oid:A', 'uid:A', expectedSize = self._test_drop_sz * ONE_MB)
b = SumupContainerChecksum('oid:B', 'uid:B')
c = InMemoryDROP('oid:C', 'uid:C')
b.addInput(a)
b.addOutput(c)
test_crc = 0
with DROPWaiterCtx(self, c):
for _ in range(self._test_num_blocks):
a.write(self._test_block)
test_crc = crc32(self._test_block, test_crc)
# Read the checksum from c
cChecksum = int(droputils.allDropContents(c))
self.assertNotEqual(a.checksum, 0)
self.assertEqual(a.checksum, test_crc)
self.assertEqual(cChecksum, test_crc)
def test_simpleCopy(self):
"""
Simple test for a dockerized application. It copies the contents of one
file into another via the command-line cp utility. It then checks that
the contents of the target DROP are correct, and that the target file is
actually owned by our process.
The test will not run if a docker daemon cannot be contacted though;
this is to avoid failures in machines that don't have a docker service
running.
"""
try:
AutoVersionClient().close()
except DockerException:
warnings.warn(
"Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = os.urandom(10)
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEqual(uid, os.stat(c.path).st_uid)
def test_simpleCopy(self):
"""
Simple test for a dockerized application. It copies the contents of one
file into another via the command-line cp utility. It then checks that
the contents of the target DROP are correct, and that the target file is
actually owned by our process.
The test will not run if a docker daemon cannot be contacted though;
this is to avoid failures in machines that don't have a docker service
running.
"""
try:
AutoVersionClient()
except DockerException:
warnings.warn("Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEquals(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEquals(uid, os.stat(c.path).st_uid)
def test_simple_chain(self):
"""
Simple test that creates a pipeline-like chain of commands.
In this case we simulate a pipeline that does this, holding
each intermediate result in memory:
cat someFile | grep 'a' | sort | rev
"""
class GrepResult(BarrierAppDROP):
def initialize(self, **kwargs):
super(GrepResult, self).initialize(**kwargs)
self._substring = kwargs["substring"]
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = StringIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
class SortResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = StringIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
class RevResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = StringIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
buf = ""
for c in line:
if c == " " or c == "\n":
output.write(buf[::-1])
output.write(c)
buf = ""
else:
buf += c
a = InMemoryDROP("oid:A", "uid:A")
b = GrepResult("oid:B", "uid:B", substring="a")
c = InMemoryDROP("oid:C", "uid:C")
d = SortResult("oid:D", "uid:D")
e = InMemoryDROP("oid:E", "uid:E")
f = RevResult("oid:F", "oid:F")
g = InMemoryDROP("oid:G", "uid:G")
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
e.addConsumer(f)
f.addOutput(g)
# Initial write
contents = "first line\nwe have an a here\nand another one\nnoone knows me"
cResExpected = "we have an a here\nand another one\n"
eResExpected = "and another one\nwe have an a here\n"
gResExpected = "dna rehtona eno\new evah na a ereh\n"
with DROPWaiterCtx(self, g):
a.write(contents)
a.setCompleted()
# Get intermediate and final results and compare
actualRes = []
for i in [c, e, g]:
actualRes.append(droputils.allDropContents(i))
map(lambda x, y: self.assertEquals(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
def test_join(self):
"""
Using the container data object to implement a join/barrier dataflow.
A1, A2 and A3 are FileDROPs
B1, B2 and B3 are SumupContainerChecksum
C1, C2 and C3 are InMemoryDROPs
D is a SumupContainerChecksum
E is a InMemoryDROP
--> A1 --> B1 --> C1 --|
--> A2 --> B2 --> C2 --|--> D --> E
--> A3 --> B3 --> C3 --|
Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
after which "C" will transition to COMPLETE. Once all "C"s have moved to
COMPLETED "D"'s execution will also be triggered, and finally E will
hold the sum of B1, B2 and B3's checksums
"""
filelen = self._test_drop_sz * ONE_MB
# create file data objects
a1 = FileDROP("oid:A1", "uid:A1", expectedSize=filelen)
a2 = FileDROP("oid:A2", "uid:A2", expectedSize=filelen)
a3 = FileDROP("oid:A3", "uid:A3", expectedSize=filelen)
# CRC Result DROPs, storing the result in memory
b1 = SumupContainerChecksum("oid:B1", "uid:B1")
b2 = SumupContainerChecksum("oid:B2", "uid:B2")
b3 = SumupContainerChecksum("oid:B3", "uid:B3")
c1 = InMemoryDROP("oid:C1", "uid:C1")
c2 = InMemoryDROP("oid:C2", "uid:C2")
c3 = InMemoryDROP("oid:C3", "uid:C3")
# The final DROP that sums up the CRCs from the container DROP
d = SumupContainerChecksum("oid:D", "uid:D")
e = InMemoryDROP("oid:E", "uid:E")
# Wire together
dropAList = [a1, a2, a3]
dropBList = [b1, b2, b3]
dropCList = [c1, c2, c3]
for dropA, dropB in map(lambda a, b: (a, b), dropAList, dropBList):
dropA.addConsumer(dropB)
for dropB, dropC in map(lambda b, c: (b, c), dropBList, dropCList):
dropB.addOutput(dropC)
for dropC in dropCList:
dropC.addConsumer(d)
d.addOutput(e)
# Write data into the initial "A" DROPs, which should trigger
# the whole chain explained above
with DROPWaiterCtx(self, e):
for dropA in dropAList: # this should be parallel for
for _ in range(self._test_num_blocks):
dropA.write(self._test_block)
# All DROPs are completed now that the chain executed correctly
for drop in dropAList + dropBList + dropCList:
self.assertEqual(drop.status, DROPStates.COMPLETED)
# The results we want to compare
sum_crc = c1.checksum + c2.checksum + c3.checksum
dropEData = int(droputils.allDropContents(e))
self.assertNotEquals(sum_crc, 0)
self.assertEquals(sum_crc, dropEData)
def test_simple_chain(self):
'''
Simple test that creates a pipeline-like chain of commands.
In this case we simulate a pipeline that does this, holding
each intermediate result in memory:
cat someFile | grep 'a' | sort | rev
'''
class GrepResult(BarrierAppDROP):
def initialize(self, **kwargs):
super(GrepResult, self).initialize(**kwargs)
self._substring = six.b(kwargs['substring'])
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = BytesIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
class SortResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = BytesIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
class RevResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allbytes = droputils.allDropContents(drop)
buf = bytearray()
for c in allbytes:
if c == six.b(' ') or c == six.b('\n'):
output.write(buf[::-1])
output.write(c)
buf = bytearray()
else:
buf.append(c)
a = InMemoryDROP('oid:A', 'uid:A')
b = GrepResult('oid:B', 'uid:B', substring="a")
c = InMemoryDROP('oid:C', 'uid:C')
d = SortResult('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
f = RevResult('oid:F', 'oid:F')
g = InMemoryDROP('oid:G', 'uid:G')
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
e.addConsumer(f)
f.addOutput(g)
# Initial write
contents = "first line\nwe have an a here\nand another one\nnoone knows me"
cResExpected = "we have an a here\nand another one\n"
eResExpected = "and another one\nwe have an a here\n"
gResExpected = "dna rehtona eno\new evah na a ereh\n"
with DROPWaiterCtx(self, g):
a.write(contents)
a.setCompleted()
# Get intermediate and final results and compare
actualRes = []
for i in [c, e, g]:
actualRes.append(droputils.allDropContents(i))
map(lambda x, y: self.assertEqual(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
def branch_failure(self, tooManyFailures):
"""
Using the container data object to implement a join/barrier dataflow.
A1, A2 and A3 are FileDROPs
B1, B2 and B3 are SumupContainerChecksum
C1, C2 and C3 are InMemoryDROPs
D is a SumupContainerChecksum
E is a InMemoryDROP
--> A1 --> B1 --> C1 --|
--> A2 --> B2 --> C2 --|--> D --> E
--> A3 --> B3 --> C3 --|
Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
after which "C" will transition to COMPLETE. Once all "C"s have moved to
COMPLETED "D"'s execution will also be triggered, and finally E will
hold the sum of B1, B2 and B3's checksums
"""
#create file data objects
a1 = InMemoryDROP('oid:A1', 'uid:A1')
a2 = InMemoryDROP('oid:A2', 'uid:A2')
a3 = InMemoryDROP('oid:A3', 'uid:A3')
# CRC Result DROPs, storing the result in memory
b1 = SumupContainerChecksum('oid:B1', 'uid:B1')
b2 = SumupContainerChecksum('oid:B2', 'uid:B2')
b3 = SumupContainerChecksum('oid:B3', 'uid:B3')
c1 = InMemoryDROP('oid:C1', 'uid:C1')
c2 = InMemoryDROP('oid:C2', 'uid:C2')
c3 = InMemoryDROP('oid:C3', 'uid:C3')
# The final DROP that sums up the CRCs from the container DROP
d = SumupContainerChecksum('oid:D', 'uid:D', input_error_threshold = 33)
e = InMemoryDROP('oid:E', 'uid:E')
# Wire together
dropAList = [a1,a2,a3]
dropBList = [b1,b2,b3]
dropCList = [c1,c2,c3]
for dropA,dropB in zip(dropAList, dropBList):
dropA.addConsumer(dropB)
for dropB,dropC in zip(dropBList, dropCList):
dropB.addOutput(dropC)
for dropC in dropCList:
dropC.addConsumer(d)
d.addOutput(e)
# Write data into the initial "A" DROPs, which should trigger
# the whole chain explained above
with DROPWaiterCtx(self, e):
#for dropA in dropAList: # this should be parallel for
a1.write(' '); a1.setCompleted()
if tooManyFailures:
a2.setError()
else:
a2.write(' '); a2.setCompleted()
a3.setError()
if tooManyFailures:
completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1]
errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e]
else:
completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e]
errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:]
for drop in completedDrops:
self.assertEqual(drop.status, DROPStates.COMPLETED)
for drop in errorDrops:
self.assertEqual(drop.status, DROPStates.ERROR)
# The results we want to compare
# (only in case that at least two branches executed)
if not tooManyFailures:
sum_crc = c1.checksum + c2.checksum
dropEData = int(droputils.allDropContents(e))
self.assertNotEqual(sum_crc, 0)
self.assertEqual(sum_crc, dropEData)
def test_join(self):
"""
Using the container data object to implement a join/barrier dataflow.
A1, A2 and A3 are FileDROPs
B1, B2 and B3 are SumupContainerChecksum
C1, C2 and C3 are InMemoryDROPs
D is a SumupContainerChecksum
E is a InMemoryDROP
--> A1 --> B1 --> C1 --|
--> A2 --> B2 --> C2 --|--> D --> E
--> A3 --> B3 --> C3 --|
Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
after which "C" will transition to COMPLETE. Once all "C"s have moved to
COMPLETED "D"'s execution will also be triggered, and finally E will
hold the sum of B1, B2 and B3's checksums
"""
filelen = self._test_drop_sz * ONE_MB
#create file data objects
a1 = FileDROP('oid:A1', 'uid:A1', expectedSize=filelen)
a2 = FileDROP('oid:A2', 'uid:A2', expectedSize=filelen)
a3 = FileDROP('oid:A3', 'uid:A3', expectedSize=filelen)
# CRC Result DROPs, storing the result in memory
b1 = SumupContainerChecksum('oid:B1', 'uid:B1')
b2 = SumupContainerChecksum('oid:B2', 'uid:B2')
b3 = SumupContainerChecksum('oid:B3', 'uid:B3')
c1 = InMemoryDROP('oid:C1', 'uid:C1')
c2 = InMemoryDROP('oid:C2', 'uid:C2')
c3 = InMemoryDROP('oid:C3', 'uid:C3')
# The final DROP that sums up the CRCs from the container DROP
d = SumupContainerChecksum('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
# Wire together
dropAList = [a1,a2,a3]
dropBList = [b1,b2,b3]
dropCList = [c1,c2,c3]
for dropA,dropB in map(lambda a,b: (a,b), dropAList, dropBList):
dropA.addConsumer(dropB)
for dropB,dropC in map(lambda b,c: (b,c), dropBList, dropCList):
dropB.addOutput(dropC)
for dropC in dropCList:
dropC.addConsumer(d)
d.addOutput(e)
# Write data into the initial "A" DROPs, which should trigger
# the whole chain explained above
with DROPWaiterCtx(self, e):
for dropA in dropAList: # this should be parallel for
for _ in range(self._test_num_blocks):
dropA.write(self._test_block)
# All DROPs are completed now that the chain executed correctly
for drop in dropAList + dropBList + dropCList:
self.assertEqual(drop.status, DROPStates.COMPLETED)
# The results we want to compare
sum_crc = c1.checksum + c2.checksum + c3.checksum
dropEData = int(droputils.allDropContents(e))
self.assertNotEqual(sum_crc, 0)
self.assertEqual(sum_crc, dropEData)
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in range(howMany):
output.write(str(i) + " ")
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in xrange(howMany):
output.write(str(i) + " ")
def test_runGraphSeveralDropsPerDM(self):
"""
A test that creates several DROPs in two different DMs and runs
the graph. The graph looks like this
DM #1 DM #2
=================== ================
| A --> C --> D --|----|-| |
| | | |--> E --> F |
| B --------------|----|-| |
=================== ================
:see: `self.test_runGraphSingleDOPerDOM`
"""
dm1, dm2 = [self._start_dm() for _ in range(2)]
sessionId = 's1'
g1 = [{
"oid": "A",
"type": "plain",
"storage": "memory",
"consumers": ["C"]
}, {
"oid": "B",
"type": "plain",
"storage": "memory"
}, {
"oid": "C",
"type": "app",
"app": "dfms.apps.crc.CRCApp"
}, {
"oid": "D",
"type": "plain",
"storage": "memory",
"producers": ["C"]
}]
g2 = [{
"oid": "E",
"type": "app",
"app": "test.test_drop.SumupContainerChecksum"
}, {
"oid": "F",
"type": "plain",
"storage": "memory",
"producers": ["E"]
}]
rels = [
DROPRel('D', DROPLinkType.INPUT, 'E'),
DROPRel('B', DROPLinkType.INPUT, 'E')
]
quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels})
quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels})
self.assertEqual(4, len(dm1._sessions[sessionId].drops))
self.assertEqual(2, len(dm2._sessions[sessionId].drops))
# Run! The sole fact that this doesn't throw exceptions is already
# a good proof that everything is working as expected
a, b, c, d = [
dm1._sessions[sessionId].drops[x] for x in ('A', 'B', 'C', 'D')
]
e, f = [dm2._sessions[sessionId].drops[x] for x in ('E', 'F')]
with droputils.DROPWaiterCtx(self, f, 5):
a.write('a')
a.setCompleted()
b.write('a')
b.setCompleted()
for drop in a, b, c, d, e, f:
self.assertEqual(DROPStates.COMPLETED, drop.status,
"DROP %s is not COMPLETED" % (drop.uid))
self.assertEqual(a.checksum, int(droputils.allDropContents(d)))
self.assertEqual(b.checksum + d.checksum,
int(droputils.allDropContents(f)))
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)
def test_runGraphSeveralDropsPerDM(self):
"""
A test that creates several DROPs in two different DMs and runs
the graph. The graph looks like this
DM #1 DM #2
=================== ================
| A --> C --> D --|----|-| |
| | | |--> E --> F |
| B --------------|----|-| |
=================== ================
:see: `self.test_runGraphSingleDOPerDOM`
"""
dm1 = NodeManager(useDLM=False)
dm2 = NodeManager(useDLM=False)
sessionId = 's1'
g1 = [{"oid":"A", "type":"plain", "storage": "memory", "consumers":["C"]},
{"oid":"B", "type":"plain", "storage": "memory"},
{"oid":"C", "type":"app", "app":"dfms.apps.crc.CRCApp"},
{"oid":"D", "type":"plain", "storage": "memory", "producers": ["C"]}]
g2 = [{"oid":"E", "type":"app", "app":"test.test_drop.SumupContainerChecksum"},
{"oid":"F", "type":"plain", "storage": "memory", "producers":["E"]}]
uris1 = dm1.quickDeploy(sessionId, g1)
uris2 = dm2.quickDeploy(sessionId, g2)
self.assertEquals(4, len(uris1))
self.assertEquals(2, len(uris2))
# We externally wire the Proxy objects to establish the inter-DM
# relationships
a = Pyro4.Proxy(uris1['A'])
b = Pyro4.Proxy(uris1['B'])
c = Pyro4.Proxy(uris1['C'])
d = Pyro4.Proxy(uris1['D'])
e = Pyro4.Proxy(uris2['E'])
f = Pyro4.Proxy(uris2['F'])
for drop,uid in [(a,'A'),(b,'B'),(c,'C'),(d,'D'),(e,'E'),(f,'F')]:
self.assertEquals(uid, drop.uid, "Proxy is not the DROP we think should be (assumed: %s/ actual: %s)" % (uid, drop.uid))
e.addInput(d)
e.addInput(b)
# Run! The sole fact that this doesn't throw exceptions is already
# a good proof that everything is working as expected
with droputils.EvtConsumerProxyCtx(self, f, 5):
a.write('a')
a.setCompleted()
b.write('a')
b.setCompleted()
for drop in a,b,c,d,e,f:
self.assertEquals(DROPStates.COMPLETED, drop.status, "DROP %s is not COMPLETED" % (drop.uid))
self.assertEquals(a.checksum, int(droputils.allDropContents(d)))
self.assertEquals(b.checksum + d.checksum, int(droputils.allDropContents(f)))
for dropProxy in a,b,c,d,e,f:
dropProxy._pyroRelease()
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)
def branch_failure(self, tooManyFailures):
"""
Using the container data object to implement a join/barrier dataflow.
A1, A2 and A3 are FileDROPs
B1, B2 and B3 are SumupContainerChecksum
C1, C2 and C3 are InMemoryDROPs
D is a SumupContainerChecksum
E is a InMemoryDROP
--> A1 --> B1 --> C1 --|
--> A2 --> B2 --> C2 --|--> D --> E
--> A3 --> B3 --> C3 --|
Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
after which "C" will transition to COMPLETE. Once all "C"s have moved to
COMPLETED "D"'s execution will also be triggered, and finally E will
hold the sum of B1, B2 and B3's checksums
"""
# create file data objects
a1 = InMemoryDROP("oid:A1", "uid:A1")
a2 = InMemoryDROP("oid:A2", "uid:A2")
a3 = InMemoryDROP("oid:A3", "uid:A3")
# CRC Result DROPs, storing the result in memory
b1 = SumupContainerChecksum("oid:B1", "uid:B1")
b2 = SumupContainerChecksum("oid:B2", "uid:B2")
b3 = SumupContainerChecksum("oid:B3", "uid:B3")
c1 = InMemoryDROP("oid:C1", "uid:C1")
c2 = InMemoryDROP("oid:C2", "uid:C2")
c3 = InMemoryDROP("oid:C3", "uid:C3")
# The final DROP that sums up the CRCs from the container DROP
d = SumupContainerChecksum("oid:D", "uid:D", input_error_threshold=33)
e = InMemoryDROP("oid:E", "uid:E")
# Wire together
dropAList = [a1, a2, a3]
dropBList = [b1, b2, b3]
dropCList = [c1, c2, c3]
for dropA, dropB in map(lambda a, b: (a, b), dropAList, dropBList):
dropA.addConsumer(dropB)
for dropB, dropC in map(lambda b, c: (b, c), dropBList, dropCList):
dropB.addOutput(dropC)
for dropC in dropCList:
dropC.addConsumer(d)
d.addOutput(e)
# Write data into the initial "A" DROPs, which should trigger
# the whole chain explained above
with DROPWaiterCtx(self, e):
# for dropA in dropAList: # this should be parallel for
a1.write(" ")
a1.setCompleted()
if tooManyFailures:
a2.setError()
else:
a2.write(" ")
a2.setCompleted()
a3.setError()
if tooManyFailures:
completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1]
errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e]
else:
completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e]
errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:]
for drop in completedDrops:
self.assertEquals(drop.status, DROPStates.COMPLETED)
for drop in errorDrops:
self.assertEquals(drop.status, DROPStates.ERROR)
# The results we want to compare
# (only in case that at least two branches executed)
if not tooManyFailures:
sum_crc = c1.checksum + c2.checksum
dropEData = int(droputils.allDropContents(e))
self.assertNotEquals(sum_crc, 0)
self.assertEquals(sum_crc, dropEData)
