def test_app_multiple_outputs(self): """ A small method that tests that the AppDROPs writing to two different DROPs outputs works The graph constructed by this example looks as follow: |--> E A --> B --> C --> D --| |--> F Here B and D are an AppDROPs, with D writing to two DROPs outputs (E and F) and reading from C. C, in turn, is written by B, which in turns reads the data from A """ # This is used as "B" class NumberWriterApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] output = self.outputs[0] howMany = int(droputils.allDropContents(inputDrop)) for i in range(howMany): output.write(str(i) + " ") # This is used as "D" class OddAndEvenContainerApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] outputs = self.outputs numbers = droputils.allDropContents(inputDrop).strip().split() for n in numbers: outputs[int(n) % 2].write(n + six.b(" ")) # Create DROPs a = InMemoryDROP('oid:A', 'uid:A') b = NumberWriterApp('oid:B', 'uid:B') c = InMemoryDROP('oid:A', 'uid:A') d = OddAndEvenContainerApp('oid:D', 'uid:D') e = InMemoryDROP('oid:E', 'uid:E') f = InMemoryDROP('oid:F', 'uid:F') # Wire them together a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) d.addOutput(f) # Start the execution with DROPWaiterCtx(self, [e,f]): a.write('20') a.setCompleted() # Check the final results are correct for drop in [a,b,c,d,e]: self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop)) self.assertEqual(six.b("0 2 4 6 8 10 12 14 16 18"), droputils.allDropContents(e).strip()) self.assertEqual(six.b("1 3 5 7 9 11 13 15 17 19"), droputils.allDropContents(f).strip())
def test_app_multiple_outputs(self): """ A small method that tests that the AppDROPs writing to two different DROPs outputs works The graph constructed by this example looks as follow: |--> E A --> B --> C --> D --| |--> F Here B and D are an AppDROPs, with D writing to two DROPs outputs (E and F) and reading from C. C, in turn, is written by B, which in turns reads the data from A """ # This is used as "B" class NumberWriterApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] output = self.outputs[0] howMany = int(droputils.allDropContents(inputDrop)) for i in xrange(howMany): output.write(str(i) + " ") # This is used as "D" class OddAndEvenContainerApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] outputs = self.outputs numbers = droputils.allDropContents(inputDrop).strip().split() for n in numbers: outputs[int(n) % 2].write(n + " ") # Create DROPs a = InMemoryDROP("oid:A", "uid:A") b = NumberWriterApp("oid:B", "uid:B") c = InMemoryDROP("oid:A", "uid:A") d = OddAndEvenContainerApp("oid:D", "uid:D") e = InMemoryDROP("oid:E", "uid:E") f = InMemoryDROP("oid:F", "uid:F") # Wire them together a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) d.addOutput(f) # Start the execution with DROPWaiterCtx(self, [e, f]): a.write("20") a.setCompleted() # Check the final results are correct for drop in [a, b, c, d, e]: self.assertEquals(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop)) self.assertEquals("0 2 4 6 8 10 12 14 16 18", droputils.allDropContents(e).strip()) self.assertEquals("1 3 5 7 9 11 13 15 17 19", droputils.allDropContents(f).strip())
def assertMsgIsCorrect(msg, command): a = DockerApp('a', 'a', image='ubuntu:14.04', command=command) b = FileDROP('b','b') a.addOutput(b) with DROPWaiterCtx(self, b, 100): a.execute() self.assertEquals(msg, droputils.allDropContents(b))
def run(self): drop = self.inputs[0] output = self.outputs[0] sortedLines = BytesIO(droputils.allDropContents(drop)).readlines() sortedLines.sort() for line in sortedLines: output.write(line)
def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = BytesIO(droputils.allDropContents(drop)).readlines() for line in allLines: if self._substring in line: output.write(line)
def test_speadApp(self): port = 1111 itemId = 0x2000 thread_pool = spead2.ThreadPool() self._stream = spead2.send.UdpStream( thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7)) a = SpeadReceiverApp('a', 'a', port=port, itemId=itemId) b = InMemoryDROP('b', 'b') a.addOutput(b) size = 1024 threading.Thread(target=lambda: a.execute()).start() time.sleep(1) msg = os.urandom(size) with DROPWaiterCtx(self, b, timeout=1): ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48)) item = ig.add_item(itemId, 'main_data', 'a char array', shape=(size, ), format=[('c', 8)]) item.value = msg self._stream.send_heap(ig.get_heap()) self._stream.send_heap(ig.get_end()) for drop in a, b: self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual(size, b.size) self.assertEqual(msg, droputils.allDropContents(b))
def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = StringIO(droputils.allDropContents(drop)).readlines() for line in allLines: if self._substring in line: output.write(line)
def test_speadApp(self): port = 1111 itemId = 0x2000 thread_pool = spead2.ThreadPool() self._stream = spead2.send.UdpStream(thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7)) a = SpeadReceiverApp('a','a',port=port, itemId=itemId) b = InMemoryDROP('b','b') a.addOutput(b) size = 1024 threading.Thread(target=lambda: a.execute()).start() time.sleep(1) msg = os.urandom(size) with DROPWaiterCtx(self, b, timeout=1): ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48)) item = ig.add_item(itemId, 'main_data', 'a char array', shape=(size,), format=[('c',8)]) item.value = msg self._stream.send_heap(ig.get_heap()) self._stream.send_heap(ig.get_end()) for drop in a,b: self.assertEquals(DROPStates.COMPLETED, drop.status) self.assertEquals(size, b.size) self.assertEquals(msg, droputils.allDropContents(b))
def assert_message_is_correct(message, command): a = BashShellApp('a', 'a', command=command) b = FileDROP('b', 'b') a.addOutput(b) with DROPWaiterCtx(self, b, 100): a.execute() self.assertEquals(message, droputils.allDropContents(b))
def run(self): inputDrop = self.inputs[0] outputs = self.outputs numbers = droputils.allDropContents(inputDrop).strip().split() for n in numbers: outputs[int(n) % 2].write(n + " ")
def run(self): drop = self.inputs[0] output = self.outputs[0] sortedLines = StringIO(droputils.allDropContents(drop)).readlines() sortedLines.sort() for line in sortedLines: output.write(line)
def assertMsgIsCorrect(msg, command): a = DockerApp('a', 'a', image='ubuntu:14.04', command=command) b = FileDROP('b', 'b') a.addOutput(b) with DROPWaiterCtx(self, b, 100): a.execute() self.assertEqual(six.b(msg), droputils.allDropContents(b))
def assert_message_is_correct(message, command): a = BashShellApp('a', 'a', command=command) b = FileDROP('b', 'b') a.addOutput(b) with DROPWaiterCtx(self, b, 100): a.async_execute() self.assertEqual(six.b(message), droputils.allDropContents(b))
def test_dropWroteFromOutside(self): """ A different scenario to those tested above, in which the data represented by the DROP isn't actually written *through* the DROP. Still, the DROP needs to be moved to COMPLETED once the data is written, and reading from it should still yield a correct result """ # Write, but not through the DROP a = FileDROP('A', 'A') filename = a.path msg = 'a message' with open(filename, 'w') as f: f.write(msg) a.setCompleted() # Read from the DROP self.assertEqual(six.b(msg), droputils.allDropContents(a)) self.assertIsNone(a.checksum) self.assertIsNone(a.size) # We can manually set the size because the DROP wasn't able to calculate # it itself; if we couldn't an exception would be thrown a.size = len(msg)
def run(self): inputDrop = self.inputs[0] outputs = self.outputs numbers = droputils.allDropContents(inputDrop).strip().split() for n in numbers: outputs[int(n) % 2].write(n + six.b(" "))
def test_dropWroteFromOutside(self): """ A different scenario to those tested above, in which the data represented by the DROP isn't actually written *through* the DROP. Still, the DROP needs to be moved to COMPLETED once the data is written, and reading from it should still yield a correct result """ # Write, but not through the DROP a = FileDROP("A", "A") filename = a.path msg = "a message" with open(filename, "w") as f: f.write(msg) a.setCompleted() # Read from the DROP self.assertEquals(msg, droputils.allDropContents(a)) self.assertIsNone(a.checksum) self.assertIsNone(a.size) # We can manually set the size because the DROP wasn't able to calculate # it itself; if we couldn't an exception would be thrown a.size = len(msg)
def test_objectAsNormalAndStreamingInput(self): """ A test that checks that a DROP can act as normal and streaming input of different AppDROPs at the same time. We use the following graph: A --|--> B --> D |--> C --> E Here B uses A as a streaming input, while C uses it as a normal input """ class LastCharWriterApp(AppDROP): def initialize(self, **kwargs): super(LastCharWriterApp, self).initialize(**kwargs) self._lastChar = None def dataWritten(self, uid, data): self.execStatus = AppDROPStates.RUNNING outputDrop = self.outputs[0] self._lastChar = data[-1] outputDrop.write(self._lastChar) def dropCompleted(self, uid, status): self.execStatus = AppDROPStates.FINISHED self._notifyAppIsFinished() a = InMemoryDROP("a", "a") b = LastCharWriterApp("b", "b") c = SumupContainerChecksum("c", "c") d = InMemoryDROP("d", "d") e = InMemoryDROP("e", "e") a.addStreamingConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) # Consumer cannot be normal and streaming at the same time self.assertRaises(Exception, lambda: a.addConsumer(b)) self.assertRaises(Exception, lambda: a.addStreamingConsumer(c)) # Write a little, then check the consumers def checkDropStates(aStatus, dStatus, eStatus, lastChar): self.assertEquals(aStatus, a.status) self.assertEquals(dStatus, d.status) self.assertEquals(eStatus, e.status) self.assertEquals(lastChar, b._lastChar) checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None) a.write("abcde") checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "e") a.write("fghij") checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "j") a.write("k") with DROPWaiterCtx(self, [d, e]): a.setCompleted() checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, "k") self.assertEquals("ejk", droputils.allDropContents(d))
def test_objectAsNormalAndStreamingInput(self): """ A test that checks that a DROP can act as normal and streaming input of different AppDROPs at the same time. We use the following graph: A --|--> B --> D |--> C --> E Here B uses A as a streaming input, while C uses it as a normal input """ class LastCharWriterApp(AppDROP): def initialize(self, **kwargs): super(LastCharWriterApp, self).initialize(**kwargs) self._lastByte = None def dataWritten(self, uid, data): self.execStatus = AppDROPStates.RUNNING outputDrop = self.outputs[0] self._lastByte = six.indexbytes(data, -1) outputDrop.write(self._lastByte) def dropCompleted(self, uid, status): self.execStatus = AppDROPStates.FINISHED self._notifyAppIsFinished() a = InMemoryDROP('a', 'a') b = LastCharWriterApp('b', 'b') c = SumupContainerChecksum('c', 'c') d = InMemoryDROP('d', 'd') e = InMemoryDROP('e', 'e') a.addStreamingConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) # Consumer cannot be normal and streaming at the same time self.assertRaises(Exception, a.addConsumer, b) self.assertRaises(Exception, a.addStreamingConsumer, c) # Write a little, then check the consumers def checkDropStates(aStatus, dStatus, eStatus, lastByte): self.assertEqual(aStatus, a.status) self.assertEqual(dStatus, d.status) self.assertEqual(eStatus, e.status) if lastByte is not None: self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte)) checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None) a.write('abcde') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e') a.write('fghij') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j') a.write('k') with DROPWaiterCtx(self, [d,e]): a.setCompleted() checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k') self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
def test_NullDROP(self): """ Check that the NullDROP is usable for testing """ a = NullDROP("A", "A", expectedSize=5) a.write("1234") a.write("5") allContents = droputils.allDropContents(a) self.assertFalse(allContents)
def _ngas_and_fs_io(self, command): a = NgasDROP('a', 'a') # not a filesystem-related DROP, we can reference its URL in the command-line b = DockerApp('b', 'b', image="ubuntu:14.04", command=command) c = FileDROP('c', 'c') b.addInput(a) b.addOutput(c) with DROPWaiterCtx(self, b, 100): a.setCompleted() self.assertEquals(a.dataURL, droputils.allDropContents(c))
def test_NullDROP(self): """ Check that the NullDROP is usable for testing """ a = NullDROP('A', 'A', expectedSize=5) a.write("1234") a.write("5") allContents = droputils.allDropContents(a) self.assertFalse(allContents)
def _ngas_and_fs_io(self, command): a = NgasDROP( 'a', 'a' ) # not a filesystem-related DROP, we can reference its URL in the command-line b = DockerApp('b', 'b', image="ubuntu:14.04", command=command) c = FileDROP('c', 'c') b.addInput(a) b.addOutput(c) with DROPWaiterCtx(self, b, 100): a.setCompleted() self.assertEqual(six.b(a.dataURL), droputils.allDropContents(c))
def run(self): drop = self.inputs[0] output = self.outputs[0] allbytes = droputils.allDropContents(drop) buf = bytearray() for c in allbytes: if c == six.b(' ') or c == six.b('\n'): output.write(buf[::-1]) output.write(c) buf = bytearray() else: buf.append(c)
def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = StringIO(droputils.allDropContents(drop)).readlines() for line in allLines: buf = "" for c in line: if c == " " or c == "\n": output.write(buf[::-1]) output.write(c) buf = "" else: buf += c
def test_two_simultaneous_pipes(self): """ A more complicated test where three bash applications run at the same time. The first streams its output to the second one, while the second one streams *its* output to the third one. ------------- -------------- ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F | | echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | | -----*------- -------------- ----*--*----- -------------- -----*------- ---------- | | | | \-------------|named-pipe|----------\ \-----------|named-pipe|-----------/ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by "cat" (BashApp C). The printed results (a copy of the original) are streamed through D and read by "sort" (BashApp E), which writes the output to F. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp('a', 'a', command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP('b', 'b') c = StreamingInputOutputBashApp('c', 'c', command="cat") d = InMemoryDROP('d', 'd') e = StreamingInputBashApp('e', 'e', command="sort -n > %o0") f = FileDROP('f', 'f', filepath=output_fname) a.addOutput(b) b.addStreamingConsumer(c) c.addOutput(d) d.addStreamingConsumer(e) e.addOutput(f) # Let's fire the app with DROPWaiterCtx(self, f, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d, e, f): self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual([1, 2, 3, 4, 5], [ int(x) for x in droputils.allDropContents(f).strip().split(six.b('\n')) ]) # Clean up and go os.remove(output_fname)
def test_socket_listener(self): ''' A simple test to check that SocketListenerApps are indeed working as expected; that is, they write the data they receive into their output, and finish when the connection is closed from the client side The data flow diagram looks like this: A --> B --> C --> D ''' host = 'localhost' port = 9933 data = os.urandom(1025) a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port) b = InMemoryDROP('oid:B', 'uid:B') c = SumupContainerChecksum('oid:C', 'uid:C') d = InMemoryDROP('oid:D', 'uid:D') a.addOutput(b) b.addConsumer(c) c.addOutput(d) # Create the socket, write, and close the connection, allowing # A to move to COMPLETED with DROPWaiterCtx(self, d, 3): # That's plenty of time a.async_execute() utils.write_to(host, port, data, 1) for drop in [a,b,c,d]: self.assertEqual(DROPStates.COMPLETED, drop.status) # Our expectations are fulfilled! bContents = droputils.allDropContents(b) dContents = int(droputils.allDropContents(d)) self.assertEqual(data, bContents) self.assertEqual(crc32(data, 0), dContents)
def test_socket_listener(self): ''' A simple test to check that SocketListenerApps are indeed working as expected; that is, they write the data they receive into their output, and finish when the connection is closed from the client side The data flow diagram looks like this: A --> B --> C --> D ''' host = 'localhost' port = 9933 data = 'shine on you crazy diamond' a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port) b = InMemoryDROP('oid:B', 'uid:B') c = SumupContainerChecksum('oid:C', 'uid:C') d = InMemoryDROP('oid:D', 'uid:D') a.addOutput(b) b.addConsumer(c) c.addOutput(d) # Create the socket, write, and close the connection, allowing # A to move to COMPLETED with DROPWaiterCtx(self, d, 3): # That's plenty of time threading.Thread(target=lambda a: a.execute(), args=(a,)).start() utils.writeToRemotePort(host, port, data, 1) for drop in [a,b,c,d]: self.assertEquals(DROPStates.COMPLETED, drop.status) # Our expectations are fulfilled! bContents = droputils.allDropContents(b) dContents = int(droputils.allDropContents(d)) self.assertEquals(data, bContents) self.assertEquals(crc32(data, 0), dContents)
def test_clientServer(self): """ A client-server duo. The server outputs the data it receives to its output DROP, which in turn is the data held in its input DROP. The graph looks like this: A --|--> B(client) --|--> D |--> C(server) --| C is a server application which B connects to. Therefore C must be started before B, so B knows C's IP address and connects successfully. Although the real writing is done by C, B in this example is also treated as a publisher of D. This way D waits for both applications to finish before proceeding. """ try: AutoVersionClient().close() except DockerException: warnings.warn( "Cannot contact the Docker daemon, skipping docker tests") return a = FileDROP('a', 'a') b = DockerApp('b', 'b', image='ubuntu:14.04', command='cat %i0 > /dev/tcp/%containerIp[c]%/8000') c = DockerApp('c', 'c', image='ubuntu:14.04', command='nc -l 8000 > %o0') d = FileDROP('d', 'd') b.addInput(a) b.addOutput(d) c.addInput(a) c.addOutput(d) # Let 'b' handle its interest in c b.handleInterest(c) data = os.urandom(10) with DROPWaiterCtx(self, d, 100): a.write(data) a.setCompleted() self.assertEqual(data, droputils.allDropContents(d))
def test_runGraphOneDOPerDOM(self): """ A test that creates three DROPs in two different DMs and runs the graph. For this the graphs that are fed into the DMs must *not* express the inter-DM relationships, although they are still passed down separately. The graph looks like: DM #1 DM #2 ======= ============= | A --|----|-> B --> C | ======= ============= """ dm1, dm2 = [self._start_dm() for _ in range(2)] sessionId = 's1' g1 = [{"oid": "A", "type": "plain", "storage": "memory"}] g2 = [{ "oid": "B", "type": "app", "app": "dfms.apps.crc.CRCApp" }, { "oid": "C", "type": "plain", "storage": "memory", "producers": ["B"] }] rels = [DROPRel('B', DROPLinkType.CONSUMER, 'A')] quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels}) quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels}) self.assertEqual(1, len(dm1._sessions[sessionId].drops)) self.assertEqual(2, len(dm2._sessions[sessionId].drops)) # Run! We wait until c is completed a = dm1._sessions[sessionId].drops['A'] b, c = [dm2._sessions[sessionId].drops[x] for x in ('B', 'C')] with droputils.DROPWaiterCtx(self, c, 1): a.write('a') a.setCompleted() for drop in a, b, c: self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual(a.checksum, int(droputils.allDropContents(c))) dm1.destroySession(sessionId) dm2.destroySession(sessionId)
def test_single_pipe(self): """ A simple test where two bash apps are connected to each other in a streaming fashion. The data flows through a pipe which is created by the framework. The data drop in between acts only as a intermediator to establish the underlying communication channel. ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | File D | | echo | | "/a/pipe" | | dc | | | -----*------- -------------- ------*------ ---------- | | \-------------|named-pipe|------------/ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by cat and redirected to D. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp('a', 'a', command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP('b', 'b') c = StreamingInputBashApp('c', 'c', command="cat > %o0") d = FileDROP('d', 'd', filepath=output_fname) a.addOutput(b) c.addStreamingInput(b) c.addOutput(d) # Let's fire the app with DROPWaiterCtx(self, d, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d): self.assertEqual(DROPStates.COMPLETED, drop.status, "Drop %r not COMPLETED: %d" % (drop, drop.status)) self.assertEqual( [5, 4, 3, 2, 1], [int(x) for x in droputils.allDropContents(d).split(six.b('\n'))]) # Clean up and go os.remove(output_fname)
def test_runGraphOneDOPerDOM(self): """ A test that creates three DROPs in two different DMs, wire two of them together externally (i.e., using their proxies), and runs the graph. For this the graphs that are fed into the DMs must *not* express the inter-DM relationships. The graph looks like: DM #1 DM #2 ======= ============= | A --|----|-> B --> C | ======= ============= """ dm1 = NodeManager(useDLM=False) dm2 = NodeManager(useDLM=False) sessionId = 's1' g1 = [{"oid":"A", "type":"plain", "storage": "memory"}] g2 = [{"oid":"B", "type":"app", "app":"dfms.apps.crc.CRCApp"}, {"oid":"C", "type":"plain", "storage": "memory", "producers":["B"]}] uris1 = dm1.quickDeploy(sessionId, g1) uris2 = dm2.quickDeploy(sessionId, g2) self.assertEquals(1, len(uris1)) self.assertEquals(2, len(uris2)) # We externally wire the Proxy objects now a = Pyro4.Proxy(uris1['A']) b = Pyro4.Proxy(uris2['B']) c = Pyro4.Proxy(uris2['C']) a.addConsumer(b) # Run! We wait until c is completed with droputils.EvtConsumerProxyCtx(self, c, 1): a.write('a') a.setCompleted() for drop in a, b, c: self.assertEquals(DROPStates.COMPLETED, drop.status) self.assertEquals(a.checksum, int(droputils.allDropContents(c))) for dropProxy in a,b,c: dropProxy._pyroRelease() dm1.destroySession(sessionId) dm2.destroySession(sessionId)
def test_echo(self): a = FileDROP('a', 'a') b = BashShellApp('b', 'b', command='cp %i0 %o0') c = FileDROP('c', 'c') b.addInput(a) b.addOutput(c) # Random data so we always check different contents data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)]) with DROPWaiterCtx(self, c, 100): a.write(data) a.setCompleted() self.assertEquals(data, droputils.allDropContents(c)) # We own the file, not root uid = os.getuid() self.assertEquals(uid, os.stat(c.path).st_uid)
def test_echo(self): a = FileDROP('a', 'a') b = BashShellApp('b', 'b', command='cp %i0 %o0') c = FileDROP('c', 'c') b.addInput(a) b.addOutput(c) # Random data so we always check different contents data = os.urandom(10) with DROPWaiterCtx(self, c, 100): a.write(data) a.setCompleted() self.assertEqual(data, droputils.allDropContents(c)) # We own the file, not root uid = os.getuid() self.assertEqual(uid, os.stat(c.path).st_uid)
def test_clientServer(self): """ A client-server duo. The server outputs the data it receives to its output DROP, which in turn is the data held in its input DROP. The graph looks like this: A --|--> B(client) --|--> D |--> C(server) --| C is a server application which B connects to. Therefore C must be started before B, so B knows C's IP address and connects successfully. Although the real writing is done by C, B in this example is also treated as a publisher of D. This way D waits for both applications to finish before proceeding. """ try: AutoVersionClient() except DockerException: warnings.warn("Cannot contact the Docker daemon, skipping docker tests") return a = FileDROP('a', 'a') b = DockerApp('b', 'b', image='ubuntu:14.04', command='cat %i0 > /dev/tcp/%containerIp[c]%/8000') c = DockerApp('c', 'c', image='ubuntu:14.04', command='nc -l 8000 > %o0') d = FileDROP('d', 'd') b.addInput(a) b.addOutput(d) c.addInput(a) c.addOutput(d) # Let 'b' handle its interest in c b.handleInterest(c) data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)]) with DROPWaiterCtx(self, d, 100): a.write(data) a.setCompleted() self.assertEquals(data, droputils.allDropContents(d))
def _test_write_withDropType(self, dropType): """ Test an AbstractDROP and a simple AppDROP (for checksum calculation) """ a = dropType("oid:A", "uid:A", expectedSize=self._test_drop_sz * ONE_MB) b = SumupContainerChecksum("oid:B", "uid:B") c = InMemoryDROP("oid:C", "uid:C") b.addInput(a) b.addOutput(c) test_crc = 0 with DROPWaiterCtx(self, c): for _ in range(self._test_num_blocks): a.write(self._test_block) test_crc = crc32(self._test_block, test_crc) # Read the checksum from c cChecksum = int(droputils.allDropContents(c)) self.assertNotEquals(a.checksum, 0) self.assertEquals(a.checksum, test_crc) self.assertEquals(cChecksum, test_crc)
def _test_write_withDropType(self, dropType): """ Test an AbstractDROP and a simple AppDROP (for checksum calculation) """ a = dropType('oid:A', 'uid:A', expectedSize = self._test_drop_sz * ONE_MB) b = SumupContainerChecksum('oid:B', 'uid:B') c = InMemoryDROP('oid:C', 'uid:C') b.addInput(a) b.addOutput(c) test_crc = 0 with DROPWaiterCtx(self, c): for _ in range(self._test_num_blocks): a.write(self._test_block) test_crc = crc32(self._test_block, test_crc) # Read the checksum from c cChecksum = int(droputils.allDropContents(c)) self.assertNotEqual(a.checksum, 0) self.assertEqual(a.checksum, test_crc) self.assertEqual(cChecksum, test_crc)
def test_simpleCopy(self): """ Simple test for a dockerized application. It copies the contents of one file into another via the command-line cp utility. It then checks that the contents of the target DROP are correct, and that the target file is actually owned by our process. The test will not run if a docker daemon cannot be contacted though; this is to avoid failures in machines that don't have a docker service running. """ try: AutoVersionClient().close() except DockerException: warnings.warn( "Cannot contact the Docker daemon, skipping docker tests") return a = FileDROP('a', 'a') b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0') c = FileDROP('c', 'c') b.addInput(a) b.addOutput(c) # Random data so we always check different contents data = os.urandom(10) with DROPWaiterCtx(self, c, 100): a.write(data) a.setCompleted() self.assertEqual(data, droputils.allDropContents(c)) # We own the file, not root uid = os.getuid() self.assertEqual(uid, os.stat(c.path).st_uid)
def test_simpleCopy(self): """ Simple test for a dockerized application. It copies the contents of one file into another via the command-line cp utility. It then checks that the contents of the target DROP are correct, and that the target file is actually owned by our process. The test will not run if a docker daemon cannot be contacted though; this is to avoid failures in machines that don't have a docker service running. """ try: AutoVersionClient() except DockerException: warnings.warn("Cannot contact the Docker daemon, skipping docker tests") return a = FileDROP('a', 'a') b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0') c = FileDROP('c', 'c') b.addInput(a) b.addOutput(c) # Random data so we always check different contents data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)]) with DROPWaiterCtx(self, c, 100): a.write(data) a.setCompleted() self.assertEquals(data, droputils.allDropContents(c)) # We own the file, not root uid = os.getuid() self.assertEquals(uid, os.stat(c.path).st_uid)
def test_simple_chain(self): """ Simple test that creates a pipeline-like chain of commands. In this case we simulate a pipeline that does this, holding each intermediate result in memory: cat someFile | grep 'a' | sort | rev """ class GrepResult(BarrierAppDROP): def initialize(self, **kwargs): super(GrepResult, self).initialize(**kwargs) self._substring = kwargs["substring"] def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = StringIO(droputils.allDropContents(drop)).readlines() for line in allLines: if self._substring in line: output.write(line) class SortResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] sortedLines = StringIO(droputils.allDropContents(drop)).readlines() sortedLines.sort() for line in sortedLines: output.write(line) class RevResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = StringIO(droputils.allDropContents(drop)).readlines() for line in allLines: buf = "" for c in line: if c == " " or c == "\n": output.write(buf[::-1]) output.write(c) buf = "" else: buf += c a = InMemoryDROP("oid:A", "uid:A") b = GrepResult("oid:B", "uid:B", substring="a") c = InMemoryDROP("oid:C", "uid:C") d = SortResult("oid:D", "uid:D") e = InMemoryDROP("oid:E", "uid:E") f = RevResult("oid:F", "oid:F") g = InMemoryDROP("oid:G", "uid:G") a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) e.addConsumer(f) f.addOutput(g) # Initial write contents = "first line\nwe have an a here\nand another one\nnoone knows me" cResExpected = "we have an a here\nand another one\n" eResExpected = "and another one\nwe have an a here\n" gResExpected = "dna rehtona eno\new evah na a ereh\n" with DROPWaiterCtx(self, g): a.write(contents) a.setCompleted() # Get intermediate and final results and compare actualRes = [] for i in [c, e, g]: actualRes.append(droputils.allDropContents(i)) map(lambda x, y: self.assertEquals(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
def test_join(self): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ filelen = self._test_drop_sz * ONE_MB # create file data objects a1 = FileDROP("oid:A1", "uid:A1", expectedSize=filelen) a2 = FileDROP("oid:A2", "uid:A2", expectedSize=filelen) a3 = FileDROP("oid:A3", "uid:A3", expectedSize=filelen) # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum("oid:B1", "uid:B1") b2 = SumupContainerChecksum("oid:B2", "uid:B2") b3 = SumupContainerChecksum("oid:B3", "uid:B3") c1 = InMemoryDROP("oid:C1", "uid:C1") c2 = InMemoryDROP("oid:C2", "uid:C2") c3 = InMemoryDROP("oid:C3", "uid:C3") # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum("oid:D", "uid:D") e = InMemoryDROP("oid:E", "uid:E") # Wire together dropAList = [a1, a2, a3] dropBList = [b1, b2, b3] dropCList = [c1, c2, c3] for dropA, dropB in map(lambda a, b: (a, b), dropAList, dropBList): dropA.addConsumer(dropB) for dropB, dropC in map(lambda b, c: (b, c), dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): for dropA in dropAList: # this should be parallel for for _ in range(self._test_num_blocks): dropA.write(self._test_block) # All DROPs are completed now that the chain executed correctly for drop in dropAList + dropBList + dropCList: self.assertEqual(drop.status, DROPStates.COMPLETED) # The results we want to compare sum_crc = c1.checksum + c2.checksum + c3.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEquals(sum_crc, 0) self.assertEquals(sum_crc, dropEData)
def test_simple_chain(self): ''' Simple test that creates a pipeline-like chain of commands. In this case we simulate a pipeline that does this, holding each intermediate result in memory: cat someFile | grep 'a' | sort | rev ''' class GrepResult(BarrierAppDROP): def initialize(self, **kwargs): super(GrepResult, self).initialize(**kwargs) self._substring = six.b(kwargs['substring']) def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = BytesIO(droputils.allDropContents(drop)).readlines() for line in allLines: if self._substring in line: output.write(line) class SortResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] sortedLines = BytesIO(droputils.allDropContents(drop)).readlines() sortedLines.sort() for line in sortedLines: output.write(line) class RevResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] allbytes = droputils.allDropContents(drop) buf = bytearray() for c in allbytes: if c == six.b(' ') or c == six.b('\n'): output.write(buf[::-1]) output.write(c) buf = bytearray() else: buf.append(c) a = InMemoryDROP('oid:A', 'uid:A') b = GrepResult('oid:B', 'uid:B', substring="a") c = InMemoryDROP('oid:C', 'uid:C') d = SortResult('oid:D', 'uid:D') e = InMemoryDROP('oid:E', 'uid:E') f = RevResult('oid:F', 'oid:F') g = InMemoryDROP('oid:G', 'uid:G') a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) e.addConsumer(f) f.addOutput(g) # Initial write contents = "first line\nwe have an a here\nand another one\nnoone knows me" cResExpected = "we have an a here\nand another one\n" eResExpected = "and another one\nwe have an a here\n" gResExpected = "dna rehtona eno\new evah na a ereh\n" with DROPWaiterCtx(self, g): a.write(contents) a.setCompleted() # Get intermediate and final results and compare actualRes = [] for i in [c, e, g]: actualRes.append(droputils.allDropContents(i)) map(lambda x, y: self.assertEqual(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
def branch_failure(self, tooManyFailures): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ #create file data objects a1 = InMemoryDROP('oid:A1', 'uid:A1') a2 = InMemoryDROP('oid:A2', 'uid:A2') a3 = InMemoryDROP('oid:A3', 'uid:A3') # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum('oid:B1', 'uid:B1') b2 = SumupContainerChecksum('oid:B2', 'uid:B2') b3 = SumupContainerChecksum('oid:B3', 'uid:B3') c1 = InMemoryDROP('oid:C1', 'uid:C1') c2 = InMemoryDROP('oid:C2', 'uid:C2') c3 = InMemoryDROP('oid:C3', 'uid:C3') # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum('oid:D', 'uid:D', input_error_threshold = 33) e = InMemoryDROP('oid:E', 'uid:E') # Wire together dropAList = [a1,a2,a3] dropBList = [b1,b2,b3] dropCList = [c1,c2,c3] for dropA,dropB in zip(dropAList, dropBList): dropA.addConsumer(dropB) for dropB,dropC in zip(dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): #for dropA in dropAList: # this should be parallel for a1.write(' '); a1.setCompleted() if tooManyFailures: a2.setError() else: a2.write(' '); a2.setCompleted() a3.setError() if tooManyFailures: completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1] errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e] else: completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e] errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:] for drop in completedDrops: self.assertEqual(drop.status, DROPStates.COMPLETED) for drop in errorDrops: self.assertEqual(drop.status, DROPStates.ERROR) # The results we want to compare # (only in case that at least two branches executed) if not tooManyFailures: sum_crc = c1.checksum + c2.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEqual(sum_crc, 0) self.assertEqual(sum_crc, dropEData)
def test_join(self): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ filelen = self._test_drop_sz * ONE_MB #create file data objects a1 = FileDROP('oid:A1', 'uid:A1', expectedSize=filelen) a2 = FileDROP('oid:A2', 'uid:A2', expectedSize=filelen) a3 = FileDROP('oid:A3', 'uid:A3', expectedSize=filelen) # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum('oid:B1', 'uid:B1') b2 = SumupContainerChecksum('oid:B2', 'uid:B2') b3 = SumupContainerChecksum('oid:B3', 'uid:B3') c1 = InMemoryDROP('oid:C1', 'uid:C1') c2 = InMemoryDROP('oid:C2', 'uid:C2') c3 = InMemoryDROP('oid:C3', 'uid:C3') # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum('oid:D', 'uid:D') e = InMemoryDROP('oid:E', 'uid:E') # Wire together dropAList = [a1,a2,a3] dropBList = [b1,b2,b3] dropCList = [c1,c2,c3] for dropA,dropB in map(lambda a,b: (a,b), dropAList, dropBList): dropA.addConsumer(dropB) for dropB,dropC in map(lambda b,c: (b,c), dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): for dropA in dropAList: # this should be parallel for for _ in range(self._test_num_blocks): dropA.write(self._test_block) # All DROPs are completed now that the chain executed correctly for drop in dropAList + dropBList + dropCList: self.assertEqual(drop.status, DROPStates.COMPLETED) # The results we want to compare sum_crc = c1.checksum + c2.checksum + c3.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEqual(sum_crc, 0) self.assertEqual(sum_crc, dropEData)
def run(self): inputDrop = self.inputs[0] output = self.outputs[0] howMany = int(droputils.allDropContents(inputDrop)) for i in range(howMany): output.write(str(i) + " ")
def run(self): inputDrop = self.inputs[0] output = self.outputs[0] howMany = int(droputils.allDropContents(inputDrop)) for i in xrange(howMany): output.write(str(i) + " ")
def test_runGraphSeveralDropsPerDM(self): """ A test that creates several DROPs in two different DMs and runs the graph. The graph looks like this DM #1 DM #2 =================== ================ | A --> C --> D --|----|-| | | | | |--> E --> F | | B --------------|----|-| | =================== ================ :see: `self.test_runGraphSingleDOPerDOM` """ dm1, dm2 = [self._start_dm() for _ in range(2)] sessionId = 's1' g1 = [{ "oid": "A", "type": "plain", "storage": "memory", "consumers": ["C"] }, { "oid": "B", "type": "plain", "storage": "memory" }, { "oid": "C", "type": "app", "app": "dfms.apps.crc.CRCApp" }, { "oid": "D", "type": "plain", "storage": "memory", "producers": ["C"] }] g2 = [{ "oid": "E", "type": "app", "app": "test.test_drop.SumupContainerChecksum" }, { "oid": "F", "type": "plain", "storage": "memory", "producers": ["E"] }] rels = [ DROPRel('D', DROPLinkType.INPUT, 'E'), DROPRel('B', DROPLinkType.INPUT, 'E') ] quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels}) quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels}) self.assertEqual(4, len(dm1._sessions[sessionId].drops)) self.assertEqual(2, len(dm2._sessions[sessionId].drops)) # Run! The sole fact that this doesn't throw exceptions is already # a good proof that everything is working as expected a, b, c, d = [ dm1._sessions[sessionId].drops[x] for x in ('A', 'B', 'C', 'D') ] e, f = [dm2._sessions[sessionId].drops[x] for x in ('E', 'F')] with droputils.DROPWaiterCtx(self, f, 5): a.write('a') a.setCompleted() b.write('a') b.setCompleted() for drop in a, b, c, d, e, f: self.assertEqual(DROPStates.COMPLETED, drop.status, "DROP %s is not COMPLETED" % (drop.uid)) self.assertEqual(a.checksum, int(droputils.allDropContents(d))) self.assertEqual(b.checksum + d.checksum, int(droputils.allDropContents(f))) dm1.destroySession(sessionId) dm2.destroySession(sessionId)
def test_runGraphSeveralDropsPerDM(self): """ A test that creates several DROPs in two different DMs and runs the graph. The graph looks like this DM #1 DM #2 =================== ================ | A --> C --> D --|----|-| | | | | |--> E --> F | | B --------------|----|-| | =================== ================ :see: `self.test_runGraphSingleDOPerDOM` """ dm1 = NodeManager(useDLM=False) dm2 = NodeManager(useDLM=False) sessionId = 's1' g1 = [{"oid":"A", "type":"plain", "storage": "memory", "consumers":["C"]}, {"oid":"B", "type":"plain", "storage": "memory"}, {"oid":"C", "type":"app", "app":"dfms.apps.crc.CRCApp"}, {"oid":"D", "type":"plain", "storage": "memory", "producers": ["C"]}] g2 = [{"oid":"E", "type":"app", "app":"test.test_drop.SumupContainerChecksum"}, {"oid":"F", "type":"plain", "storage": "memory", "producers":["E"]}] uris1 = dm1.quickDeploy(sessionId, g1) uris2 = dm2.quickDeploy(sessionId, g2) self.assertEquals(4, len(uris1)) self.assertEquals(2, len(uris2)) # We externally wire the Proxy objects to establish the inter-DM # relationships a = Pyro4.Proxy(uris1['A']) b = Pyro4.Proxy(uris1['B']) c = Pyro4.Proxy(uris1['C']) d = Pyro4.Proxy(uris1['D']) e = Pyro4.Proxy(uris2['E']) f = Pyro4.Proxy(uris2['F']) for drop,uid in [(a,'A'),(b,'B'),(c,'C'),(d,'D'),(e,'E'),(f,'F')]: self.assertEquals(uid, drop.uid, "Proxy is not the DROP we think should be (assumed: %s/ actual: %s)" % (uid, drop.uid)) e.addInput(d) e.addInput(b) # Run! The sole fact that this doesn't throw exceptions is already # a good proof that everything is working as expected with droputils.EvtConsumerProxyCtx(self, f, 5): a.write('a') a.setCompleted() b.write('a') b.setCompleted() for drop in a,b,c,d,e,f: self.assertEquals(DROPStates.COMPLETED, drop.status, "DROP %s is not COMPLETED" % (drop.uid)) self.assertEquals(a.checksum, int(droputils.allDropContents(d))) self.assertEquals(b.checksum + d.checksum, int(droputils.allDropContents(f))) for dropProxy in a,b,c,d,e,f: dropProxy._pyroRelease() dm1.destroySession(sessionId) dm2.destroySession(sessionId)
def branch_failure(self, tooManyFailures): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ # create file data objects a1 = InMemoryDROP("oid:A1", "uid:A1") a2 = InMemoryDROP("oid:A2", "uid:A2") a3 = InMemoryDROP("oid:A3", "uid:A3") # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum("oid:B1", "uid:B1") b2 = SumupContainerChecksum("oid:B2", "uid:B2") b3 = SumupContainerChecksum("oid:B3", "uid:B3") c1 = InMemoryDROP("oid:C1", "uid:C1") c2 = InMemoryDROP("oid:C2", "uid:C2") c3 = InMemoryDROP("oid:C3", "uid:C3") # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum("oid:D", "uid:D", input_error_threshold=33) e = InMemoryDROP("oid:E", "uid:E") # Wire together dropAList = [a1, a2, a3] dropBList = [b1, b2, b3] dropCList = [c1, c2, c3] for dropA, dropB in map(lambda a, b: (a, b), dropAList, dropBList): dropA.addConsumer(dropB) for dropB, dropC in map(lambda b, c: (b, c), dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): # for dropA in dropAList: # this should be parallel for a1.write(" ") a1.setCompleted() if tooManyFailures: a2.setError() else: a2.write(" ") a2.setCompleted() a3.setError() if tooManyFailures: completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1] errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e] else: completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e] errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:] for drop in completedDrops: self.assertEquals(drop.status, DROPStates.COMPLETED) for drop in errorDrops: self.assertEquals(drop.status, DROPStates.ERROR) # The results we want to compare # (only in case that at least two branches executed) if not tooManyFailures: sum_crc = c1.checksum + c2.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEquals(sum_crc, 0) self.assertEquals(sum_crc, dropEData)