def test_objectAsNormalAndStreamingInput(self): """ A test that checks that a DROP can act as normal and streaming input of different AppDROPs at the same time. We use the following graph: A --|--> B --> D |--> C --> E Here B uses A as a streaming input, while C uses it as a normal input """ class LastCharWriterApp(AppDROP): def initialize(self, **kwargs): super(LastCharWriterApp, self).initialize(**kwargs) self._lastByte = None def dataWritten(self, uid, data): self.execStatus = AppDROPStates.RUNNING outputDrop = self.outputs[0] self._lastByte = data[-1:] outputDrop.write(self._lastByte) def dropCompleted(self, uid, status): self.execStatus = AppDROPStates.FINISHED self._notifyAppIsFinished() a = InMemoryDROP('a', 'a') b = LastCharWriterApp('b', 'b') c = SumupContainerChecksum('c', 'c') d = InMemoryDROP('d', 'd') e = InMemoryDROP('e', 'e') a.addStreamingConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) # Consumer cannot be normal and streaming at the same time self.assertRaises(Exception, a.addConsumer, b) self.assertRaises(Exception, a.addStreamingConsumer, c) # Write a little, then check the consumers def checkDropStates(aStatus, dStatus, eStatus, lastByte): self.assertEqual(aStatus, a.status) self.assertEqual(dStatus, d.status) self.assertEqual(eStatus, e.status) if lastByte is not None: self.assertEqual(lastByte, b._lastByte) checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None) a.write(b'abcde') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, b'e') a.write(b'fghij') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, b'j') a.write(b'k') with DROPWaiterCtx(self, [d, e]): a.setCompleted() checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, b'k') self.assertEqual(b'ejk', droputils.allDropContents(d))
def _test_graphExecutionDriver(self, mode): """ A small test to check that DROPs executions can be driven externally if required, and not always internally by themselves """ a = InMemoryDROP("a", "a", executionMode=mode, expectedSize=1) b = SumupContainerChecksum("b", "b") c = InMemoryDROP("c", "c") a.addConsumer(b) c.addProducer(b) # Write and check dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c] with DROPWaiterCtx(self, dropsToWaitFor): a.write(b"1") if mode == ExecutionMode.EXTERNAL: # b hasn't been triggered self.assertEqual(c.status, DROPStates.INITIALIZED) self.assertEqual(b.status, DROPStates.INITIALIZED) self.assertEqual(b.execStatus, AppDROPStates.NOT_RUN) # Now let b consume a with DROPWaiterCtx(self, [c]): b.dropCompleted("a", DROPStates.COMPLETED) self.assertEqual(c.status, DROPStates.COMPLETED) elif mode == ExecutionMode.DROP: # b is already done self.assertEqual(c.status, DROPStates.COMPLETED)
def test_app_multiple_outputs(self): """ A small method that tests that the AppDROPs writing to two different DROPs outputs works The graph constructed by this example looks as follow: |--> E A --> B --> C --> D --| |--> F Here B and D are an AppDROPs, with D writing to two DROPs outputs (E and F) and reading from C. C, in turn, is written by B, which in turns reads the data from A """ # This is used as "B" class NumberWriterApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] output = self.outputs[0] howMany = int(droputils.allDropContents(inputDrop)) for i in range(howMany): output.write(six.b(str(i)) + b" ") # This is used as "D" class OddAndEvenContainerApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] outputs = self.outputs numbers = droputils.allDropContents(inputDrop).strip().split() for n in numbers: outputs[int(n) % 2].write(n + b" ") # Create DROPs a = InMemoryDROP('oid:A', 'uid:A') b = NumberWriterApp('oid:B', 'uid:B') c = InMemoryDROP('oid:A', 'uid:A') d = OddAndEvenContainerApp('oid:D', 'uid:D') e = InMemoryDROP('oid:E', 'uid:E') f = InMemoryDROP('oid:F', 'uid:F') # Wire them together a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) d.addOutput(f) # Start the execution with DROPWaiterCtx(self, [e, f]): a.write(b'20') a.setCompleted() # Check the final results are correct for drop in [a, b, c, d, e]: self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop)) self.assertEqual(b"0 2 4 6 8 10 12 14 16 18", droputils.allDropContents(e).strip()) self.assertEqual(b"1 3 5 7 9 11 13 15 17 19", droputils.allDropContents(f).strip())
def _createGraph(self): """ Creates the following graph of DROPs: A |--> B ----> D --> G --> I --| |--> C -|--> E --------------|-> H --> J |--> F B, C, G and H are AppDOs. The names have been given in breadth-first order (although H has a dependency on I) """ a = InMemoryDROP("a", "a") b = BarrierAppDROP("b", "b") c = BarrierAppDROP("c", "c") d = InMemoryDROP("d", "d") e = InMemoryDROP("e", "e") f = InMemoryDROP("f", "f") g = BarrierAppDROP("g", "g") h = BarrierAppDROP("h", "h") i = InMemoryDROP("i", "i") j = InMemoryDROP("j", "j") a.addConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) c.addOutput(f) d.addConsumer(g) e.addConsumer(h) g.addOutput(i) i.addConsumer(h) h.addOutput(j) return a, b, c, d, e, f, g, h, i, j
def _createGraph(self): """ Creates the following graph of DROPs: A |--> B ----> D --> G --> I --| |--> C -|--> E --------------|-> H --> J |--> F B, C, G and H are AppDOs. The names have been given in breadth-first order (although H has a dependency on I) """ a = InMemoryDROP('a', 'a') b = BarrierAppDROP('b', 'b') c = BarrierAppDROP('c', 'c') d = InMemoryDROP('d', 'd') e = InMemoryDROP('e', 'e') f = InMemoryDROP('f', 'f') g = BarrierAppDROP('g', 'g') h = BarrierAppDROP('h', 'h') i = InMemoryDROP('i', 'i') j = InMemoryDROP('j', 'j') a.addConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) c.addOutput(f) d.addConsumer(g) e.addConsumer(h) g.addOutput(i) i.addConsumer(h) h.addOutput(j) return a, b, c, d, e, f, g, h, i, j
def _test_socket_listener(self, **kwargs): ''' A simple test to check that SocketListenerApps are indeed working as expected; that is, they write the data they receive into their output, and finish when the connection is closed from the client side The data flow diagram looks like this: A --> B --> C --> D ''' host = 'localhost' port = 9933 data = os.urandom(1025) a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port, **kwargs) b = InMemoryDROP('oid:B', 'uid:B') c = SumupContainerChecksum('oid:C', 'uid:C') d = InMemoryDROP('oid:D', 'uid:D') a.addOutput(b) b.addConsumer(c) c.addOutput(d) # Create the socket, write, and close the connection, allowing # A to move to COMPLETED with DROPWaiterCtx(self, d, 3): # That's plenty of time a.async_execute() utils.write_to(host, port, data, 1) for drop in [a, b, c, d]: self.assertEqual(DROPStates.COMPLETED, drop.status) # Our expectations are fulfilled! bContents = droputils.allDropContents(b) dContents = int(droputils.allDropContents(d)) self.assertEqual(data, bContents) self.assertEqual(crc32(data, 0), dContents)
def test_simple_chain(self): """ Simple test that creates a pipeline-like chain of commands. In this case we simulate a pipeline that does this, holding each intermediate result in memory: cat someFile | grep 'a' | sort | rev """ class GrepResult(BarrierAppDROP): def initialize(self, **kwargs): super(GrepResult, self).initialize(**kwargs) self._substring = kwargs["substring"] def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = io.BytesIO(droputils.allDropContents(drop)).readlines() for line in allLines: if self._substring in line: output.write(line) class SortResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] sortedLines = io.BytesIO(droputils.allDropContents(drop)).readlines() sortedLines.sort() for line in sortedLines: output.write(line) class RevResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] allbytes = droputils.allDropContents(drop) buf = io.BytesIO() for c in allbytes: if c == b" " or c == b"\n": output.write(buf.getvalue()[::-1]) output.write(bytes([c])) buf = io.BytesIO() else: buf.write(bytes([c])) a = InMemoryDROP("oid:A", "uid:A") b = GrepResult("oid:B", "uid:B", substring=b"a") c = InMemoryDROP("oid:C", "uid:C") d = SortResult("oid:D", "uid:D") e = InMemoryDROP("oid:E", "uid:E") f = RevResult("oid:F", "oid:F") g = InMemoryDROP("oid:G", "uid:G") a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) e.addConsumer(f) f.addOutput(g) # Initial write contents = b"first line\nwe have an a here\nand another one\nnoone knows me" cResExpected = b"we have an a here\nand another one\n" eResExpected = b"and another one\nwe have an a here\n" gResExpected = b"dna rehtona eno\new evah na a ereh\n" with DROPWaiterCtx(self, g): a.write(contents) a.setCompleted() # Get intermediate and final results and compare actualRes = [] for i in [c, e, g]: actualRes.append(droputils.allDropContents(i)) map( lambda x, y: self.assertEqual(x, y), [cResExpected, eResExpected, gResExpected], actualRes, )