def test_objectAsNormalAndStreamingInput(self): """ A test that checks that a DROP can act as normal and streaming input of different AppDROPs at the same time. We use the following graph: A --|--> B --> D |--> C --> E Here B uses A as a streaming input, while C uses it as a normal input """ class LastCharWriterApp(AppDROP): def initialize(self, **kwargs): super(LastCharWriterApp, self).initialize(**kwargs) self._lastChar = None def dataWritten(self, uid, data): self.execStatus = AppDROPStates.RUNNING outputDrop = self.outputs[0] self._lastChar = data[-1] outputDrop.write(self._lastChar) def dropCompleted(self, uid, status): self.execStatus = AppDROPStates.FINISHED self._notifyAppIsFinished() a = InMemoryDROP("a", "a") b = LastCharWriterApp("b", "b") c = SumupContainerChecksum("c", "c") d = InMemoryDROP("d", "d") e = InMemoryDROP("e", "e") a.addStreamingConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) # Consumer cannot be normal and streaming at the same time self.assertRaises(Exception, lambda: a.addConsumer(b)) self.assertRaises(Exception, lambda: a.addStreamingConsumer(c)) # Write a little, then check the consumers def checkDropStates(aStatus, dStatus, eStatus, lastChar): self.assertEquals(aStatus, a.status) self.assertEquals(dStatus, d.status) self.assertEquals(eStatus, e.status) self.assertEquals(lastChar, b._lastChar) checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None) a.write("abcde") checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "e") a.write("fghij") checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "j") a.write("k") with DROPWaiterCtx(self, [d, e]): a.setCompleted() checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, "k") self.assertEquals("ejk", droputils.allDropContents(d))
def test_objectAsNormalAndStreamingInput(self): """ A test that checks that a DROP can act as normal and streaming input of different AppDROPs at the same time. We use the following graph: A --|--> B --> D |--> C --> E Here B uses A as a streaming input, while C uses it as a normal input """ class LastCharWriterApp(AppDROP): def initialize(self, **kwargs): super(LastCharWriterApp, self).initialize(**kwargs) self._lastByte = None def dataWritten(self, uid, data): self.execStatus = AppDROPStates.RUNNING outputDrop = self.outputs[0] self._lastByte = six.indexbytes(data, -1) outputDrop.write(self._lastByte) def dropCompleted(self, uid, status): self.execStatus = AppDROPStates.FINISHED self._notifyAppIsFinished() a = InMemoryDROP('a', 'a') b = LastCharWriterApp('b', 'b') c = SumupContainerChecksum('c', 'c') d = InMemoryDROP('d', 'd') e = InMemoryDROP('e', 'e') a.addStreamingConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) # Consumer cannot be normal and streaming at the same time self.assertRaises(Exception, a.addConsumer, b) self.assertRaises(Exception, a.addStreamingConsumer, c) # Write a little, then check the consumers def checkDropStates(aStatus, dStatus, eStatus, lastByte): self.assertEqual(aStatus, a.status) self.assertEqual(dStatus, d.status) self.assertEqual(eStatus, e.status) if lastByte is not None: self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte)) checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None) a.write('abcde') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e') a.write('fghij') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j') a.write('k') with DROPWaiterCtx(self, [d,e]): a.setCompleted() checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k') self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
def test_two_simultaneous_pipes(self): """ A more complicated test where three bash applications run at the same time. The first streams its output to the second one, while the second one streams *its* output to the third one. ------------- -------------- ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F | | echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | | -----*------- -------------- ----*--*----- -------------- -----*------- ---------- | | | | \-------------|named-pipe|----------\ \-----------|named-pipe|-----------/ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by "cat" (BashApp C). The printed results (a copy of the original) are streamed through D and read by "sort" (BashApp E), which writes the output to F. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp('a', 'a', command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP('b', 'b') c = StreamingInputOutputBashApp('c', 'c', command="cat") d = InMemoryDROP('d', 'd') e = StreamingInputBashApp('e', 'e', command="sort -n > %o0") f = FileDROP('f', 'f', filepath=output_fname) a.addOutput(b) b.addStreamingConsumer(c) c.addOutput(d) d.addStreamingConsumer(e) e.addOutput(f) # Let's fire the app with DROPWaiterCtx(self, f, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d, e, f): self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual([1, 2, 3, 4, 5], [ int(x) for x in droputils.allDropContents(f).strip().split(six.b('\n')) ]) # Clean up and go os.remove(output_fname)