def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastChar = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastChar = data[-1]
outputDrop.write(self._lastChar)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP("a", "a")
b = LastCharWriterApp("b", "b")
c = SumupContainerChecksum("c", "c")
d = InMemoryDROP("d", "d")
e = InMemoryDROP("e", "e")
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, lambda: a.addConsumer(b))
self.assertRaises(Exception, lambda: a.addStreamingConsumer(c))
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastChar):
self.assertEquals(aStatus, a.status)
self.assertEquals(dStatus, d.status)
self.assertEquals(eStatus, e.status)
self.assertEquals(lastChar, b._lastChar)
checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write("abcde")
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "e")
a.write("fghij")
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "j")
a.write("k")
with DROPWaiterCtx(self, [d, e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, "k")
self.assertEquals("ejk", droputils.allDropContents(d))
def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastByte = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastByte = six.indexbytes(data, -1)
outputDrop.write(self._lastByte)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP('a', 'a')
b = LastCharWriterApp('b', 'b')
c = SumupContainerChecksum('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, a.addConsumer, b)
self.assertRaises(Exception, a.addStreamingConsumer, c)
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastByte):
self.assertEqual(aStatus, a.status)
self.assertEqual(dStatus, d.status)
self.assertEqual(eStatus, e.status)
if lastByte is not None:
self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte))
checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write('abcde')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e')
a.write('fghij')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j')
a.write('k')
with DROPWaiterCtx(self, [d,e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k')
self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
def test_two_simultaneous_pipes(self):
"""
A more complicated test where three bash applications run at the same
time. The first streams its output to the second one, while the second
one streams *its* output to the third one.
------------- -------------- ------------- -------------- ------------- ----------
| BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F |
| echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | |
-----*------- -------------- ----*--*----- -------------- -----*------- ----------
| | | |
\-------------|named-pipe|----------\ \-----------|named-pipe|-----------/
BashApp A writes "5 4 3 2 1" (each on a new line), which is read
by "cat" (BashApp C). The printed results (a copy of the original) are
streamed through D and read by "sort" (BashApp E), which writes the
output to F.
"""
output_fname = tempfile.mktemp()
a = StreamingOutputBashApp('a',
'a',
command=r"echo -en '5\n4\n3\n2\n1'")
b = InMemoryDROP('b', 'b')
c = StreamingInputOutputBashApp('c', 'c', command="cat")
d = InMemoryDROP('d', 'd')
e = StreamingInputBashApp('e', 'e', command="sort -n > %o0")
f = FileDROP('f', 'f', filepath=output_fname)
a.addOutput(b)
b.addStreamingConsumer(c)
c.addOutput(d)
d.addStreamingConsumer(e)
e.addOutput(f)
# Let's fire the app
with DROPWaiterCtx(self, f, 2):
a.async_execute()
# The application executed, finished, and its output was recorded
for drop in (a, b, c, d, e, f):
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual([1, 2, 3, 4, 5], [
int(x)
for x in droputils.allDropContents(f).strip().split(six.b('\n'))
])
# Clean up and go
os.remove(output_fname)