def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastByte = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastByte = data[-1:]
outputDrop.write(self._lastByte)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP('a', 'a')
b = LastCharWriterApp('b', 'b')
c = SumupContainerChecksum('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, a.addConsumer, b)
self.assertRaises(Exception, a.addStreamingConsumer, c)
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastByte):
self.assertEqual(aStatus, a.status)
self.assertEqual(dStatus, d.status)
self.assertEqual(eStatus, e.status)
if lastByte is not None:
self.assertEqual(lastByte, b._lastByte)
checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write(b'abcde')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, b'e')
a.write(b'fghij')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, b'j')
a.write(b'k')
with DROPWaiterCtx(self, [d, e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, b'k')
self.assertEqual(b'ejk', droputils.allDropContents(d))
def _test_graphExecutionDriver(self, mode):
"""
A small test to check that DROPs executions can be driven externally if
required, and not always internally by themselves
"""
a = InMemoryDROP("a", "a", executionMode=mode, expectedSize=1)
b = SumupContainerChecksum("b", "b")
c = InMemoryDROP("c", "c")
a.addConsumer(b)
c.addProducer(b)
# Write and check
dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c]
with DROPWaiterCtx(self, dropsToWaitFor):
a.write(b"1")
if mode == ExecutionMode.EXTERNAL:
# b hasn't been triggered
self.assertEqual(c.status, DROPStates.INITIALIZED)
self.assertEqual(b.status, DROPStates.INITIALIZED)
self.assertEqual(b.execStatus, AppDROPStates.NOT_RUN)
# Now let b consume a
with DROPWaiterCtx(self, [c]):
b.dropCompleted("a", DROPStates.COMPLETED)
self.assertEqual(c.status, DROPStates.COMPLETED)
elif mode == ExecutionMode.DROP:
# b is already done
self.assertEqual(c.status, DROPStates.COMPLETED)
def test_app_multiple_outputs(self):
"""
A small method that tests that the AppDROPs writing to two
different DROPs outputs works
The graph constructed by this example looks as follow:
|--> E
A --> B --> C --> D --|
|--> F
Here B and D are an AppDROPs, with D writing to two DROPs
outputs (E and F) and reading from C. C, in turn, is written by B, which
in turns reads the data from A
"""
# This is used as "B"
class NumberWriterApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in range(howMany):
output.write(six.b(str(i)) + b" ")
# This is used as "D"
class OddAndEvenContainerApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + b" ")
# Create DROPs
a = InMemoryDROP('oid:A', 'uid:A')
b = NumberWriterApp('oid:B', 'uid:B')
c = InMemoryDROP('oid:A', 'uid:A')
d = OddAndEvenContainerApp('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
f = InMemoryDROP('oid:F', 'uid:F')
# Wire them together
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
d.addOutput(f)
# Start the execution
with DROPWaiterCtx(self, [e, f]):
a.write(b'20')
a.setCompleted()
# Check the final results are correct
for drop in [a, b, c, d, e]:
self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
self.assertEqual(b"0 2 4 6 8 10 12 14 16 18", droputils.allDropContents(e).strip())
self.assertEqual(b"1 3 5 7 9 11 13 15 17 19", droputils.allDropContents(f).strip())
def _createGraph(self):
"""
Creates the following graph of DROPs:
A |--> B ----> D --> G --> I --|
|--> C -|--> E --------------|-> H --> J
|--> F
B, C, G and H are AppDOs. The names have been given in breadth-first
order (although H has a dependency on I)
"""
a = InMemoryDROP("a", "a")
b = BarrierAppDROP("b", "b")
c = BarrierAppDROP("c", "c")
d = InMemoryDROP("d", "d")
e = InMemoryDROP("e", "e")
f = InMemoryDROP("f", "f")
g = BarrierAppDROP("g", "g")
h = BarrierAppDROP("h", "h")
i = InMemoryDROP("i", "i")
j = InMemoryDROP("j", "j")
a.addConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
c.addOutput(f)
d.addConsumer(g)
e.addConsumer(h)
g.addOutput(i)
i.addConsumer(h)
h.addOutput(j)
return a, b, c, d, e, f, g, h, i, j
def _createGraph(self):
"""
Creates the following graph of DROPs:
A |--> B ----> D --> G --> I --|
|--> C -|--> E --------------|-> H --> J
|--> F
B, C, G and H are AppDOs. The names have been given in breadth-first
order (although H has a dependency on I)
"""
a = InMemoryDROP('a', 'a')
b = BarrierAppDROP('b', 'b')
c = BarrierAppDROP('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
f = InMemoryDROP('f', 'f')
g = BarrierAppDROP('g', 'g')
h = BarrierAppDROP('h', 'h')
i = InMemoryDROP('i', 'i')
j = InMemoryDROP('j', 'j')
a.addConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
c.addOutput(f)
d.addConsumer(g)
e.addConsumer(h)
g.addOutput(i)
i.addConsumer(h)
h.addOutput(j)
return a, b, c, d, e, f, g, h, i, j
def _test_socket_listener(self, **kwargs):
'''
A simple test to check that SocketListenerApps are indeed working as
expected; that is, they write the data they receive into their output,
and finish when the connection is closed from the client side
The data flow diagram looks like this:
A --> B --> C --> D
'''
host = 'localhost'
port = 9933
data = os.urandom(1025)
a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port, **kwargs)
b = InMemoryDROP('oid:B', 'uid:B')
c = SumupContainerChecksum('oid:C', 'uid:C')
d = InMemoryDROP('oid:D', 'uid:D')
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
# Create the socket, write, and close the connection, allowing
# A to move to COMPLETED
with DROPWaiterCtx(self, d, 3): # That's plenty of time
a.async_execute()
utils.write_to(host, port, data, 1)
for drop in [a, b, c, d]:
self.assertEqual(DROPStates.COMPLETED, drop.status)
# Our expectations are fulfilled!
bContents = droputils.allDropContents(b)
dContents = int(droputils.allDropContents(d))
self.assertEqual(data, bContents)
self.assertEqual(crc32(data, 0), dContents)
def test_simple_chain(self):
"""
Simple test that creates a pipeline-like chain of commands.
In this case we simulate a pipeline that does this, holding
each intermediate result in memory:
cat someFile | grep 'a' | sort | rev
"""
class GrepResult(BarrierAppDROP):
def initialize(self, **kwargs):
super(GrepResult, self).initialize(**kwargs)
self._substring = kwargs["substring"]
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = io.BytesIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
class SortResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = io.BytesIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
class RevResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allbytes = droputils.allDropContents(drop)
buf = io.BytesIO()
for c in allbytes:
if c == b" " or c == b"\n":
output.write(buf.getvalue()[::-1])
output.write(bytes([c]))
buf = io.BytesIO()
else:
buf.write(bytes([c]))
a = InMemoryDROP("oid:A", "uid:A")
b = GrepResult("oid:B", "uid:B", substring=b"a")
c = InMemoryDROP("oid:C", "uid:C")
d = SortResult("oid:D", "uid:D")
e = InMemoryDROP("oid:E", "uid:E")
f = RevResult("oid:F", "oid:F")
g = InMemoryDROP("oid:G", "uid:G")
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
e.addConsumer(f)
f.addOutput(g)
# Initial write
contents = b"first line\nwe have an a here\nand another one\nnoone knows me"
cResExpected = b"we have an a here\nand another one\n"
eResExpected = b"and another one\nwe have an a here\n"
gResExpected = b"dna rehtona eno\new evah na a ereh\n"
with DROPWaiterCtx(self, g):
a.write(contents)
a.setCompleted()
# Get intermediate and final results and compare
actualRes = []
for i in [c, e, g]:
actualRes.append(droputils.allDropContents(i))
map(
lambda x, y: self.assertEqual(x, y),
[cResExpected, eResExpected, gResExpected],
actualRes,
)