def test_app_multiple_outputs(self):
"""
A small method that tests that the AppDROPs writing to two
different DROPs outputs works
The graph constructed by this example looks as follow:
|--> E
A --> B --> C --> D --|
|--> F
Here B and D are an AppDROPs, with D writing to two DROPs
outputs (E and F) and reading from C. C, in turn, is written by B, which
in turns reads the data from A
"""
# This is used as "B"
class NumberWriterApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in range(howMany):
output.write(str(i) + " ")
# This is used as "D"
class OddAndEvenContainerApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + six.b(" "))
# Create DROPs
a = InMemoryDROP('oid:A', 'uid:A')
b = NumberWriterApp('oid:B', 'uid:B')
c = InMemoryDROP('oid:A', 'uid:A')
d = OddAndEvenContainerApp('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
f = InMemoryDROP('oid:F', 'uid:F')
# Wire them together
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
d.addOutput(f)
# Start the execution
with DROPWaiterCtx(self, [e,f]):
a.write('20')
a.setCompleted()
# Check the final results are correct
for drop in [a,b,c,d,e]:
self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
self.assertEqual(six.b("0 2 4 6 8 10 12 14 16 18"), droputils.allDropContents(e).strip())
self.assertEqual(six.b("1 3 5 7 9 11 13 15 17 19"), droputils.allDropContents(f).strip())
def test_addDropInstances(self):
a1 = InMemoryDROP('a', 'a1')
a2 = InMemoryDROP('a', 'a2')
registry = RDBMSRegistry('sqlite3', DBFILE)
registry.addDrop(a1)
uids = registry.getDropUids(a1)
self.assertEqual(1, len(uids))
self.assertEqual('a1', uids[0])
registry.addDropInstance(a2)
uids = registry.getDropUids(a1)
uids.sort()
self.assertEqual(2, len(uids))
self.assertEqual('a1', uids[0])
self.assertEqual('a2', uids[1])
# Check accessing the database separately
conn = sqlite3.connect(DBFILE) # @UndefinedVariable
cur = conn.cursor()
cur.execute("SELECT uid FROM dfms_dropinstance WHERE oid = 'a'")
rows = cur.fetchall()
self.assertEqual(2, len(rows))
uids = [r[0] for r in rows]
uids.sort()
self.assertEqual(2, len(uids))
self.assertEqual('a1', uids[0])
self.assertEqual('a2', uids[1])
cur.close()
conn.close()
def _test_graphExecutionDriver(self, mode):
"""
A small test to check that DROPs executions can be driven externally if
required, and not always internally by themselves
"""
a = InMemoryDROP('a', 'a', executionMode=mode, expectedSize=1)
b = SumupContainerChecksum('b', 'b')
c = InMemoryDROP('c', 'c')
a.addConsumer(b)
c.addProducer(b)
# Write and check
dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c]
with DROPWaiterCtx(self, dropsToWaitFor):
a.write('1')
if mode == ExecutionMode.EXTERNAL:
# b hasn't been triggered
self.assertEqual(c.status, DROPStates.INITIALIZED)
self.assertEqual(b.status, DROPStates.INITIALIZED)
self.assertEqual(b.execStatus, AppDROPStates.NOT_RUN)
# Now let b consume a
with DROPWaiterCtx(self, [c]):
b.dropCompleted('a', DROPStates.COMPLETED)
self.assertEqual(c.status, DROPStates.COMPLETED)
elif mode == ExecutionMode.DROP:
# b is already done
self.assertEqual(c.status, DROPStates.COMPLETED)
def test_eager_inputFired_app(self):
"""
Tests that InputFiredApps works as expected
"""
# No n_effective_inputs given
self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a')
# Invalid values
self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=-2)
self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=0)
# More effective inputs than inputs
a = InMemoryDROP('b', 'b')
b = InputFiredAppDROP('a', 'a', n_effective_inputs=2)
b.addInput(a)
self.assertRaises(Exception, a.setCompleted)
# 2 effective inputs, 4 outputs. Trigger 2 inputs and make sure the
# app has run
a,b,c,d = [InMemoryDROP(str(i), str(i)) for i in range(4)]
e = InputFiredAppDROP('e', 'e', n_effective_inputs=2)
for x in a,b,c,d:
e.addInput(x)
with DROPWaiterCtx(self, e, 5):
a.setCompleted()
b.setCompleted()
self.assertEqual(AppDROPStates.FINISHED, e.execStatus)
self.assertEqual(DROPStates.COMPLETED, a.status)
self.assertEqual(DROPStates.COMPLETED, b.status)
self.assertEqual(DROPStates.INITIALIZED, c.status)
self.assertEqual(DROPStates.INITIALIZED, d.status)
def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastByte = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastByte = six.indexbytes(data, -1)
outputDrop.write(self._lastByte)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP('a', 'a')
b = LastCharWriterApp('b', 'b')
c = SumupContainerChecksum('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, a.addConsumer, b)
self.assertRaises(Exception, a.addStreamingConsumer, c)
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastByte):
self.assertEqual(aStatus, a.status)
self.assertEqual(dStatus, d.status)
self.assertEqual(eStatus, e.status)
if lastByte is not None:
self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte))
checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write('abcde')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e')
a.write('fghij')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j')
a.write('k')
with DROPWaiterCtx(self, [d,e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k')
self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
def test_errorState(self):
a = InMemoryDROP('a', 'a')
b = SumupContainerChecksum('b', 'b')
c = InMemoryDROP('c', 'c')
c.addProducer(b)
b.addInput(a)
a.setError()
self.assertEqual(DROPStates.ERROR, a.status)
self.assertEqual(DROPStates.ERROR, b.status)
self.assertEqual(DROPStates.ERROR, c.status)
def test_invalid(self):
# Shouldn't allow inputs
a = SocketListenerApp('a', 'a', port=1)
a.addOutput(InMemoryDROP('c', 'c'))
self.assertRaises(Exception, a.addInput, InMemoryDROP('b', 'b'))
self.assertRaises(Exception, a.addStreamingInput, InMemoryDROP('b', 'b'))
# Shouldn't be able to open ports <= 1024
a.execute()
self.assertEqual(a.status, DROPStates.ERROR)
def test_two_simultaneous_pipes(self):
"""
A more complicated test where three bash applications run at the same
time. The first streams its output to the second one, while the second
one streams *its* output to the third one.
------------- -------------- ------------- -------------- ------------- ----------
| BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F |
| echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | |
-----*------- -------------- ----*--*----- -------------- -----*------- ----------
| | | |
\-------------|named-pipe|----------\ \-----------|named-pipe|-----------/
BashApp A writes "5 4 3 2 1" (each on a new line), which is read
by "cat" (BashApp C). The printed results (a copy of the original) are
streamed through D and read by "sort" (BashApp E), which writes the
output to F.
"""
output_fname = tempfile.mktemp()
a = StreamingOutputBashApp('a',
'a',
command=r"echo -en '5\n4\n3\n2\n1'")
b = InMemoryDROP('b', 'b')
c = StreamingInputOutputBashApp('c', 'c', command="cat")
d = InMemoryDROP('d', 'd')
e = StreamingInputBashApp('e', 'e', command="sort -n > %o0")
f = FileDROP('f', 'f', filepath=output_fname)
a.addOutput(b)
b.addStreamingConsumer(c)
c.addOutput(d)
d.addStreamingConsumer(e)
e.addOutput(f)
# Let's fire the app
with DROPWaiterCtx(self, f, 2):
a.async_execute()
# The application executed, finished, and its output was recorded
for drop in (a, b, c, d, e, f):
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual([1, 2, 3, 4, 5], [
int(x)
for x in droputils.allDropContents(f).strip().split(six.b('\n'))
])
# Clean up and go
os.remove(output_fname)
def test_speadApp(self):
port = 1111
itemId = 0x2000
thread_pool = spead2.ThreadPool()
self._stream = spead2.send.UdpStream(
thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7))
a = SpeadReceiverApp('a', 'a', port=port, itemId=itemId)
b = InMemoryDROP('b', 'b')
a.addOutput(b)
size = 1024
threading.Thread(target=lambda: a.execute()).start()
time.sleep(1)
msg = os.urandom(size)
with DROPWaiterCtx(self, b, timeout=1):
ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48))
item = ig.add_item(itemId,
'main_data',
'a char array',
shape=(size, ),
format=[('c', 8)])
item.value = msg
self._stream.send_heap(ig.get_heap())
self._stream.send_heap(ig.get_end())
for drop in a, b:
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual(size, b.size)
self.assertEqual(msg, droputils.allDropContents(b))
def test_multipleProducers(self):
"""
A test that checks that multiple-producers correctly drive the state of
their shared output
"""
class App(BarrierAppDROP): pass
a,b,c,d,e = [App(chr(ord('A') + i), chr(ord('A') + i)) for i in range(5)]
f = InMemoryDROP('F', 'F')
for drop in a,b,c,d,e:
drop.addOutput(f)
self.assertEqual(DROPStates.INITIALIZED, f.status)
for drop in a,b,c,d,e:
self.assertEqual(AppDROPStates.NOT_RUN, drop.execStatus)
# Run the first 4 ones, F should still be in INITIALIZED
for drop in a,b,c,d:
drop.execute()
self.assertEqual(DROPStates.INITIALIZED, f.status)
self.assertEqual(AppDROPStates.NOT_RUN, e.execStatus)
for drop in a,b,c,d:
self.assertEqual(AppDROPStates.FINISHED, drop.execStatus)
# Run the final one, now F should be COMPLETED
e.execute()
self.assertEqual(DROPStates.COMPLETED, f.status)
for drop in a,b,c,d,e:
self.assertEqual(AppDROPStates.FINISHED, drop.execStatus)
def test_dropAccess(self):
a1 = InMemoryDROP('a', 'a1')
registry = RDBMSRegistry('sqlite3', DBFILE)
registry.addDrop(a1)
self.assertEqual(-1, registry.getLastAccess('a'))
registry.recordNewAccess('a')
self.assertNotEqual(-1, registry.getLastAccess('a'))
def test_stateMachine(self):
"""
A simple test to check that some transitions are invalid
"""
# Nice and easy
drop = InMemoryDROP('a', 'a')
self.assertEqual(drop.status, DROPStates.INITIALIZED)
drop.write('a')
self.assertEqual(drop.status, DROPStates.WRITING)
drop.setCompleted()
self.assertEqual(drop.status, DROPStates.COMPLETED)
# Try to overwrite the DROP's checksum and size
self.assertRaises(Exception, lambda: setattr(drop, 'checksum', 0))
self.assertRaises(Exception, lambda: setattr(drop, 'size', 0))
# Try to write on a DROP that is already COMPLETED
self.assertRaises(Exception, drop.write, '')
# Invalid reading on a DROP that isn't COMPLETED yet
drop = InMemoryDROP('a', 'a')
self.assertRaises(Exception, drop.open)
self.assertRaises(Exception, drop.read, 1)
self.assertRaises(Exception, drop.close, 1)
# Invalid file descriptors used to read/close
drop.setCompleted()
fd = drop.open()
otherFd = random.SystemRandom().randint(0, 1000)
self.assertNotEqual(fd, otherFd)
self.assertRaises(Exception, drop.read, otherFd)
self.assertRaises(Exception, drop.close, otherFd)
# but using the correct one should be OK
drop.read(fd)
self.assertTrue(drop.isBeingRead())
drop.close(fd)
# Expire it, then try to set it as COMPLETED again
drop.status = DROPStates.EXPIRED
self.assertRaises(Exception, drop.setCompleted)
def _createGraph(self):
"""
Creates the following graph of DROPs:
A |--> B ----> D --> G --> I --|
|--> C -|--> E --------------|-> H --> J
|--> F
B, C, G and H are AppDOs. The names have been given in breadth-first
order (although H has a dependency on I)
"""
a = InMemoryDROP('a', 'a')
b = BarrierAppDROP('b', 'b')
c = BarrierAppDROP('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
f = InMemoryDROP('f', 'f')
g = BarrierAppDROP('g', 'g')
h = BarrierAppDROP('h', 'h')
i = InMemoryDROP('i', 'i')
j = InMemoryDROP('j', 'j')
a.addConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
c.addOutput(f)
d.addConsumer(g)
e.addConsumer(h)
g.addOutput(i)
i.addConsumer(h)
h.addOutput(j)
return a, b, c, d, e, f, g, h, i, j
def test_addDrop(self):
a = InMemoryDROP('a', 'a')
registry = RDBMSRegistry('sqlite3', DBFILE)
registry.addDrop(a)
conn = sqlite3.connect(DBFILE) # @UndefinedVariable
cur = conn.cursor()
cur.execute('SELECT oid FROM dfms_drop')
r = cur.fetchone()
self.assertEqual(1, len(r))
self.assertEqual('a', r[0])
cur.close()
conn.close()
def test_socket_listener(self):
'''
A simple test to check that SocketListenerApps are indeed working as
expected; that is, they write the data they receive into their output,
and finish when the connection is closed from the client side
The data flow diagram looks like this:
A --> B --> C --> D
'''
host = 'localhost'
port = 9933
data = os.urandom(1025)
a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
b = InMemoryDROP('oid:B', 'uid:B')
c = SumupContainerChecksum('oid:C', 'uid:C')
d = InMemoryDROP('oid:D', 'uid:D')
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
# Create the socket, write, and close the connection, allowing
# A to move to COMPLETED
with DROPWaiterCtx(self, d, 3): # That's plenty of time
a.async_execute()
utils.write_to(host, port, data, 1)
for drop in [a,b,c,d]:
self.assertEqual(DROPStates.COMPLETED, drop.status)
# Our expectations are fulfilled!
bContents = droputils.allDropContents(b)
dContents = int(droputils.allDropContents(d))
self.assertEqual(data, bContents)
self.assertEqual(crc32(data, 0), dContents)
def test_single_pipe(self):
"""
A simple test where two bash apps are connected to each other in a
streaming fashion. The data flows through a pipe which is created by
the framework. The data drop in between acts only as a intermediator
to establish the underlying communication channel.
------------- -------------- ------------- ----------
| BashApp A | --> | InMemory B | --> | BashApp C | --> | File D |
| echo | | "/a/pipe" | | dc | | |
-----*------- -------------- ------*------ ----------
| |
\-------------|named-pipe|------------/
BashApp A writes "5 4 3 2 1" (each on a new line), which is read by cat
and redirected to D.
"""
output_fname = tempfile.mktemp()
a = StreamingOutputBashApp('a',
'a',
command=r"echo -en '5\n4\n3\n2\n1'")
b = InMemoryDROP('b', 'b')
c = StreamingInputBashApp('c', 'c', command="cat > %o0")
d = FileDROP('d', 'd', filepath=output_fname)
a.addOutput(b)
c.addStreamingInput(b)
c.addOutput(d)
# Let's fire the app
with DROPWaiterCtx(self, d, 2):
a.async_execute()
# The application executed, finished, and its output was recorded
for drop in (a, b, c, d):
self.assertEqual(DROPStates.COMPLETED, drop.status,
"Drop %r not COMPLETED: %d" % (drop, drop.status))
self.assertEqual(
[5, 4, 3, 2, 1],
[int(x) for x in droputils.allDropContents(d).split(six.b('\n'))])
# Clean up and go
os.remove(output_fname)
def test_directoryContainer(self):
"""
A small, simple test for the DirectoryContainer DROP that checks it allows
only valid children to be added
"""
# Prepare our playground
cwd = os.getcwd()
os.chdir('/tmp')
dirname = ".hidden"
dirname2 = ".hidden/inside"
if not os.path.exists(dirname2):
os.makedirs(dirname2)
# DROPs involved
a = FileDROP('a', 'a', dirname=dirname)
b = FileDROP('b', 'b', dirname=dirname)
c = FileDROP('c', 'c', dirname=dirname2)
d = FileDROP('d', 'd', dirname=dirname2)
cont1 = DirectoryContainer('e', 'e', dirname=dirname)
cont2 = DirectoryContainer('f', 'f', dirname=dirname2)
# Paths are absolutely reported
self.assertEqual(os.path.realpath('/tmp/.hidden'), os.path.realpath(cont1.path))
self.assertEqual(os.path.realpath('/tmp/.hidden/inside'), os.path.realpath(cont2.path))
# Certain children-to-be are rejected
self.assertRaises(TypeError, cont1.addChild, NullDROP('g', 'g'))
self.assertRaises(TypeError, cont1.addChild, InMemoryDROP('h', 'h'))
self.assertRaises(TypeError, cont1.addChild, ContainerDROP('i', 'i'))
self.assertRaises(Exception, cont1.addChild, c)
self.assertRaises(Exception, cont1.addChild, d)
self.assertRaises(Exception, cont2.addChild, a)
self.assertRaises(Exception, cont2.addChild, b)
# These children are correct
cont1.addChild(a)
cont1.addChild(b)
cont2.addChild(c)
cont2.addChild(d)
# Revert to previous state
shutil.rmtree(dirname, True)
os.chdir(cwd)
def _test_write_withDropType(self, dropType):
"""
Test an AbstractDROP and a simple AppDROP (for checksum calculation)
"""
a = dropType('oid:A', 'uid:A', expectedSize = self._test_drop_sz * ONE_MB)
b = SumupContainerChecksum('oid:B', 'uid:B')
c = InMemoryDROP('oid:C', 'uid:C')
b.addInput(a)
b.addOutput(c)
test_crc = 0
with DROPWaiterCtx(self, c):
for _ in range(self._test_num_blocks):
a.write(self._test_block)
test_crc = crc32(self._test_block, test_crc)
# Read the checksum from c
cChecksum = int(droputils.allDropContents(c))
self.assertNotEqual(a.checksum, 0)
self.assertEqual(a.checksum, test_crc)
self.assertEqual(cChecksum, test_crc)
def test_sql_create(self):
sqlite01 = get_oid('sqlite')
sqlite_drop = FileDROP(
sqlite01,
get_uuid(),
precious=False,
dirname=os.path.join(TestSqlite._temp, sqlite01),
check_exists=False,
)
initialize_sqlite = InitializeSqliteApp(
get_oid('app'),
get_uuid(),
user='******',
)
sqlite_in_memory = InMemoryDROP(
get_oid('memory'),
get_uuid(),
precious=False,
)
initialize_sqlite.addInput(sqlite_drop)
initialize_sqlite.addOutput(sqlite_in_memory)
with droputils.DROPWaiterCtx(self, sqlite_in_memory, 50000):
sqlite_drop.setCompleted()
def test_join(self):
"""
Using the container data object to implement a join/barrier dataflow.
A1, A2 and A3 are FileDROPs
B1, B2 and B3 are SumupContainerChecksum
C1, C2 and C3 are InMemoryDROPs
D is a SumupContainerChecksum
E is a InMemoryDROP
--> A1 --> B1 --> C1 --|
--> A2 --> B2 --> C2 --|--> D --> E
--> A3 --> B3 --> C3 --|
Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
after which "C" will transition to COMPLETE. Once all "C"s have moved to
COMPLETED "D"'s execution will also be triggered, and finally E will
hold the sum of B1, B2 and B3's checksums
"""
filelen = self._test_drop_sz * ONE_MB
#create file data objects
a1 = FileDROP('oid:A1', 'uid:A1', expectedSize=filelen)
a2 = FileDROP('oid:A2', 'uid:A2', expectedSize=filelen)
a3 = FileDROP('oid:A3', 'uid:A3', expectedSize=filelen)
# CRC Result DROPs, storing the result in memory
b1 = SumupContainerChecksum('oid:B1', 'uid:B1')
b2 = SumupContainerChecksum('oid:B2', 'uid:B2')
b3 = SumupContainerChecksum('oid:B3', 'uid:B3')
c1 = InMemoryDROP('oid:C1', 'uid:C1')
c2 = InMemoryDROP('oid:C2', 'uid:C2')
c3 = InMemoryDROP('oid:C3', 'uid:C3')
# The final DROP that sums up the CRCs from the container DROP
d = SumupContainerChecksum('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
# Wire together
dropAList = [a1,a2,a3]
dropBList = [b1,b2,b3]
dropCList = [c1,c2,c3]
for dropA,dropB in map(lambda a,b: (a,b), dropAList, dropBList):
dropA.addConsumer(dropB)
for dropB,dropC in map(lambda b,c: (b,c), dropBList, dropCList):
dropB.addOutput(dropC)
for dropC in dropCList:
dropC.addConsumer(d)
d.addOutput(e)
# Write data into the initial "A" DROPs, which should trigger
# the whole chain explained above
with DROPWaiterCtx(self, e):
for dropA in dropAList: # this should be parallel for
for _ in range(self._test_num_blocks):
dropA.write(self._test_block)
# All DROPs are completed now that the chain executed correctly
for drop in dropAList + dropBList + dropCList:
self.assertEqual(drop.status, DROPStates.COMPLETED)
# The results we want to compare
sum_crc = c1.checksum + c2.checksum + c3.checksum
dropEData = int(droputils.allDropContents(e))
self.assertNotEqual(sum_crc, 0)
self.assertEqual(sum_crc, dropEData)
def branch_failure(self, tooManyFailures):
"""
Using the container data object to implement a join/barrier dataflow.
A1, A2 and A3 are FileDROPs
B1, B2 and B3 are SumupContainerChecksum
C1, C2 and C3 are InMemoryDROPs
D is a SumupContainerChecksum
E is a InMemoryDROP
--> A1 --> B1 --> C1 --|
--> A2 --> B2 --> C2 --|--> D --> E
--> A3 --> B3 --> C3 --|
Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
after which "C" will transition to COMPLETE. Once all "C"s have moved to
COMPLETED "D"'s execution will also be triggered, and finally E will
hold the sum of B1, B2 and B3's checksums
"""
#create file data objects
a1 = InMemoryDROP('oid:A1', 'uid:A1')
a2 = InMemoryDROP('oid:A2', 'uid:A2')
a3 = InMemoryDROP('oid:A3', 'uid:A3')
# CRC Result DROPs, storing the result in memory
b1 = SumupContainerChecksum('oid:B1', 'uid:B1')
b2 = SumupContainerChecksum('oid:B2', 'uid:B2')
b3 = SumupContainerChecksum('oid:B3', 'uid:B3')
c1 = InMemoryDROP('oid:C1', 'uid:C1')
c2 = InMemoryDROP('oid:C2', 'uid:C2')
c3 = InMemoryDROP('oid:C3', 'uid:C3')
# The final DROP that sums up the CRCs from the container DROP
d = SumupContainerChecksum('oid:D', 'uid:D', input_error_threshold = 33)
e = InMemoryDROP('oid:E', 'uid:E')
# Wire together
dropAList = [a1,a2,a3]
dropBList = [b1,b2,b3]
dropCList = [c1,c2,c3]
for dropA,dropB in zip(dropAList, dropBList):
dropA.addConsumer(dropB)
for dropB,dropC in zip(dropBList, dropCList):
dropB.addOutput(dropC)
for dropC in dropCList:
dropC.addConsumer(d)
d.addOutput(e)
# Write data into the initial "A" DROPs, which should trigger
# the whole chain explained above
with DROPWaiterCtx(self, e):
#for dropA in dropAList: # this should be parallel for
a1.write(' '); a1.setCompleted()
if tooManyFailures:
a2.setError()
else:
a2.write(' '); a2.setCompleted()
a3.setError()
if tooManyFailures:
completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1]
errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e]
else:
completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e]
errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:]
for drop in completedDrops:
self.assertEqual(drop.status, DROPStates.COMPLETED)
for drop in errorDrops:
self.assertEqual(drop.status, DROPStates.ERROR)
# The results we want to compare
# (only in case that at least two branches executed)
if not tooManyFailures:
sum_crc = c1.checksum + c2.checksum
dropEData = int(droputils.allDropContents(e))
self.assertNotEqual(sum_crc, 0)
self.assertEqual(sum_crc, dropEData)
def test_simple_chain(self):
'''
Simple test that creates a pipeline-like chain of commands.
In this case we simulate a pipeline that does this, holding
each intermediate result in memory:
cat someFile | grep 'a' | sort | rev
'''
class GrepResult(BarrierAppDROP):
def initialize(self, **kwargs):
super(GrepResult, self).initialize(**kwargs)
self._substring = six.b(kwargs['substring'])
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = BytesIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
class SortResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = BytesIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
class RevResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allbytes = droputils.allDropContents(drop)
buf = bytearray()
for c in allbytes:
if c == six.b(' ') or c == six.b('\n'):
output.write(buf[::-1])
output.write(c)
buf = bytearray()
else:
buf.append(c)
a = InMemoryDROP('oid:A', 'uid:A')
b = GrepResult('oid:B', 'uid:B', substring="a")
c = InMemoryDROP('oid:C', 'uid:C')
d = SortResult('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
f = RevResult('oid:F', 'oid:F')
g = InMemoryDROP('oid:G', 'uid:G')
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
e.addConsumer(f)
f.addOutput(g)
# Initial write
contents = "first line\nwe have an a here\nand another one\nnoone knows me"
cResExpected = "we have an a here\nand another one\n"
eResExpected = "and another one\nwe have an a here\n"
gResExpected = "dna rehtona eno\new evah na a ereh\n"
with DROPWaiterCtx(self, g):
a.write(contents)
a.setCompleted()
# Get intermediate and final results and compare
actualRes = []
for i in [c, e, g]:
actualRes.append(droputils.allDropContents(i))
map(lambda x, y: self.assertEqual(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
self._evt = threading.Event()
drop.addConsumer(self)
def dropCompleted(self, drop, state):
self._evt.set()
def wait(self, timeout=None):
return self._evt.wait(timeout)
if __name__ == '__main__':
try:
vis_in_a = []
flux_out = InMemoryDROP(uuid.uuid1(), uuid.uuid1())
flux = SourceFlux(uuid.uuid1(), uuid.uuid1(), casapy_path=CASAPY)
cl = Clean(uuid.uuid1(),
uuid.uuid1(),
field='deepfield',
mode='frequency',
restfreq='1420.405752MHz',
nchan=-1,
start='',
width='',
interpolation='nearest',
gain=0.1,
imsize=[256],
cell=['1.0arcsec'],
def createDrop(self, oid, uid, **kwargs):
return InMemoryDROP(oid, uid, **kwargs)
