def test_eager_inputFired_app(self):
"""
Tests that InputFiredApps works as expected
"""
# No n_effective_inputs given
self.assertRaises(ValueError, InputFiredAppDROP, "a", "a")
# Invalid values
self.assertRaises(ValueError, InputFiredAppDROP, "a", "a", n_effective_inputs=-2)
self.assertRaises(ValueError, InputFiredAppDROP, "a", "a", n_effective_inputs=0)
# More effective inputs than inputs
a = InMemoryDROP("b", "b")
b = InputFiredAppDROP("a", "a", n_effective_inputs=2)
b.addInput(a)
self.assertRaises(Exception, a.setCompleted)
# 2 effective inputs, 4 outputs. Trigger 2 inputs and make sure the
# app has run
a, b, c, d = [InMemoryDROP(str(i), str(i)) for i in xrange(4)]
e = InputFiredAppDROP("e", "e", n_effective_inputs=2)
map(lambda x: e.addInput(x), [a, b, c, d])
with DROPWaiterCtx(self, e, 5):
a.setCompleted()
b.setCompleted()
self.assertEquals(AppDROPStates.FINISHED, e.execStatus)
self.assertEquals(DROPStates.COMPLETED, a.status)
self.assertEquals(DROPStates.COMPLETED, b.status)
self.assertEquals(DROPStates.INITIALIZED, c.status)
self.assertEquals(DROPStates.INITIALIZED, d.status)
def test_eager_inputFired_app(self):
"""
Tests that InputFiredApps works as expected
"""
# No n_effective_inputs given
self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a')
# Invalid values
self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=-2)
self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=0)
# More effective inputs than inputs
a = InMemoryDROP('b', 'b')
b = InputFiredAppDROP('a', 'a', n_effective_inputs=2)
b.addInput(a)
self.assertRaises(Exception, a.setCompleted)
# 2 effective inputs, 4 outputs. Trigger 2 inputs and make sure the
# app has run
a,b,c,d = [InMemoryDROP(str(i), str(i)) for i in range(4)]
e = InputFiredAppDROP('e', 'e', n_effective_inputs=2)
for x in a,b,c,d:
e.addInput(x)
with DROPWaiterCtx(self, e, 5):
a.setCompleted()
b.setCompleted()
self.assertEqual(AppDROPStates.FINISHED, e.execStatus)
self.assertEqual(DROPStates.COMPLETED, a.status)
self.assertEqual(DROPStates.COMPLETED, b.status)
self.assertEqual(DROPStates.INITIALIZED, c.status)
self.assertEqual(DROPStates.INITIALIZED, d.status)
def test_app_multiple_outputs(self):
"""
A small method that tests that the AppDROPs writing to two
different DROPs outputs works
The graph constructed by this example looks as follow:
|--> E
A --> B --> C --> D --|
|--> F
Here B and D are an AppDROPs, with D writing to two DROPs
outputs (E and F) and reading from C. C, in turn, is written by B, which
in turns reads the data from A
"""
# This is used as "B"
class NumberWriterApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in xrange(howMany):
output.write(str(i) + " ")
# This is used as "D"
class OddAndEvenContainerApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + " ")
# Create DROPs
a = InMemoryDROP("oid:A", "uid:A")
b = NumberWriterApp("oid:B", "uid:B")
c = InMemoryDROP("oid:A", "uid:A")
d = OddAndEvenContainerApp("oid:D", "uid:D")
e = InMemoryDROP("oid:E", "uid:E")
f = InMemoryDROP("oid:F", "uid:F")
# Wire them together
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
d.addOutput(f)
# Start the execution
with DROPWaiterCtx(self, [e, f]):
a.write("20")
a.setCompleted()
# Check the final results are correct
for drop in [a, b, c, d, e]:
self.assertEquals(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
self.assertEquals("0 2 4 6 8 10 12 14 16 18", droputils.allDropContents(e).strip())
self.assertEquals("1 3 5 7 9 11 13 15 17 19", droputils.allDropContents(f).strip())
def _test_graphExecutionDriver(self, mode):
"""
A small test to check that DROPs executions can be driven externally if
required, and not always internally by themselves
"""
a = InMemoryDROP('a', 'a', executionMode=mode, expectedSize=1)
b = SumupContainerChecksum('b', 'b')
c = InMemoryDROP('c', 'c')
a.addConsumer(b)
c.addProducer(b)
# Write and check
dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c]
with DROPWaiterCtx(self, dropsToWaitFor):
a.write('1')
if mode == ExecutionMode.EXTERNAL:
# b hasn't been triggered
self.assertEqual(c.status, DROPStates.INITIALIZED)
self.assertEqual(b.status, DROPStates.INITIALIZED)
self.assertEqual(b.execStatus, AppDROPStates.NOT_RUN)
# Now let b consume a
with DROPWaiterCtx(self, [c]):
b.dropCompleted('a', DROPStates.COMPLETED)
self.assertEqual(c.status, DROPStates.COMPLETED)
elif mode == ExecutionMode.DROP:
# b is already done
self.assertEqual(c.status, DROPStates.COMPLETED)
def test_addDropInstances(self):
a1 = InMemoryDROP('a', 'a1')
a2 = InMemoryDROP('a', 'a2')
registry = RDBMSRegistry('sqlite3', DBFILE)
registry.addDrop(a1)
uids = registry.getDropUids(a1)
self.assertEqual(1, len(uids))
self.assertEqual('a1', uids[0])
registry.addDropInstance(a2)
uids = registry.getDropUids(a1)
uids.sort()
self.assertEqual(2, len(uids))
self.assertEqual('a1', uids[0])
self.assertEqual('a2', uids[1])
# Check accessing the database separately
conn = sqlite3.connect(DBFILE) # @UndefinedVariable
cur = conn.cursor()
cur.execute("SELECT uid FROM dfms_dropinstance WHERE oid = 'a'")
rows = cur.fetchall()
self.assertEqual(2, len(rows))
uids = [r[0] for r in rows]
uids.sort()
self.assertEqual(2, len(uids))
self.assertEqual('a1', uids[0])
self.assertEqual('a2', uids[1])
cur.close()
conn.close()
def container_pg():
'''
Creates the following graph:
|--> B --> D --|
SL --> A --| |--> F --> G --> H --> I
|--> C --> E --|
'''
sl= SocketListenerApp('SL', 'SL', lifespan=lifespan)
a = InMemoryDROP('A', 'A', lifespan=lifespan)
b = SleepAndCopyApp('B', 'B', lifespan=lifespan)
c = SleepAndCopyApp('C', 'C', lifespan=lifespan)
d = InMemoryDROP('D', 'D', lifespan=lifespan)
e = InMemoryDROP('E', 'E', lifespan=lifespan)
f = SimpleBarrierApp('F', 'F', lifespan=lifespan)
g = ContainerDROP('G', 'G', lifespan=lifespan)
h = SleepAndCopyApp('H', 'H', lifespan=lifespan)
i = InMemoryDROP('I', 'I', lifespan=lifespan)
# Wire together
sl.addOutput(a)
a.addConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
d.addConsumer(f)
e.addConsumer(f)
f.addOutput(g)
g.addChild(d)
g.addChild(e)
g.addConsumer(h)
h.addOutput(i)
return sl
def test_invalid(self):
# Shouldn't allow inputs
a = SocketListenerApp('a', 'a', port=1)
a.addOutput(InMemoryDROP('c', 'c'))
self.assertRaises(Exception, a.addInput, InMemoryDROP('b', 'b'))
self.assertRaises(Exception, a.addStreamingInput, InMemoryDROP('b', 'b'))
# Shouldn't be able to open ports <= 1024
a.execute()
self.assertEqual(a.status, DROPStates.ERROR)
def _test_graphExecutionDriver(self, mode):
"""
A small test to check that DROPs executions can be driven externally if
required, and not always internally by themselves
"""
a = InMemoryDROP("a", "a", executionMode=mode, expectedSize=1)
b = SumupContainerChecksum("b", "b")
c = InMemoryDROP("c", "c")
a.addConsumer(b)
c.addProducer(b)
# Write and check
dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c]
with DROPWaiterCtx(self, dropsToWaitFor):
a.write("1")
if mode == ExecutionMode.EXTERNAL:
# b hasn't been triggered
self.assertEquals(c.status, DROPStates.INITIALIZED)
self.assertEquals(b.status, DROPStates.INITIALIZED)
self.assertEquals(b.execStatus, AppDROPStates.NOT_RUN)
# Now let b consume a
with DROPWaiterCtx(self, [c]):
b.dropCompleted("a", DROPStates.COMPLETED)
self.assertEquals(c.status, DROPStates.COMPLETED)
elif mode == ExecutionMode.DROP:
# b is already done
self.assertEquals(c.status, DROPStates.COMPLETED)
def test_pg():
"""
A very simple graph that looks like this:
A --> B --> C --> D
"""
a = SocketListenerApp('A', 'A', lifespan=lifespan)
b = InMemoryDROP('B', 'B', lifespan=lifespan)
c = SleepAndCopyApp('C', 'C', lifespan=lifespan)
d = InMemoryDROP('D', 'D', lifespan=lifespan)
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
return a
def test_speadApp(self):
port = 1111
itemId = 0x2000
thread_pool = spead2.ThreadPool()
self._stream = spead2.send.UdpStream(
thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7))
a = SpeadReceiverApp('a', 'a', port=port, itemId=itemId)
b = InMemoryDROP('b', 'b')
a.addOutput(b)
size = 1024
threading.Thread(target=lambda: a.execute()).start()
time.sleep(1)
msg = os.urandom(size)
with DROPWaiterCtx(self, b, timeout=1):
ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48))
item = ig.add_item(itemId,
'main_data',
'a char array',
shape=(size, ),
format=[('c', 8)])
item.value = msg
self._stream.send_heap(ig.get_heap())
self._stream.send_heap(ig.get_end())
for drop in a, b:
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual(size, b.size)
self.assertEqual(msg, droputils.allDropContents(b))
def test_multipleProducers(self):
"""
A test that checks that multiple-producers correctly drive the state of
their shared output
"""
class App(BarrierAppDROP): pass
a,b,c,d,e = [App(chr(ord('A') + i), chr(ord('A') + i)) for i in range(5)]
f = InMemoryDROP('F', 'F')
for drop in a,b,c,d,e:
drop.addOutput(f)
self.assertEqual(DROPStates.INITIALIZED, f.status)
for drop in a,b,c,d,e:
self.assertEqual(AppDROPStates.NOT_RUN, drop.execStatus)
# Run the first 4 ones, F should still be in INITIALIZED
for drop in a,b,c,d:
drop.execute()
self.assertEqual(DROPStates.INITIALIZED, f.status)
self.assertEqual(AppDROPStates.NOT_RUN, e.execStatus)
for drop in a,b,c,d:
self.assertEqual(AppDROPStates.FINISHED, drop.execStatus)
# Run the final one, now F should be COMPLETED
e.execute()
self.assertEqual(DROPStates.COMPLETED, f.status)
for drop in a,b,c,d,e:
self.assertEqual(AppDROPStates.FINISHED, drop.execStatus)
def test_errorState(self):
a = InMemoryDROP('a', 'a')
b = SumupContainerChecksum('b', 'b')
c = InMemoryDROP('c', 'c')
c.addProducer(b)
b.addInput(a)
a.setError()
self.assertEqual(DROPStates.ERROR, a.status)
self.assertEqual(DROPStates.ERROR, b.status)
self.assertEqual(DROPStates.ERROR, c.status)
def test_dropAccess(self):
a1 = InMemoryDROP('a', 'a1')
registry = RDBMSRegistry('sqlite3', DBFILE)
registry.addDrop(a1)
self.assertEqual(-1, registry.getLastAccess('a'))
registry.recordNewAccess('a')
self.assertNotEqual(-1, registry.getLastAccess('a'))
def test_two_simultaneous_pipes(self):
"""
A more complicated test where three bash applications run at the same
time. The first streams its output to the second one, while the second
one streams *its* output to the third one.
------------- -------------- ------------- -------------- ------------- ----------
| BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F |
| echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | |
-----*------- -------------- ----*--*----- -------------- -----*------- ----------
| | | |
\-------------|named-pipe|----------\ \-----------|named-pipe|-----------/
BashApp A writes "5 4 3 2 1" (each on a new line), which is read
by "cat" (BashApp C). The printed results (a copy of the original) are
streamed through D and read by "sort" (BashApp E), which writes the
output to F.
"""
output_fname = tempfile.mktemp()
a = StreamingOutputBashApp('a',
'a',
command=r"echo -en '5\n4\n3\n2\n1'")
b = InMemoryDROP('b', 'b')
c = StreamingInputOutputBashApp('c', 'c', command="cat")
d = InMemoryDROP('d', 'd')
e = StreamingInputBashApp('e', 'e', command="sort -n > %o0")
f = FileDROP('f', 'f', filepath=output_fname)
a.addOutput(b)
b.addStreamingConsumer(c)
c.addOutput(d)
d.addStreamingConsumer(e)
e.addOutput(f)
# Let's fire the app
with DROPWaiterCtx(self, f, 2):
a.async_execute()
# The application executed, finished, and its output was recorded
for drop in (a, b, c, d, e, f):
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual([1, 2, 3, 4, 5], [
int(x)
for x in droputils.allDropContents(f).strip().split(six.b('\n'))
])
# Clean up and go
os.remove(output_fname)
def test_addDrop(self):
a = InMemoryDROP('a', 'a')
registry = RDBMSRegistry('sqlite3', DBFILE)
registry.addDrop(a)
conn = sqlite3.connect(DBFILE) # @UndefinedVariable
cur = conn.cursor()
cur.execute('SELECT oid FROM dfms_drop')
r = cur.fetchone()
self.assertEqual(1, len(r))
self.assertEqual('a', r[0])
cur.close()
conn.close()
def test_socket_listener(self):
'''
A simple test to check that SocketListenerApps are indeed working as
expected; that is, they write the data they receive into their output,
and finish when the connection is closed from the client side
The data flow diagram looks like this:
A --> B --> C --> D
'''
host = 'localhost'
port = 9933
data = 'shine on you crazy diamond'
a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
b = InMemoryDROP('oid:B', 'uid:B')
c = SumupContainerChecksum('oid:C', 'uid:C')
d = InMemoryDROP('oid:D', 'uid:D')
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
# Create the socket, write, and close the connection, allowing
# A to move to COMPLETED
with DROPWaiterCtx(self, d, 3): # That's plenty of time
threading.Thread(target=lambda a: a.execute(), args=(a,)).start()
utils.writeToRemotePort(host, port, data, 1)
for drop in [a,b,c,d]:
self.assertEquals(DROPStates.COMPLETED, drop.status)
# Our expectations are fulfilled!
bContents = droputils.allDropContents(b)
dContents = int(droputils.allDropContents(d))
self.assertEquals(data, bContents)
self.assertEquals(crc32(data, 0), dContents)
def test_socket_listener(self):
'''
A simple test to check that SocketListenerApps are indeed working as
expected; that is, they write the data they receive into their output,
and finish when the connection is closed from the client side
The data flow diagram looks like this:
A --> B --> C --> D
'''
host = 'localhost'
port = 9933
data = os.urandom(1025)
a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
b = InMemoryDROP('oid:B', 'uid:B')
c = SumupContainerChecksum('oid:C', 'uid:C')
d = InMemoryDROP('oid:D', 'uid:D')
a.addOutput(b)
b.addConsumer(c)
c.addOutput(d)
# Create the socket, write, and close the connection, allowing
# A to move to COMPLETED
with DROPWaiterCtx(self, d, 3): # That's plenty of time
a.async_execute()
utils.write_to(host, port, data, 1)
for drop in [a,b,c,d]:
self.assertEqual(DROPStates.COMPLETED, drop.status)
# Our expectations are fulfilled!
bContents = droputils.allDropContents(b)
dContents = int(droputils.allDropContents(d))
self.assertEqual(data, bContents)
self.assertEqual(crc32(data, 0), dContents)
def test_errorState(self):
a = InMemoryDROP("a", "a")
b = SumupContainerChecksum("b", "b")
c = InMemoryDROP("c", "c")
c.addProducer(b)
b.addInput(a)
a.setError()
self.assertEquals(DROPStates.ERROR, a.status)
self.assertEquals(DROPStates.ERROR, b.status)
self.assertEquals(DROPStates.ERROR, c.status)
def _testGraph(execMode):
"""
A test graph that looks like this:
|--> B1 --> C1 --|
|--> B2 --> C2 --|
SL_A --> A --|--> B3 --> C3 --| --> D --> E
|--> .. --> .. --|
|--> BN --> CN --|
B and C DROPs are InMemorySleepAndCopyApp DROPs (see above). D is simply a
container. A is a socket listener, so we can actually write to it externally
and watch the progress of the luigi tasks. We give DROPs a long lifespan;
otherwise they will expire and luigi will see it as a failed task (which is
actually right!)
If execMode is given we use that in all DROPs. If it's None we use a mixture
of DROP/EXTERNAL execution modes.
"""
aMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
bMode = execMode if execMode is not None else ExecutionMode.DROP
cMode = execMode if execMode is not None else ExecutionMode.DROP
dMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
eMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
sl_a = SocketListenerApp('oid:SL_A', 'uid:SL_A', executionMode=aMode, lifespan=lifespan)
a = InMemoryDROP('oid:A', 'uid:A', executionMode=aMode, lifespan=lifespan)
d = SumupContainerChecksum('oid:D', 'uid:D', executionMode=dMode, lifespan=lifespan)
e = InMemoryDROP('oid:E', 'uid:E', executionMode=eMode, lifespan=lifespan)
sl_a.addOutput(a)
e.addProducer(d)
for i in xrange(random.SystemRandom().randint(10, 20)):
b = SleepAndCopyApp('oid:B%d' % (i), 'uid:B%d' % (i), executionMode=bMode, lifespan=lifespan)
c = InMemoryDROP('oid:C%d' % (i), 'uid:C%d' % (i), executionMode=cMode, lifespan=lifespan)
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
return sl_a
def test_single_pipe(self):
"""
A simple test where two bash apps are connected to each other in a
streaming fashion. The data flows through a pipe which is created by
the framework. The data drop in between acts only as a intermediator
to establish the underlying communication channel.
------------- -------------- ------------- ----------
| BashApp A | --> | InMemory B | --> | BashApp C | --> | File D |
| echo | | "/a/pipe" | | dc | | |
-----*------- -------------- ------*------ ----------
| |
\-------------|named-pipe|------------/
BashApp A writes "5 4 3 2 1" (each on a new line), which is read by cat
and redirected to D.
"""
output_fname = tempfile.mktemp()
a = StreamingOutputBashApp('a',
'a',
command=r"echo -en '5\n4\n3\n2\n1'")
b = InMemoryDROP('b', 'b')
c = StreamingInputBashApp('c', 'c', command="cat > %o0")
d = FileDROP('d', 'd', filepath=output_fname)
a.addOutput(b)
c.addStreamingInput(b)
c.addOutput(d)
# Let's fire the app
with DROPWaiterCtx(self, d, 2):
a.async_execute()
# The application executed, finished, and its output was recorded
for drop in (a, b, c, d):
self.assertEqual(DROPStates.COMPLETED, drop.status,
"Drop %r not COMPLETED: %d" % (drop, drop.status))
self.assertEqual(
[5, 4, 3, 2, 1],
[int(x) for x in droputils.allDropContents(d).split(six.b('\n'))])
# Clean up and go
os.remove(output_fname)
def test_directoryContainer(self):
"""
A small, simple test for the DirectoryContainer DROP that checks it allows
only valid children to be added
"""
# Prepare our playground
cwd = os.getcwd()
os.chdir('/tmp')
dirname = ".hidden"
dirname2 = ".hidden/inside"
if not os.path.exists(dirname2):
os.makedirs(dirname2)
# DROPs involved
a = FileDROP('a', 'a', dirname=dirname)
b = FileDROP('b', 'b', dirname=dirname)
c = FileDROP('c', 'c', dirname=dirname2)
d = FileDROP('d', 'd', dirname=dirname2)
cont1 = DirectoryContainer('e', 'e', dirname=dirname)
cont2 = DirectoryContainer('f', 'f', dirname=dirname2)
# Paths are absolutely reported
self.assertEqual(os.path.realpath('/tmp/.hidden'), os.path.realpath(cont1.path))
self.assertEqual(os.path.realpath('/tmp/.hidden/inside'), os.path.realpath(cont2.path))
# Certain children-to-be are rejected
self.assertRaises(TypeError, cont1.addChild, NullDROP('g', 'g'))
self.assertRaises(TypeError, cont1.addChild, InMemoryDROP('h', 'h'))
self.assertRaises(TypeError, cont1.addChild, ContainerDROP('i', 'i'))
self.assertRaises(Exception, cont1.addChild, c)
self.assertRaises(Exception, cont1.addChild, d)
self.assertRaises(Exception, cont2.addChild, a)
self.assertRaises(Exception, cont2.addChild, b)
# These children are correct
cont1.addChild(a)
cont1.addChild(b)
cont2.addChild(c)
cont2.addChild(d)
# Revert to previous state
shutil.rmtree(dirname, True)
os.chdir(cwd)
def _test_write_withDropType(self, dropType):
"""
Test an AbstractDROP and a simple AppDROP (for checksum calculation)
"""
a = dropType('oid:A', 'uid:A', expectedSize = self._test_drop_sz * ONE_MB)
b = SumupContainerChecksum('oid:B', 'uid:B')
c = InMemoryDROP('oid:C', 'uid:C')
b.addInput(a)
b.addOutput(c)
test_crc = 0
with DROPWaiterCtx(self, c):
for _ in range(self._test_num_blocks):
a.write(self._test_block)
test_crc = crc32(self._test_block, test_crc)
# Read the checksum from c
cChecksum = int(droputils.allDropContents(c))
self.assertNotEqual(a.checksum, 0)
self.assertEqual(a.checksum, test_crc)
self.assertEqual(cChecksum, test_crc)
def test_sql_create(self):
sqlite01 = get_oid('sqlite')
sqlite_drop = FileDROP(
sqlite01,
get_uuid(),
precious=False,
dirname=os.path.join(TestSqlite._temp, sqlite01),
check_exists=False,
)
initialize_sqlite = InitializeSqliteApp(
get_oid('app'),
get_uuid(),
user='******',
)
sqlite_in_memory = InMemoryDROP(
get_oid('memory'),
get_uuid(),
precious=False,
)
initialize_sqlite.addInput(sqlite_drop)
initialize_sqlite.addOutput(sqlite_in_memory)
with droputils.DROPWaiterCtx(self, sqlite_in_memory, 50000):
sqlite_drop.setCompleted()
def test_app_multiple_outputs(self):
"""
A small method that tests that the AppDROPs writing to two
different DROPs outputs works
The graph constructed by this example looks as follow:
|--> E
A --> B --> C --> D --|
|--> F
Here B and D are an AppDROPs, with D writing to two DROPs
outputs (E and F) and reading from C. C, in turn, is written by B, which
in turns reads the data from A
"""
# This is used as "B"
class NumberWriterApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
output = self.outputs[0]
howMany = int(droputils.allDropContents(inputDrop))
for i in range(howMany):
output.write(str(i) + " ")
# This is used as "D"
class OddAndEvenContainerApp(BarrierAppDROP):
def run(self):
inputDrop = self.inputs[0]
outputs = self.outputs
numbers = droputils.allDropContents(inputDrop).strip().split()
for n in numbers:
outputs[int(n) % 2].write(n + six.b(" "))
# Create DROPs
a = InMemoryDROP('oid:A', 'uid:A')
b = NumberWriterApp('oid:B', 'uid:B')
c = InMemoryDROP('oid:A', 'uid:A')
d = OddAndEvenContainerApp('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
f = InMemoryDROP('oid:F', 'uid:F')
# Wire them together
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
d.addOutput(f)
# Start the execution
with DROPWaiterCtx(self, [e,f]):
a.write('20')
a.setCompleted()
# Check the final results are correct
for drop in [a,b,c,d,e]:
self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
self.assertEqual(six.b("0 2 4 6 8 10 12 14 16 18"), droputils.allDropContents(e).strip())
self.assertEqual(six.b("1 3 5 7 9 11 13 15 17 19"), droputils.allDropContents(f).strip())
def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastByte = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastByte = six.indexbytes(data, -1)
outputDrop.write(self._lastByte)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP('a', 'a')
b = LastCharWriterApp('b', 'b')
c = SumupContainerChecksum('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, a.addConsumer, b)
self.assertRaises(Exception, a.addStreamingConsumer, c)
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastByte):
self.assertEqual(aStatus, a.status)
self.assertEqual(dStatus, d.status)
self.assertEqual(eStatus, e.status)
if lastByte is not None:
self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte))
checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write('abcde')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e')
a.write('fghij')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j')
a.write('k')
with DROPWaiterCtx(self, [d,e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k')
self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
def test_stateMachine(self):
"""
A simple test to check that some transitions are invalid
"""
# Nice and easy
drop = InMemoryDROP('a', 'a')
self.assertEqual(drop.status, DROPStates.INITIALIZED)
drop.write('a')
self.assertEqual(drop.status, DROPStates.WRITING)
drop.setCompleted()
self.assertEqual(drop.status, DROPStates.COMPLETED)
# Try to overwrite the DROP's checksum and size
self.assertRaises(Exception, lambda: setattr(drop, 'checksum', 0))
self.assertRaises(Exception, lambda: setattr(drop, 'size', 0))
# Try to write on a DROP that is already COMPLETED
self.assertRaises(Exception, drop.write, '')
# Invalid reading on a DROP that isn't COMPLETED yet
drop = InMemoryDROP('a', 'a')
self.assertRaises(Exception, drop.open)
self.assertRaises(Exception, drop.read, 1)
self.assertRaises(Exception, drop.close, 1)
# Invalid file descriptors used to read/close
drop.setCompleted()
fd = drop.open()
otherFd = random.SystemRandom().randint(0, 1000)
self.assertNotEqual(fd, otherFd)
self.assertRaises(Exception, drop.read, otherFd)
self.assertRaises(Exception, drop.close, otherFd)
# but using the correct one should be OK
drop.read(fd)
self.assertTrue(drop.isBeingRead())
drop.close(fd)
# Expire it, then try to set it as COMPLETED again
drop.status = DROPStates.EXPIRED
self.assertRaises(Exception, drop.setCompleted)
def test_simple_chain(self):
'''
Simple test that creates a pipeline-like chain of commands.
In this case we simulate a pipeline that does this, holding
each intermediate result in memory:
cat someFile | grep 'a' | sort | rev
'''
class GrepResult(BarrierAppDROP):
def initialize(self, **kwargs):
super(GrepResult, self).initialize(**kwargs)
self._substring = six.b(kwargs['substring'])
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = BytesIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
class SortResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = BytesIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
class RevResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allbytes = droputils.allDropContents(drop)
buf = bytearray()
for c in allbytes:
if c == six.b(' ') or c == six.b('\n'):
output.write(buf[::-1])
output.write(c)
buf = bytearray()
else:
buf.append(c)
a = InMemoryDROP('oid:A', 'uid:A')
b = GrepResult('oid:B', 'uid:B', substring="a")
c = InMemoryDROP('oid:C', 'uid:C')
d = SortResult('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
f = RevResult('oid:F', 'oid:F')
g = InMemoryDROP('oid:G', 'uid:G')
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
e.addConsumer(f)
f.addOutput(g)
# Initial write
contents = "first line\nwe have an a here\nand another one\nnoone knows me"
cResExpected = "we have an a here\nand another one\n"
eResExpected = "and another one\nwe have an a here\n"
gResExpected = "dna rehtona eno\new evah na a ereh\n"
with DROPWaiterCtx(self, g):
a.write(contents)
a.setCompleted()
# Get intermediate and final results and compare
actualRes = []
for i in [c, e, g]:
actualRes.append(droputils.allDropContents(i))
map(lambda x, y: self.assertEqual(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
def _createGraph(self):
"""
Creates the following graph of DROPs:
A |--> B ----> D --> G --> I --|
|--> C -|--> E --------------|-> H --> J
|--> F
B, C, G and H are AppDOs. The names have been given in breadth-first
order (although H has a dependency on I)
"""
a = InMemoryDROP('a', 'a')
b = BarrierAppDROP('b', 'b')
c = BarrierAppDROP('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
f = InMemoryDROP('f', 'f')
g = BarrierAppDROP('g', 'g')
h = BarrierAppDROP('h', 'h')
i = InMemoryDROP('i', 'i')
j = InMemoryDROP('j', 'j')
a.addConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
c.addOutput(f)
d.addConsumer(g)
e.addConsumer(h)
g.addOutput(i)
i.addConsumer(h)
h.addOutput(j)
return a, b, c, d, e, f, g, h, i, j
def lofar_standard_pip_pg():
"""
Lofar simple imaging pipeline
based on
Ger's email and
https://github.com/lofar-astron/lofar-profiling
"""
roots = []
num_time_slice = 3
total_num_subband = 8
num_subb_per_image = 2
Cal_model_oid = "A-team/Calibrator"
dob_cal_model = InMemoryDROP(Cal_model_oid, Cal_model_oid, lifespan=lifespan)
dob_cal_model.location = "catalogue.Groningen"
roots.append(dob_cal_model)
make_source_app_oid = "makesourcedb"
make_source_app = SleepAndCopyApp(make_source_app_oid, make_source_app_oid)
make_source_app.location = dob_cal_model.location
dob_cal_model.addConsumer(make_source_app)
source_db_oid = "CalSource.db"
dob_source_db = InMemoryDROP(source_db_oid, source_db_oid, lifespan=lifespan)
dob_source_db.location = dob_cal_model.location
make_source_app.addOutput(dob_source_db)
GSM_oid = "GlobalSkyModel"
dob_gsm = InMemoryDROP(GSM_oid, GSM_oid, lifespan=lifespan)
dob_gsm.location = "catalogue.Groningen"
roots.append(dob_gsm)
dob_img_dict = dict()
sb_chunks = chunks(range(1, total_num_subband + 1), num_subb_per_image)
for k, img_list in enumerate(sb_chunks):
imger_oid = "AWImager_SB_{0}~{1}".format(img_list[0], img_list[-1])
dob_img = SleepAndCopyApp(imger_oid, imger_oid, lifespan=lifespan)
kk = k + 1
#print k, img_list, kk
img_prod_oid = "Image_SB_{0}~{1}".format(img_list[0], img_list[-1])
dob_img_prod = InMemoryDROP(img_prod_oid, img_prod_oid, lifespan=lifespan)
dob_img.addOutput(dob_img_prod)
dob_img_dict[kk] = (dob_img, dob_img_prod)
for j in range(1, num_time_slice + 1):
sli = "T%02d" % j
for i in range(1, total_num_subband + 1):
# for LBA, assume each subband should have two beams
# one for calibration pipeline, the other for target / imaging pipeline
stri = "%s_SB%02d" % (sli, i)
time_slice_oid = "Socket_{0}".format(stri)
dob_time_slice_sl = SocketListenerApp(time_slice_oid + "_SL", time_slice_oid + "_SL", lifespan=lifespan)
dob_time_slice = InMemoryDROP(time_slice_oid, time_slice_oid, lifespan=lifespan)
dob_time_slice_sl.addOutput(dob_time_slice)
# all the same sub-bands (i) will be on the same node regardless of its time slice j
dob_time_slice.location = "Node%03d.Groningen" % i
roots.append(dob_time_slice_sl)
oid_data_writer = "DataWriter_{0}".format(stri)
dob_ingest = SleepAndCopyApp(oid_data_writer, oid_data_writer, lifespan=lifespan)
dob_ingest.location = dob_time_slice.location
dob_time_slice.addConsumer(dob_ingest)
# assume a single measuremenset has multiple beams
oid = "CalNTgt.MS_{0}".format(stri)
dob_input = InMemoryDROP(oid, oid, lifespan=lifespan)
dob_ingest.addOutput(dob_input)
dob_input.location = dob_ingest.location
#split by beam - calibrator beam and target beam
appOid = "Split_{0}".format(stri)
splitApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
splitApp.location = dob_ingest.location
dob_input.addConsumer(splitApp)
oid = "Cal_BEAM_{0}".format(stri)
dob_cal_beam = InMemoryDROP(oid, oid, lifespan=lifespan)
dob_cal_beam.location = dob_ingest.location
splitApp.addOutput(dob_cal_beam)
oid = "Tgt_BEAM_{0}".format(stri)
dob_tgt_beam = InMemoryDROP(oid, oid, lifespan=lifespan)
dob_tgt_beam.location = dob_ingest.location
splitApp.addOutput(dob_tgt_beam)
# flag the cal beam
oid = "FlgCal.MS_{0}".format(stri)
appOid = "NDPPP_Cal" + stri
dob_cal_flagged = InMemoryDROP(oid, oid, lifespan=lifespan)
flagCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
dob_cal_beam.addConsumer(flagCalApp)
dob_source_db.addConsumer(flagCalApp)
flagCalApp.addOutput(dob_cal_flagged)
flagCalApp.location = dob_ingest.location
dob_cal_flagged.location = dob_ingest.location #"C%03d.Groningen" % i
# flag the target beam
oid = "FlgTgt.MS_{0}".format(stri)
appOid = "NDPPP_Tgt" + stri
dob_tgt_flagged = InMemoryDROP(oid, oid, lifespan=lifespan)
flagTgtApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
dob_tgt_beam.addConsumer(flagTgtApp)
dob_source_db.addConsumer(flagTgtApp)
flagTgtApp.addOutput(dob_tgt_flagged)
flagTgtApp.location = dob_ingest.location
dob_tgt_flagged.location = dob_ingest.location
# For calibration, each time slice is calibrated independently, and
# the result is an output MS per subband
# solve the gain
oid = "Gain.TBL_{0}".format(stri)
appOid = "BBS_GainCal_{0}".format(stri)
dob_param_tbl = InMemoryDROP(oid, oid, lifespan=lifespan)
gainCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
dob_cal_flagged.addConsumer(gainCalApp)
gainCalApp.addOutput(dob_param_tbl)
gainCalApp.location = dob_ingest.location
dob_param_tbl.location = dob_ingest.location
# or apply the gain to the dataset
oid = "CAL.MS_{0}".format(stri)
appOid = "BBS_ApplyCal_{0}".format(stri)
dob_calibrated = InMemoryDROP(oid, oid, lifespan=lifespan)
applyCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
dob_tgt_flagged.addConsumer(applyCalApp)
dob_param_tbl.addConsumer(applyCalApp)
applyCalApp.addOutput(dob_calibrated)
applyCalApp.location = dob_ingest.location
dob_calibrated.location = dob_ingest.location
# extract local sky model (LSM) from GSM
appOid = "Extract_LSM_{0}".format(stri)
extractLSMApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
extractLSMApp.location = dob_ingest.location
dob_gsm.addConsumer(extractLSMApp)
LSM_oid = "LSM_{0}".format(stri)
dob_lsm = InMemoryDROP(LSM_oid, LSM_oid, lifespan=lifespan)
dob_lsm.location = dob_ingest.location
extractLSMApp.addOutput(dob_lsm)
# convert LSM to source.db
makesource_app_oid = "makesourcedb_{0}".format(stri)
makesource_app = SleepAndCopyApp(makesource_app_oid, makesource_app_oid)
makesource_app.location = dob_ingest.location
dob_lsm.addConsumer(makesource_app)
sourcedb_oid = "LSM_db_{0}".format(stri)
dob_lsmdb = InMemoryDROP(sourcedb_oid, sourcedb_oid, lifespan=lifespan)
dob_lsmdb.location = dob_ingest.location
makesource_app.addOutput(dob_lsmdb)
# add awimager
img_k = (i - 1) / num_subb_per_image + 1
dob_img, dob_img_prod = dob_img_dict[img_k]
dob_calibrated.addConsumer(dob_img)
dob_lsmdb.addConsumer(dob_img)
dob_img.location = dob_calibrated.location #overwrite to the last i
dob_img_prod.location = dob_img.location
return roots
def branch_failure(self, tooManyFailures):
"""
Using the container data object to implement a join/barrier dataflow.
A1, A2 and A3 are FileDROPs
B1, B2 and B3 are SumupContainerChecksum
C1, C2 and C3 are InMemoryDROPs
D is a SumupContainerChecksum
E is a InMemoryDROP
--> A1 --> B1 --> C1 --|
--> A2 --> B2 --> C2 --|--> D --> E
--> A3 --> B3 --> C3 --|
Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
after which "C" will transition to COMPLETE. Once all "C"s have moved to
COMPLETED "D"'s execution will also be triggered, and finally E will
hold the sum of B1, B2 and B3's checksums
"""
#create file data objects
a1 = InMemoryDROP('oid:A1', 'uid:A1')
a2 = InMemoryDROP('oid:A2', 'uid:A2')
a3 = InMemoryDROP('oid:A3', 'uid:A3')
# CRC Result DROPs, storing the result in memory
b1 = SumupContainerChecksum('oid:B1', 'uid:B1')
b2 = SumupContainerChecksum('oid:B2', 'uid:B2')
b3 = SumupContainerChecksum('oid:B3', 'uid:B3')
c1 = InMemoryDROP('oid:C1', 'uid:C1')
c2 = InMemoryDROP('oid:C2', 'uid:C2')
c3 = InMemoryDROP('oid:C3', 'uid:C3')
# The final DROP that sums up the CRCs from the container DROP
d = SumupContainerChecksum('oid:D', 'uid:D', input_error_threshold = 33)
e = InMemoryDROP('oid:E', 'uid:E')
# Wire together
dropAList = [a1,a2,a3]
dropBList = [b1,b2,b3]
dropCList = [c1,c2,c3]
for dropA,dropB in zip(dropAList, dropBList):
dropA.addConsumer(dropB)
for dropB,dropC in zip(dropBList, dropCList):
dropB.addOutput(dropC)
for dropC in dropCList:
dropC.addConsumer(d)
d.addOutput(e)
# Write data into the initial "A" DROPs, which should trigger
# the whole chain explained above
with DROPWaiterCtx(self, e):
#for dropA in dropAList: # this should be parallel for
a1.write(' '); a1.setCompleted()
if tooManyFailures:
a2.setError()
else:
a2.write(' '); a2.setCompleted()
a3.setError()
if tooManyFailures:
completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1]
errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e]
else:
completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e]
errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:]
for drop in completedDrops:
self.assertEqual(drop.status, DROPStates.COMPLETED)
for drop in errorDrops:
self.assertEqual(drop.status, DROPStates.ERROR)
# The results we want to compare
# (only in case that at least two branches executed)
if not tooManyFailures:
sum_crc = c1.checksum + c2.checksum
dropEData = int(droputils.allDropContents(e))
self.assertNotEqual(sum_crc, 0)
self.assertEqual(sum_crc, dropEData)
def pip_cont_img_pg():
"""
PIP continuum imaging pipeline
"""
num_beam = 1
num_time = 2
num_freq = 2
num_facet = 2
num_grid = 4
stokes = ['I', 'Q', 'U', 'V']
dob_root_sl = SocketListenerApp("FullDataset_SL", "FullDataset_SL", lifespan=lifespan)
dob_root = InMemoryDROP("FullDataset", "FullDataset", lifespan=lifespan)
dob_root_sl.addOutput(dob_root)
dob_root.location = "Buf01"
adob_sp_beam = SleepAndCopyApp("SPLT_BEAM", "SPLT_BEAM", lifespan=lifespan)
adob_sp_beam.location = "Buf01"
dob_root.addConsumer(adob_sp_beam)
for i in range(1, num_beam + 1):
uid = i
dob_beam = InMemoryDROP("BEAM_{0}".format(uid), "BEAM_{0}".format(uid), lifespan=lifespan)
adob_sp_beam.addOutput(dob_beam)
adob_sp_time = SleepAndCopyApp("SPLT_TIME_{0}".format(uid), "SPLT_TIME_{0}".format(uid), lifespan=lifespan)
dob_beam.addConsumer(adob_sp_time)
for j in range(1, num_time + 1):
uid = "%d-%d" % (i, j)
dob_time = InMemoryDROP("TIME_{0}".format(uid), "TIME_{0}".format(uid), lifespan=lifespan)
adob_sp_time.addOutput(dob_time)
adob_sp_freq = SleepAndCopyApp("SPLT_FREQ_{0}".format(uid), "SPLT_FREQ_{0}".format(uid), lifespan=lifespan)
dob_time.addConsumer(adob_sp_freq)
for k in range(1, num_freq + 1):
uid = "%d-%d-%d" % (i, j, k)
dob_freq = InMemoryDROP("FREQ_{0}".format(uid), "FREQ_{0}".format(uid), lifespan=lifespan)
adob_sp_freq.addOutput(dob_freq)
adob_sp_facet = SleepAndCopyApp("SPLT_FACET_{0}".format(uid), "SPLT_FACET_{0}".format(uid), lifespan=lifespan)
dob_freq.addConsumer(adob_sp_facet)
for l in range(1, num_facet + 1):
uid = "%d-%d-%d-%d" % (i, j, k, l)
dob_facet = InMemoryDROP("FACET_{0}".format(uid), "FACET_{0}".format(uid), lifespan=lifespan)
adob_sp_facet.addOutput(dob_facet)
adob_ph_rot = SleepAndCopyApp("PH_ROTATN_{0}".format(uid), "PH_ROTATN_{0}".format(uid), lifespan=lifespan)
dob_facet.addConsumer(adob_ph_rot)
dob_ph_rot = InMemoryDROP("PH_ROTD_{0}".format(uid), "PH_ROTD_{0}".format(uid), lifespan=lifespan)
adob_ph_rot.addOutput(dob_ph_rot)
adob_sp_stokes = SleepAndCopyApp("SPLT_STOKES_{0}".format(uid), "SPLT_STOKES_{0}".format(uid), lifespan=lifespan)
dob_ph_rot.addConsumer(adob_sp_stokes)
adob_w_kernel = SleepAndCopyApp("CACL_W_Knl_{0}".format(uid), "CACL_W_Knl_{0}".format(uid), lifespan=lifespan)
dob_facet.addConsumer(adob_w_kernel)
dob_w_knl = InMemoryDROP("W_Knl_{0}".format(uid), "W_Knl_{0}".format(uid), lifespan=lifespan)
adob_w_kernel.addOutput(dob_w_knl)
adob_a_kernel = SleepAndCopyApp("CACL_A_Knl_{0}".format(uid), "CACL_A_Knl_{0}".format(uid), lifespan=lifespan)
dob_facet.addConsumer(adob_a_kernel)
dob_a_knl = InMemoryDROP("A_Knl_{0}".format(uid), "A_Knl_{0}".format(uid), lifespan=lifespan)
adob_a_kernel.addOutput(dob_a_knl)
adob_create_kernel = SleepAndCopyApp("CREATE_Knl_{0}".format(uid), "CREATE_Knl_{0}".format(uid), lifespan=lifespan)
dob_w_knl.addConsumer(adob_create_kernel)
dob_a_knl.addConsumer(adob_create_kernel)
for stoke in stokes:
uid = "%s-%d-%d-%d-%d" % (stoke, i, j, k, l)
#print "uid = {0}".format(uid)
dob_stoke = InMemoryDROP("STOKE_{0}".format(uid), "STOKE_{0}".format(uid), lifespan=lifespan)
adob_sp_stokes.addOutput(dob_stoke)
dob_stoke.addConsumer(adob_create_kernel)
dob_aw = InMemoryDROP("A_{0}".format(uid), "A_{0}".format(uid), lifespan=lifespan)
adob_create_kernel.addOutput(dob_aw)
# we do not do loop yet
griders = []
for m in range(1, num_grid + 1):
gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m)
adob_gridding = SleepAndCopyApp("Gridding_{0}".format(gid), "Gridding_{0}".format(gid), lifespan=lifespan)
dob_stoke.addConsumer(adob_gridding)
dob_gridded_cell = InMemoryDROP("Grided_Cell_{0}".format(gid), "Grided_Cell_{0}".format(gid), lifespan=lifespan)
adob_gridding.addOutput(dob_gridded_cell)
griders.append(dob_gridded_cell)
adob_gridded_bar = SleepAndCopyApp("GRIDDED_BARRIER_{0}".format(uid), "GRIDDED_BARRIER_{0}".format(uid), lifespan=lifespan)
for grider in griders:
grider.addConsumer(adob_gridded_bar)
dob_gridded_stoke = InMemoryDROP("GRIDDED_STOKE_{0}".format(uid), "GRIDDED_STOKE_{0}".format(uid), lifespan=lifespan)
adob_gridded_bar.addOutput(dob_gridded_stoke)
FFTers = []
for m in range(1, num_grid + 1):
gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m)
adob_fft = SleepAndCopyApp("FFT_{0}".format(gid), "FFT_{0}".format(gid), lifespan=lifespan)
dob_gridded_stoke.addConsumer(adob_fft)
dob_ffted_cell = InMemoryDROP("FFTed_Cell_{0}".format(gid), "FFTed_Cell_{0}".format(gid), lifespan=lifespan)
adob_fft.addOutput(dob_ffted_cell)
FFTers.append(dob_ffted_cell)
adob_ffted_bar = SleepAndCopyApp("FFTed_BARRIER_{0}".format(uid), "FFTed_BARRIER_{0}".format(uid), lifespan=lifespan)
for ffter in FFTers:
ffter.addConsumer(adob_ffted_bar)
dob_ffted_stoke = InMemoryDROP("FFTed_STOKE_{0}".format(uid), "FFTed_STOKE_{0}".format(uid), lifespan=lifespan)
adob_ffted_bar.addOutput(dob_ffted_stoke)
cleaners = []
for m in range(1, num_grid + 1):
gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m)
adob_cleaner = SleepAndCopyApp("DECONV_{0}".format(gid), "DECONV_{0}".format(gid), lifespan=lifespan)
dob_ffted_stoke.addConsumer(adob_cleaner)
dob_cleaned_cell = InMemoryDROP("CLEANed_Cell_{0}".format(gid), "CLEANed_Cell_{0}".format(gid), lifespan=lifespan)
adob_cleaner.addOutput(dob_cleaned_cell)
cleaners.append(dob_cleaned_cell)
adob_cleaned_bar = SleepAndCopyApp("CLEANed_BARRIER_{0}".format(uid), "CLEANed_BARRIER_{0}".format(uid), lifespan=lifespan)
for cleaner in cleaners:
cleaner.addConsumer(adob_cleaned_bar)
dob_decon_stoke = InMemoryDROP("CLEANed_STOKE_{0}".format(uid), "CLEANed_STOKE_{0}".format(uid), lifespan=lifespan)
adob_cleaned_bar.addOutput(dob_decon_stoke)
adob_create_prod = SleepAndCopyApp("CRT-PROD_{0}".format(uid), "CRT-PROD_{0}".format(uid), lifespan=lifespan)
dob_decon_stoke.addConsumer(adob_create_prod)
dob_prod = InMemoryDROP("PRODUCT_{0}".format(uid), "PRODUCT_{0}".format(uid), lifespan=lifespan)
adob_create_prod.addOutput(dob_prod)
return dob_root_sl
def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastChar = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastChar = data[-1]
outputDrop.write(self._lastChar)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP("a", "a")
b = LastCharWriterApp("b", "b")
c = SumupContainerChecksum("c", "c")
d = InMemoryDROP("d", "d")
e = InMemoryDROP("e", "e")
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, lambda: a.addConsumer(b))
self.assertRaises(Exception, lambda: a.addStreamingConsumer(c))
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastChar):
self.assertEquals(aStatus, a.status)
self.assertEquals(dStatus, d.status)
self.assertEquals(eStatus, e.status)
self.assertEquals(lastChar, b._lastChar)
checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write("abcde")
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "e")
a.write("fghij")
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "j")
a.write("k")
with DROPWaiterCtx(self, [d, e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, "k")
self.assertEquals("ejk", droputils.allDropContents(d))
def test_stateMachine(self):
"""
A simple test to check that some transitions are invalid
"""
# Nice and easy
drop = InMemoryDROP("a", "a")
self.assertEquals(drop.status, DROPStates.INITIALIZED)
drop.write("a")
self.assertEquals(drop.status, DROPStates.WRITING)
drop.setCompleted()
self.assertEquals(drop.status, DROPStates.COMPLETED)
# Try to overwrite the DROP's checksum and size
self.assertRaises(Exception, lambda: setattr(drop, "checksum", 0))
self.assertRaises(Exception, lambda: setattr(drop, "size", 0))
# Try to write on a DROP that is already COMPLETED
self.assertRaises(Exception, drop.write, "")
# Invalid reading on a DROP that isn't COMPLETED yet
drop = InMemoryDROP("a", "a")
self.assertRaises(Exception, drop.open)
self.assertRaises(Exception, drop.read, 1)
self.assertRaises(Exception, drop.close, 1)
# Invalid file descriptors used to read/close
drop.setCompleted()
fd = drop.open()
otherFd = random.SystemRandom().randint(0, 1000)
self.assertNotEquals(fd, otherFd)
self.assertRaises(Exception, drop.read, otherFd)
self.assertRaises(Exception, drop.close, otherFd)
# but using the correct one should be OK
drop.read(fd)
self.assertTrue(drop.isBeingRead())
drop.close(fd)
# Expire it, then try to set it as COMPLETED again
drop.status = DROPStates.EXPIRED
self.assertRaises(Exception, drop.setCompleted)
def chiles_pg():
total_bandwidth = 480
num_obs = 8 # the same as num of data island
subband_width = 60 # MHz
num_subb = total_bandwidth / subband_width
subband_dict = collections.defaultdict(list) # for corner turning
img_list = []
start_freq = 940
# this should be removed
roots = []
for i in range(1, num_obs + 1):
stri = "%02d" % i
oid = "Obs_day_{0}".format(stri)
appOid = "Receive_" + oid
dob_obs_sl = SocketListenerApp(oid + "_SL", oid + "_SL", lifespan=lifespan, port=1110+i)
dob_obs = InMemoryDROP(oid, oid, lifespan=lifespan)
dob_obs_sl.addOutput(dob_obs)
dob_obs.location = "{0}.aws-ec2.sydney".format(i)
roots.append(dob_obs_sl)
for j in range(1, num_subb + 1):
app_oid = "mstransform_{0}_{1}".format(stri, "%02d" % j)
adob_split = SleepAndCopyApp(app_oid, app_oid, lifespan=lifespan)
adob_split.location = dob_obs.location
dob_obs.addConsumer(adob_split)
dob_sboid = "Split_{0}_{1}~{2}MHz".format(stri,
start_freq + subband_width * j,
start_freq + subband_width * (j + 1))
dob_sb = InMemoryDROP(dob_sboid, dob_sboid, lifespan=lifespan)
dob_sb.location = dob_obs.location
adob_split.addOutput(dob_sb)
subband_dict[j].append(dob_sb)
for j, v in subband_dict.items():
oid = "Subband_{0}~{1}MHz".format(start_freq + subband_width * j,
start_freq + subband_width * (j + 1))
appOid = "Combine_" + oid
dob = InMemoryDROP(oid, oid)
dob.location = "{0}.aws-ec2.sydney".format(j % num_obs)
subbandCombineAppDrop = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
subbandCombineAppDrop.addOutput(dob)
for dob_sb in v:
subbandCombineAppDrop.addInput(dob_sb)
app_oid = oid.replace("Subband_", "Clean_")
adob_clean = SleepAndCopyApp(app_oid, app_oid, lifespan=lifespan)
adob_clean.location = dob.location
dob.addConsumer(adob_clean)
img_oid = oid.replace("Subband_", "Image_")
dob_img = InMemoryDROP(img_oid, img_oid, lifespan=lifespan)
dob_img.location = dob.location
adob_clean.addOutput(dob_img)
img_list.append(dob_img)
#container + app that produces it
combineAppDrop = SleepAndCopyApp('Combine', 'Combine')
dob_comb_img_oid = "Combined_image"
dob_comb_img = InMemoryDROP(dob_comb_img_oid, dob_comb_img_oid, lifespan=lifespan)
dob_comb_img.location = "10.1.1.100:7777"
combineAppDrop.addOutput(dob_comb_img)
for dob_img in img_list:
combineAppDrop.addInput(dob_img)
#concatenate all images
adob_concat_oid = "Concat_image"
adob_concat = SleepAndCopyApp(adob_concat_oid, adob_concat_oid, lifespan=lifespan)
adob_concat.location = dob_comb_img.location
dob_comb_img.addConsumer(adob_concat)
# produce cube
dob_cube_oid = "Cube_Day{0}~{1}_{2}~{3}MHz".format(1,
num_obs,
start_freq,
start_freq + total_bandwidth)
dob_cube = InMemoryDROP(dob_cube_oid, dob_cube_oid, lifespan=lifespan)
dob_cube.location = dob_comb_img.location
adob_concat.addOutput(dob_cube)
return roots
def branch_failure(self, tooManyFailures):
"""
Using the container data object to implement a join/barrier dataflow.
A1, A2 and A3 are FileDROPs
B1, B2 and B3 are SumupContainerChecksum
C1, C2 and C3 are InMemoryDROPs
D is a SumupContainerChecksum
E is a InMemoryDROP
--> A1 --> B1 --> C1 --|
--> A2 --> B2 --> C2 --|--> D --> E
--> A3 --> B3 --> C3 --|
Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
after which "C" will transition to COMPLETE. Once all "C"s have moved to
COMPLETED "D"'s execution will also be triggered, and finally E will
hold the sum of B1, B2 and B3's checksums
"""
# create file data objects
a1 = InMemoryDROP("oid:A1", "uid:A1")
a2 = InMemoryDROP("oid:A2", "uid:A2")
a3 = InMemoryDROP("oid:A3", "uid:A3")
# CRC Result DROPs, storing the result in memory
b1 = SumupContainerChecksum("oid:B1", "uid:B1")
b2 = SumupContainerChecksum("oid:B2", "uid:B2")
b3 = SumupContainerChecksum("oid:B3", "uid:B3")
c1 = InMemoryDROP("oid:C1", "uid:C1")
c2 = InMemoryDROP("oid:C2", "uid:C2")
c3 = InMemoryDROP("oid:C3", "uid:C3")
# The final DROP that sums up the CRCs from the container DROP
d = SumupContainerChecksum("oid:D", "uid:D", input_error_threshold=33)
e = InMemoryDROP("oid:E", "uid:E")
# Wire together
dropAList = [a1, a2, a3]
dropBList = [b1, b2, b3]
dropCList = [c1, c2, c3]
for dropA, dropB in map(lambda a, b: (a, b), dropAList, dropBList):
dropA.addConsumer(dropB)
for dropB, dropC in map(lambda b, c: (b, c), dropBList, dropCList):
dropB.addOutput(dropC)
for dropC in dropCList:
dropC.addConsumer(d)
d.addOutput(e)
# Write data into the initial "A" DROPs, which should trigger
# the whole chain explained above
with DROPWaiterCtx(self, e):
# for dropA in dropAList: # this should be parallel for
a1.write(" ")
a1.setCompleted()
if tooManyFailures:
a2.setError()
else:
a2.write(" ")
a2.setCompleted()
a3.setError()
if tooManyFailures:
completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1]
errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e]
else:
completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e]
errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:]
for drop in completedDrops:
self.assertEquals(drop.status, DROPStates.COMPLETED)
for drop in errorDrops:
self.assertEquals(drop.status, DROPStates.ERROR)
# The results we want to compare
# (only in case that at least two branches executed)
if not tooManyFailures:
sum_crc = c1.checksum + c2.checksum
dropEData = int(droputils.allDropContents(e))
self.assertNotEquals(sum_crc, 0)
self.assertEquals(sum_crc, dropEData)
def test_join(self):
"""
Using the container data object to implement a join/barrier dataflow.
A1, A2 and A3 are FileDROPs
B1, B2 and B3 are SumupContainerChecksum
C1, C2 and C3 are InMemoryDROPs
D is a SumupContainerChecksum
E is a InMemoryDROP
--> A1 --> B1 --> C1 --|
--> A2 --> B2 --> C2 --|--> D --> E
--> A3 --> B3 --> C3 --|
Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
after which "C" will transition to COMPLETE. Once all "C"s have moved to
COMPLETED "D"'s execution will also be triggered, and finally E will
hold the sum of B1, B2 and B3's checksums
"""
filelen = self._test_drop_sz * ONE_MB
#create file data objects
a1 = FileDROP('oid:A1', 'uid:A1', expectedSize=filelen)
a2 = FileDROP('oid:A2', 'uid:A2', expectedSize=filelen)
a3 = FileDROP('oid:A3', 'uid:A3', expectedSize=filelen)
# CRC Result DROPs, storing the result in memory
b1 = SumupContainerChecksum('oid:B1', 'uid:B1')
b2 = SumupContainerChecksum('oid:B2', 'uid:B2')
b3 = SumupContainerChecksum('oid:B3', 'uid:B3')
c1 = InMemoryDROP('oid:C1', 'uid:C1')
c2 = InMemoryDROP('oid:C2', 'uid:C2')
c3 = InMemoryDROP('oid:C3', 'uid:C3')
# The final DROP that sums up the CRCs from the container DROP
d = SumupContainerChecksum('oid:D', 'uid:D')
e = InMemoryDROP('oid:E', 'uid:E')
# Wire together
dropAList = [a1,a2,a3]
dropBList = [b1,b2,b3]
dropCList = [c1,c2,c3]
for dropA,dropB in map(lambda a,b: (a,b), dropAList, dropBList):
dropA.addConsumer(dropB)
for dropB,dropC in map(lambda b,c: (b,c), dropBList, dropCList):
dropB.addOutput(dropC)
for dropC in dropCList:
dropC.addConsumer(d)
d.addOutput(e)
# Write data into the initial "A" DROPs, which should trigger
# the whole chain explained above
with DROPWaiterCtx(self, e):
for dropA in dropAList: # this should be parallel for
for _ in range(self._test_num_blocks):
dropA.write(self._test_block)
# All DROPs are completed now that the chain executed correctly
for drop in dropAList + dropBList + dropCList:
self.assertEqual(drop.status, DROPStates.COMPLETED)
# The results we want to compare
sum_crc = c1.checksum + c2.checksum + c3.checksum
dropEData = int(droputils.allDropContents(e))
self.assertNotEqual(sum_crc, 0)
self.assertEqual(sum_crc, dropEData)
def mwa_fornax_pg():
num_coarse_ch = 24
num_split = 3 # number of time splits per channel
dob_root_sl = SocketListenerApp("MWA_LTA_SL", "MWA_LTA_SL", lifespan=lifespan)
dob_root = InMemoryDROP("MWA_LTA", "MWA_LTA", lifespan=lifespan)
dob_root.location = "Pawsey"
dob_root_sl.addOutput(dob_root)
#container
dob_comb_img_oid = "Combined_image"
combineImgAppOid = "Combine image"
dob_comb_img = InMemoryDROP(dob_comb_img_oid, dob_comb_img_oid, lifespan=lifespan)
combineImgApp = SleepAndCopyApp(combineImgAppOid, combineImgAppOid)
combineImgApp.addOutput(dob_comb_img)
dob_comb_img.location = "f032.fornax"
for i in range(1, num_coarse_ch + 1):
stri = "%02d" % i
oid = "Subband_{0}".format(stri)
appOid = "Combine_" + oid
dob_obs = InMemoryDROP(oid, oid, lifespan=lifespan)
combineSubbandApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
combineSubbandApp.addOutput(dob_obs)
dob_obs.location = "f%03d.fornax" % i
oid_ingest = "NGAS_{0}".format(stri)
dob_ingest = SleepAndCopyApp(oid_ingest, oid_ingest, lifespan=lifespan)
dob_ingest.location = "f%03d.fornax:7777" % i
dob_root.addConsumer(dob_ingest)
for j in range(1, num_split + 1):
strj = "%02d" % j
split_oid = "Subband_{0}_Split_{1}".format(stri, strj)
dob_split = InMemoryDROP(split_oid, split_oid, lifespan=lifespan)
dob_split.location = dob_obs.location
dob_ingest.addOutput(dob_split)
combineSubbandApp.addInput(dob_split)
oid_rts = "RTS_{0}".format(stri)
dob_rts = SleepAndCopyApp(oid_rts, oid_rts, lifespan=lifespan)
dob_rts.location = dob_obs.location
dob_obs.addConsumer(dob_rts)
oid_subimg = "Subcube_{0}".format(stri)
dob_subimg = InMemoryDROP(oid_subimg, oid_subimg, lifespan=lifespan)
dob_subimg.location = dob_obs.location
dob_rts.addOutput(dob_subimg)
combineImgApp.addInput(dob_subimg)
#concatenate all images
adob_concat_oid = "Concat_image"
adob_concat = SleepAndCopyApp(adob_concat_oid, adob_concat_oid, lifespan=lifespan)
adob_concat.location = dob_comb_img.location
dob_comb_img.addConsumer(adob_concat)
# produce cube
dob_cube_oid = "Cube_30.72MHz"
dob_cube = InMemoryDROP(dob_cube_oid, dob_cube_oid, lifespan=lifespan)
dob_cube.location = dob_comb_img.location
adob_concat.addOutput(dob_cube)
return dob_root_sl
def _createGraph(self):
"""
Creates the following graph of DROPs:
A |--> B ----> D --> G --> I --|
|--> C -|--> E --------------|-> H --> J
|--> F
B, C, G and H are AppDOs. The names have been given in breadth-first
order (although H has a dependency on I)
"""
a = InMemoryDROP('a', 'a')
b = BarrierAppDROP('b', 'b')
c = BarrierAppDROP('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
f = InMemoryDROP('f', 'f')
g = BarrierAppDROP('g', 'g')
h = BarrierAppDROP('h', 'h')
i = InMemoryDROP('i', 'i')
j = InMemoryDROP('j', 'j')
a.addConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
c.addOutput(f)
d.addConsumer(g)
e.addConsumer(h)
g.addOutput(i)
i.addConsumer(h)
h.addOutput(j)
return a, b, c, d, e, f, g, h, i, j
def createDrop(self, oid, uid, **kwargs):
return InMemoryDROP(oid, uid, **kwargs)
self._evt = threading.Event()
drop.addConsumer(self)
def dropCompleted(self, drop, state):
self._evt.set()
def wait(self, timeout=None):
return self._evt.wait(timeout)
if __name__ == '__main__':
try:
vis_in_a = []
flux_out = InMemoryDROP(uuid.uuid1(), uuid.uuid1())
flux = SourceFlux(uuid.uuid1(), uuid.uuid1(), casapy_path=CASAPY)
cl = Clean(uuid.uuid1(),
uuid.uuid1(),
field='deepfield',
mode='frequency',
restfreq='1420.405752MHz',
nchan=-1,
start='',
width='',
interpolation='nearest',
gain=0.1,
imsize=[256],
cell=['1.0arcsec'],
def test_simple_chain(self):
"""
Simple test that creates a pipeline-like chain of commands.
In this case we simulate a pipeline that does this, holding
each intermediate result in memory:
cat someFile | grep 'a' | sort | rev
"""
class GrepResult(BarrierAppDROP):
def initialize(self, **kwargs):
super(GrepResult, self).initialize(**kwargs)
self._substring = kwargs["substring"]
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = StringIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
if self._substring in line:
output.write(line)
class SortResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
sortedLines = StringIO(droputils.allDropContents(drop)).readlines()
sortedLines.sort()
for line in sortedLines:
output.write(line)
class RevResult(BarrierAppDROP):
def run(self):
drop = self.inputs[0]
output = self.outputs[0]
allLines = StringIO(droputils.allDropContents(drop)).readlines()
for line in allLines:
buf = ""
for c in line:
if c == " " or c == "\n":
output.write(buf[::-1])
output.write(c)
buf = ""
else:
buf += c
a = InMemoryDROP("oid:A", "uid:A")
b = GrepResult("oid:B", "uid:B", substring="a")
c = InMemoryDROP("oid:C", "uid:C")
d = SortResult("oid:D", "uid:D")
e = InMemoryDROP("oid:E", "uid:E")
f = RevResult("oid:F", "oid:F")
g = InMemoryDROP("oid:G", "uid:G")
a.addConsumer(b)
b.addOutput(c)
c.addConsumer(d)
d.addOutput(e)
e.addConsumer(f)
f.addOutput(g)
# Initial write
contents = "first line\nwe have an a here\nand another one\nnoone knows me"
cResExpected = "we have an a here\nand another one\n"
eResExpected = "and another one\nwe have an a here\n"
gResExpected = "dna rehtona eno\new evah na a ereh\n"
with DROPWaiterCtx(self, g):
a.write(contents)
a.setCompleted()
# Get intermediate and final results and compare
actualRes = []
for i in [c, e, g]:
actualRes.append(droputils.allDropContents(i))
map(lambda x, y: self.assertEquals(x, y), [cResExpected, eResExpected, gResExpected], actualRes)