def test_eager_inputFired_app(self): """ Tests that InputFiredApps works as expected """ # No n_effective_inputs given self.assertRaises(ValueError, InputFiredAppDROP, "a", "a") # Invalid values self.assertRaises(ValueError, InputFiredAppDROP, "a", "a", n_effective_inputs=-2) self.assertRaises(ValueError, InputFiredAppDROP, "a", "a", n_effective_inputs=0) # More effective inputs than inputs a = InMemoryDROP("b", "b") b = InputFiredAppDROP("a", "a", n_effective_inputs=2) b.addInput(a) self.assertRaises(Exception, a.setCompleted) # 2 effective inputs, 4 outputs. Trigger 2 inputs and make sure the # app has run a, b, c, d = [InMemoryDROP(str(i), str(i)) for i in xrange(4)] e = InputFiredAppDROP("e", "e", n_effective_inputs=2) map(lambda x: e.addInput(x), [a, b, c, d]) with DROPWaiterCtx(self, e, 5): a.setCompleted() b.setCompleted() self.assertEquals(AppDROPStates.FINISHED, e.execStatus) self.assertEquals(DROPStates.COMPLETED, a.status) self.assertEquals(DROPStates.COMPLETED, b.status) self.assertEquals(DROPStates.INITIALIZED, c.status) self.assertEquals(DROPStates.INITIALIZED, d.status)
def test_eager_inputFired_app(self): """ Tests that InputFiredApps works as expected """ # No n_effective_inputs given self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a') # Invalid values self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=-2) self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=0) # More effective inputs than inputs a = InMemoryDROP('b', 'b') b = InputFiredAppDROP('a', 'a', n_effective_inputs=2) b.addInput(a) self.assertRaises(Exception, a.setCompleted) # 2 effective inputs, 4 outputs. Trigger 2 inputs and make sure the # app has run a,b,c,d = [InMemoryDROP(str(i), str(i)) for i in range(4)] e = InputFiredAppDROP('e', 'e', n_effective_inputs=2) for x in a,b,c,d: e.addInput(x) with DROPWaiterCtx(self, e, 5): a.setCompleted() b.setCompleted() self.assertEqual(AppDROPStates.FINISHED, e.execStatus) self.assertEqual(DROPStates.COMPLETED, a.status) self.assertEqual(DROPStates.COMPLETED, b.status) self.assertEqual(DROPStates.INITIALIZED, c.status) self.assertEqual(DROPStates.INITIALIZED, d.status)
def test_app_multiple_outputs(self): """ A small method that tests that the AppDROPs writing to two different DROPs outputs works The graph constructed by this example looks as follow: |--> E A --> B --> C --> D --| |--> F Here B and D are an AppDROPs, with D writing to two DROPs outputs (E and F) and reading from C. C, in turn, is written by B, which in turns reads the data from A """ # This is used as "B" class NumberWriterApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] output = self.outputs[0] howMany = int(droputils.allDropContents(inputDrop)) for i in xrange(howMany): output.write(str(i) + " ") # This is used as "D" class OddAndEvenContainerApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] outputs = self.outputs numbers = droputils.allDropContents(inputDrop).strip().split() for n in numbers: outputs[int(n) % 2].write(n + " ") # Create DROPs a = InMemoryDROP("oid:A", "uid:A") b = NumberWriterApp("oid:B", "uid:B") c = InMemoryDROP("oid:A", "uid:A") d = OddAndEvenContainerApp("oid:D", "uid:D") e = InMemoryDROP("oid:E", "uid:E") f = InMemoryDROP("oid:F", "uid:F") # Wire them together a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) d.addOutput(f) # Start the execution with DROPWaiterCtx(self, [e, f]): a.write("20") a.setCompleted() # Check the final results are correct for drop in [a, b, c, d, e]: self.assertEquals(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop)) self.assertEquals("0 2 4 6 8 10 12 14 16 18", droputils.allDropContents(e).strip()) self.assertEquals("1 3 5 7 9 11 13 15 17 19", droputils.allDropContents(f).strip())
def _test_graphExecutionDriver(self, mode): """ A small test to check that DROPs executions can be driven externally if required, and not always internally by themselves """ a = InMemoryDROP('a', 'a', executionMode=mode, expectedSize=1) b = SumupContainerChecksum('b', 'b') c = InMemoryDROP('c', 'c') a.addConsumer(b) c.addProducer(b) # Write and check dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c] with DROPWaiterCtx(self, dropsToWaitFor): a.write('1') if mode == ExecutionMode.EXTERNAL: # b hasn't been triggered self.assertEqual(c.status, DROPStates.INITIALIZED) self.assertEqual(b.status, DROPStates.INITIALIZED) self.assertEqual(b.execStatus, AppDROPStates.NOT_RUN) # Now let b consume a with DROPWaiterCtx(self, [c]): b.dropCompleted('a', DROPStates.COMPLETED) self.assertEqual(c.status, DROPStates.COMPLETED) elif mode == ExecutionMode.DROP: # b is already done self.assertEqual(c.status, DROPStates.COMPLETED)
def test_addDropInstances(self): a1 = InMemoryDROP('a', 'a1') a2 = InMemoryDROP('a', 'a2') registry = RDBMSRegistry('sqlite3', DBFILE) registry.addDrop(a1) uids = registry.getDropUids(a1) self.assertEqual(1, len(uids)) self.assertEqual('a1', uids[0]) registry.addDropInstance(a2) uids = registry.getDropUids(a1) uids.sort() self.assertEqual(2, len(uids)) self.assertEqual('a1', uids[0]) self.assertEqual('a2', uids[1]) # Check accessing the database separately conn = sqlite3.connect(DBFILE) # @UndefinedVariable cur = conn.cursor() cur.execute("SELECT uid FROM dfms_dropinstance WHERE oid = 'a'") rows = cur.fetchall() self.assertEqual(2, len(rows)) uids = [r[0] for r in rows] uids.sort() self.assertEqual(2, len(uids)) self.assertEqual('a1', uids[0]) self.assertEqual('a2', uids[1]) cur.close() conn.close()
def container_pg(): ''' Creates the following graph: |--> B --> D --| SL --> A --| |--> F --> G --> H --> I |--> C --> E --| ''' sl= SocketListenerApp('SL', 'SL', lifespan=lifespan) a = InMemoryDROP('A', 'A', lifespan=lifespan) b = SleepAndCopyApp('B', 'B', lifespan=lifespan) c = SleepAndCopyApp('C', 'C', lifespan=lifespan) d = InMemoryDROP('D', 'D', lifespan=lifespan) e = InMemoryDROP('E', 'E', lifespan=lifespan) f = SimpleBarrierApp('F', 'F', lifespan=lifespan) g = ContainerDROP('G', 'G', lifespan=lifespan) h = SleepAndCopyApp('H', 'H', lifespan=lifespan) i = InMemoryDROP('I', 'I', lifespan=lifespan) # Wire together sl.addOutput(a) a.addConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) d.addConsumer(f) e.addConsumer(f) f.addOutput(g) g.addChild(d) g.addChild(e) g.addConsumer(h) h.addOutput(i) return sl
def test_invalid(self): # Shouldn't allow inputs a = SocketListenerApp('a', 'a', port=1) a.addOutput(InMemoryDROP('c', 'c')) self.assertRaises(Exception, a.addInput, InMemoryDROP('b', 'b')) self.assertRaises(Exception, a.addStreamingInput, InMemoryDROP('b', 'b')) # Shouldn't be able to open ports <= 1024 a.execute() self.assertEqual(a.status, DROPStates.ERROR)
def _test_graphExecutionDriver(self, mode): """ A small test to check that DROPs executions can be driven externally if required, and not always internally by themselves """ a = InMemoryDROP("a", "a", executionMode=mode, expectedSize=1) b = SumupContainerChecksum("b", "b") c = InMemoryDROP("c", "c") a.addConsumer(b) c.addProducer(b) # Write and check dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c] with DROPWaiterCtx(self, dropsToWaitFor): a.write("1") if mode == ExecutionMode.EXTERNAL: # b hasn't been triggered self.assertEquals(c.status, DROPStates.INITIALIZED) self.assertEquals(b.status, DROPStates.INITIALIZED) self.assertEquals(b.execStatus, AppDROPStates.NOT_RUN) # Now let b consume a with DROPWaiterCtx(self, [c]): b.dropCompleted("a", DROPStates.COMPLETED) self.assertEquals(c.status, DROPStates.COMPLETED) elif mode == ExecutionMode.DROP: # b is already done self.assertEquals(c.status, DROPStates.COMPLETED)
def test_pg(): """ A very simple graph that looks like this: A --> B --> C --> D """ a = SocketListenerApp('A', 'A', lifespan=lifespan) b = InMemoryDROP('B', 'B', lifespan=lifespan) c = SleepAndCopyApp('C', 'C', lifespan=lifespan) d = InMemoryDROP('D', 'D', lifespan=lifespan) a.addOutput(b) b.addConsumer(c) c.addOutput(d) return a
def test_speadApp(self): port = 1111 itemId = 0x2000 thread_pool = spead2.ThreadPool() self._stream = spead2.send.UdpStream( thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7)) a = SpeadReceiverApp('a', 'a', port=port, itemId=itemId) b = InMemoryDROP('b', 'b') a.addOutput(b) size = 1024 threading.Thread(target=lambda: a.execute()).start() time.sleep(1) msg = os.urandom(size) with DROPWaiterCtx(self, b, timeout=1): ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48)) item = ig.add_item(itemId, 'main_data', 'a char array', shape=(size, ), format=[('c', 8)]) item.value = msg self._stream.send_heap(ig.get_heap()) self._stream.send_heap(ig.get_end()) for drop in a, b: self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual(size, b.size) self.assertEqual(msg, droputils.allDropContents(b))
def test_multipleProducers(self): """ A test that checks that multiple-producers correctly drive the state of their shared output """ class App(BarrierAppDROP): pass a,b,c,d,e = [App(chr(ord('A') + i), chr(ord('A') + i)) for i in range(5)] f = InMemoryDROP('F', 'F') for drop in a,b,c,d,e: drop.addOutput(f) self.assertEqual(DROPStates.INITIALIZED, f.status) for drop in a,b,c,d,e: self.assertEqual(AppDROPStates.NOT_RUN, drop.execStatus) # Run the first 4 ones, F should still be in INITIALIZED for drop in a,b,c,d: drop.execute() self.assertEqual(DROPStates.INITIALIZED, f.status) self.assertEqual(AppDROPStates.NOT_RUN, e.execStatus) for drop in a,b,c,d: self.assertEqual(AppDROPStates.FINISHED, drop.execStatus) # Run the final one, now F should be COMPLETED e.execute() self.assertEqual(DROPStates.COMPLETED, f.status) for drop in a,b,c,d,e: self.assertEqual(AppDROPStates.FINISHED, drop.execStatus)
def test_errorState(self): a = InMemoryDROP('a', 'a') b = SumupContainerChecksum('b', 'b') c = InMemoryDROP('c', 'c') c.addProducer(b) b.addInput(a) a.setError() self.assertEqual(DROPStates.ERROR, a.status) self.assertEqual(DROPStates.ERROR, b.status) self.assertEqual(DROPStates.ERROR, c.status)
def test_dropAccess(self): a1 = InMemoryDROP('a', 'a1') registry = RDBMSRegistry('sqlite3', DBFILE) registry.addDrop(a1) self.assertEqual(-1, registry.getLastAccess('a')) registry.recordNewAccess('a') self.assertNotEqual(-1, registry.getLastAccess('a'))
def test_two_simultaneous_pipes(self): """ A more complicated test where three bash applications run at the same time. The first streams its output to the second one, while the second one streams *its* output to the third one. ------------- -------------- ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F | | echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | | -----*------- -------------- ----*--*----- -------------- -----*------- ---------- | | | | \-------------|named-pipe|----------\ \-----------|named-pipe|-----------/ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by "cat" (BashApp C). The printed results (a copy of the original) are streamed through D and read by "sort" (BashApp E), which writes the output to F. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp('a', 'a', command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP('b', 'b') c = StreamingInputOutputBashApp('c', 'c', command="cat") d = InMemoryDROP('d', 'd') e = StreamingInputBashApp('e', 'e', command="sort -n > %o0") f = FileDROP('f', 'f', filepath=output_fname) a.addOutput(b) b.addStreamingConsumer(c) c.addOutput(d) d.addStreamingConsumer(e) e.addOutput(f) # Let's fire the app with DROPWaiterCtx(self, f, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d, e, f): self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual([1, 2, 3, 4, 5], [ int(x) for x in droputils.allDropContents(f).strip().split(six.b('\n')) ]) # Clean up and go os.remove(output_fname)
def test_addDrop(self): a = InMemoryDROP('a', 'a') registry = RDBMSRegistry('sqlite3', DBFILE) registry.addDrop(a) conn = sqlite3.connect(DBFILE) # @UndefinedVariable cur = conn.cursor() cur.execute('SELECT oid FROM dfms_drop') r = cur.fetchone() self.assertEqual(1, len(r)) self.assertEqual('a', r[0]) cur.close() conn.close()
def test_socket_listener(self): ''' A simple test to check that SocketListenerApps are indeed working as expected; that is, they write the data they receive into their output, and finish when the connection is closed from the client side The data flow diagram looks like this: A --> B --> C --> D ''' host = 'localhost' port = 9933 data = 'shine on you crazy diamond' a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port) b = InMemoryDROP('oid:B', 'uid:B') c = SumupContainerChecksum('oid:C', 'uid:C') d = InMemoryDROP('oid:D', 'uid:D') a.addOutput(b) b.addConsumer(c) c.addOutput(d) # Create the socket, write, and close the connection, allowing # A to move to COMPLETED with DROPWaiterCtx(self, d, 3): # That's plenty of time threading.Thread(target=lambda a: a.execute(), args=(a,)).start() utils.writeToRemotePort(host, port, data, 1) for drop in [a,b,c,d]: self.assertEquals(DROPStates.COMPLETED, drop.status) # Our expectations are fulfilled! bContents = droputils.allDropContents(b) dContents = int(droputils.allDropContents(d)) self.assertEquals(data, bContents) self.assertEquals(crc32(data, 0), dContents)
def test_socket_listener(self): ''' A simple test to check that SocketListenerApps are indeed working as expected; that is, they write the data they receive into their output, and finish when the connection is closed from the client side The data flow diagram looks like this: A --> B --> C --> D ''' host = 'localhost' port = 9933 data = os.urandom(1025) a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port) b = InMemoryDROP('oid:B', 'uid:B') c = SumupContainerChecksum('oid:C', 'uid:C') d = InMemoryDROP('oid:D', 'uid:D') a.addOutput(b) b.addConsumer(c) c.addOutput(d) # Create the socket, write, and close the connection, allowing # A to move to COMPLETED with DROPWaiterCtx(self, d, 3): # That's plenty of time a.async_execute() utils.write_to(host, port, data, 1) for drop in [a,b,c,d]: self.assertEqual(DROPStates.COMPLETED, drop.status) # Our expectations are fulfilled! bContents = droputils.allDropContents(b) dContents = int(droputils.allDropContents(d)) self.assertEqual(data, bContents) self.assertEqual(crc32(data, 0), dContents)
def test_errorState(self): a = InMemoryDROP("a", "a") b = SumupContainerChecksum("b", "b") c = InMemoryDROP("c", "c") c.addProducer(b) b.addInput(a) a.setError() self.assertEquals(DROPStates.ERROR, a.status) self.assertEquals(DROPStates.ERROR, b.status) self.assertEquals(DROPStates.ERROR, c.status)
def _testGraph(execMode): """ A test graph that looks like this: |--> B1 --> C1 --| |--> B2 --> C2 --| SL_A --> A --|--> B3 --> C3 --| --> D --> E |--> .. --> .. --| |--> BN --> CN --| B and C DROPs are InMemorySleepAndCopyApp DROPs (see above). D is simply a container. A is a socket listener, so we can actually write to it externally and watch the progress of the luigi tasks. We give DROPs a long lifespan; otherwise they will expire and luigi will see it as a failed task (which is actually right!) If execMode is given we use that in all DROPs. If it's None we use a mixture of DROP/EXTERNAL execution modes. """ aMode = execMode if execMode is not None else ExecutionMode.EXTERNAL bMode = execMode if execMode is not None else ExecutionMode.DROP cMode = execMode if execMode is not None else ExecutionMode.DROP dMode = execMode if execMode is not None else ExecutionMode.EXTERNAL eMode = execMode if execMode is not None else ExecutionMode.EXTERNAL sl_a = SocketListenerApp('oid:SL_A', 'uid:SL_A', executionMode=aMode, lifespan=lifespan) a = InMemoryDROP('oid:A', 'uid:A', executionMode=aMode, lifespan=lifespan) d = SumupContainerChecksum('oid:D', 'uid:D', executionMode=dMode, lifespan=lifespan) e = InMemoryDROP('oid:E', 'uid:E', executionMode=eMode, lifespan=lifespan) sl_a.addOutput(a) e.addProducer(d) for i in xrange(random.SystemRandom().randint(10, 20)): b = SleepAndCopyApp('oid:B%d' % (i), 'uid:B%d' % (i), executionMode=bMode, lifespan=lifespan) c = InMemoryDROP('oid:C%d' % (i), 'uid:C%d' % (i), executionMode=cMode, lifespan=lifespan) a.addConsumer(b) b.addOutput(c) c.addConsumer(d) return sl_a
def test_single_pipe(self): """ A simple test where two bash apps are connected to each other in a streaming fashion. The data flows through a pipe which is created by the framework. The data drop in between acts only as a intermediator to establish the underlying communication channel. ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | File D | | echo | | "/a/pipe" | | dc | | | -----*------- -------------- ------*------ ---------- | | \-------------|named-pipe|------------/ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by cat and redirected to D. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp('a', 'a', command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP('b', 'b') c = StreamingInputBashApp('c', 'c', command="cat > %o0") d = FileDROP('d', 'd', filepath=output_fname) a.addOutput(b) c.addStreamingInput(b) c.addOutput(d) # Let's fire the app with DROPWaiterCtx(self, d, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d): self.assertEqual(DROPStates.COMPLETED, drop.status, "Drop %r not COMPLETED: %d" % (drop, drop.status)) self.assertEqual( [5, 4, 3, 2, 1], [int(x) for x in droputils.allDropContents(d).split(six.b('\n'))]) # Clean up and go os.remove(output_fname)
def test_directoryContainer(self): """ A small, simple test for the DirectoryContainer DROP that checks it allows only valid children to be added """ # Prepare our playground cwd = os.getcwd() os.chdir('/tmp') dirname = ".hidden" dirname2 = ".hidden/inside" if not os.path.exists(dirname2): os.makedirs(dirname2) # DROPs involved a = FileDROP('a', 'a', dirname=dirname) b = FileDROP('b', 'b', dirname=dirname) c = FileDROP('c', 'c', dirname=dirname2) d = FileDROP('d', 'd', dirname=dirname2) cont1 = DirectoryContainer('e', 'e', dirname=dirname) cont2 = DirectoryContainer('f', 'f', dirname=dirname2) # Paths are absolutely reported self.assertEqual(os.path.realpath('/tmp/.hidden'), os.path.realpath(cont1.path)) self.assertEqual(os.path.realpath('/tmp/.hidden/inside'), os.path.realpath(cont2.path)) # Certain children-to-be are rejected self.assertRaises(TypeError, cont1.addChild, NullDROP('g', 'g')) self.assertRaises(TypeError, cont1.addChild, InMemoryDROP('h', 'h')) self.assertRaises(TypeError, cont1.addChild, ContainerDROP('i', 'i')) self.assertRaises(Exception, cont1.addChild, c) self.assertRaises(Exception, cont1.addChild, d) self.assertRaises(Exception, cont2.addChild, a) self.assertRaises(Exception, cont2.addChild, b) # These children are correct cont1.addChild(a) cont1.addChild(b) cont2.addChild(c) cont2.addChild(d) # Revert to previous state shutil.rmtree(dirname, True) os.chdir(cwd)
def _test_write_withDropType(self, dropType): """ Test an AbstractDROP and a simple AppDROP (for checksum calculation) """ a = dropType('oid:A', 'uid:A', expectedSize = self._test_drop_sz * ONE_MB) b = SumupContainerChecksum('oid:B', 'uid:B') c = InMemoryDROP('oid:C', 'uid:C') b.addInput(a) b.addOutput(c) test_crc = 0 with DROPWaiterCtx(self, c): for _ in range(self._test_num_blocks): a.write(self._test_block) test_crc = crc32(self._test_block, test_crc) # Read the checksum from c cChecksum = int(droputils.allDropContents(c)) self.assertNotEqual(a.checksum, 0) self.assertEqual(a.checksum, test_crc) self.assertEqual(cChecksum, test_crc)
def test_sql_create(self): sqlite01 = get_oid('sqlite') sqlite_drop = FileDROP( sqlite01, get_uuid(), precious=False, dirname=os.path.join(TestSqlite._temp, sqlite01), check_exists=False, ) initialize_sqlite = InitializeSqliteApp( get_oid('app'), get_uuid(), user='******', ) sqlite_in_memory = InMemoryDROP( get_oid('memory'), get_uuid(), precious=False, ) initialize_sqlite.addInput(sqlite_drop) initialize_sqlite.addOutput(sqlite_in_memory) with droputils.DROPWaiterCtx(self, sqlite_in_memory, 50000): sqlite_drop.setCompleted()
def test_app_multiple_outputs(self): """ A small method that tests that the AppDROPs writing to two different DROPs outputs works The graph constructed by this example looks as follow: |--> E A --> B --> C --> D --| |--> F Here B and D are an AppDROPs, with D writing to two DROPs outputs (E and F) and reading from C. C, in turn, is written by B, which in turns reads the data from A """ # This is used as "B" class NumberWriterApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] output = self.outputs[0] howMany = int(droputils.allDropContents(inputDrop)) for i in range(howMany): output.write(str(i) + " ") # This is used as "D" class OddAndEvenContainerApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] outputs = self.outputs numbers = droputils.allDropContents(inputDrop).strip().split() for n in numbers: outputs[int(n) % 2].write(n + six.b(" ")) # Create DROPs a = InMemoryDROP('oid:A', 'uid:A') b = NumberWriterApp('oid:B', 'uid:B') c = InMemoryDROP('oid:A', 'uid:A') d = OddAndEvenContainerApp('oid:D', 'uid:D') e = InMemoryDROP('oid:E', 'uid:E') f = InMemoryDROP('oid:F', 'uid:F') # Wire them together a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) d.addOutput(f) # Start the execution with DROPWaiterCtx(self, [e,f]): a.write('20') a.setCompleted() # Check the final results are correct for drop in [a,b,c,d,e]: self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop)) self.assertEqual(six.b("0 2 4 6 8 10 12 14 16 18"), droputils.allDropContents(e).strip()) self.assertEqual(six.b("1 3 5 7 9 11 13 15 17 19"), droputils.allDropContents(f).strip())
def test_objectAsNormalAndStreamingInput(self): """ A test that checks that a DROP can act as normal and streaming input of different AppDROPs at the same time. We use the following graph: A --|--> B --> D |--> C --> E Here B uses A as a streaming input, while C uses it as a normal input """ class LastCharWriterApp(AppDROP): def initialize(self, **kwargs): super(LastCharWriterApp, self).initialize(**kwargs) self._lastByte = None def dataWritten(self, uid, data): self.execStatus = AppDROPStates.RUNNING outputDrop = self.outputs[0] self._lastByte = six.indexbytes(data, -1) outputDrop.write(self._lastByte) def dropCompleted(self, uid, status): self.execStatus = AppDROPStates.FINISHED self._notifyAppIsFinished() a = InMemoryDROP('a', 'a') b = LastCharWriterApp('b', 'b') c = SumupContainerChecksum('c', 'c') d = InMemoryDROP('d', 'd') e = InMemoryDROP('e', 'e') a.addStreamingConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) # Consumer cannot be normal and streaming at the same time self.assertRaises(Exception, a.addConsumer, b) self.assertRaises(Exception, a.addStreamingConsumer, c) # Write a little, then check the consumers def checkDropStates(aStatus, dStatus, eStatus, lastByte): self.assertEqual(aStatus, a.status) self.assertEqual(dStatus, d.status) self.assertEqual(eStatus, e.status) if lastByte is not None: self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte)) checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None) a.write('abcde') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e') a.write('fghij') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j') a.write('k') with DROPWaiterCtx(self, [d,e]): a.setCompleted() checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k') self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
def test_stateMachine(self): """ A simple test to check that some transitions are invalid """ # Nice and easy drop = InMemoryDROP('a', 'a') self.assertEqual(drop.status, DROPStates.INITIALIZED) drop.write('a') self.assertEqual(drop.status, DROPStates.WRITING) drop.setCompleted() self.assertEqual(drop.status, DROPStates.COMPLETED) # Try to overwrite the DROP's checksum and size self.assertRaises(Exception, lambda: setattr(drop, 'checksum', 0)) self.assertRaises(Exception, lambda: setattr(drop, 'size', 0)) # Try to write on a DROP that is already COMPLETED self.assertRaises(Exception, drop.write, '') # Invalid reading on a DROP that isn't COMPLETED yet drop = InMemoryDROP('a', 'a') self.assertRaises(Exception, drop.open) self.assertRaises(Exception, drop.read, 1) self.assertRaises(Exception, drop.close, 1) # Invalid file descriptors used to read/close drop.setCompleted() fd = drop.open() otherFd = random.SystemRandom().randint(0, 1000) self.assertNotEqual(fd, otherFd) self.assertRaises(Exception, drop.read, otherFd) self.assertRaises(Exception, drop.close, otherFd) # but using the correct one should be OK drop.read(fd) self.assertTrue(drop.isBeingRead()) drop.close(fd) # Expire it, then try to set it as COMPLETED again drop.status = DROPStates.EXPIRED self.assertRaises(Exception, drop.setCompleted)
def test_simple_chain(self): ''' Simple test that creates a pipeline-like chain of commands. In this case we simulate a pipeline that does this, holding each intermediate result in memory: cat someFile | grep 'a' | sort | rev ''' class GrepResult(BarrierAppDROP): def initialize(self, **kwargs): super(GrepResult, self).initialize(**kwargs) self._substring = six.b(kwargs['substring']) def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = BytesIO(droputils.allDropContents(drop)).readlines() for line in allLines: if self._substring in line: output.write(line) class SortResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] sortedLines = BytesIO(droputils.allDropContents(drop)).readlines() sortedLines.sort() for line in sortedLines: output.write(line) class RevResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] allbytes = droputils.allDropContents(drop) buf = bytearray() for c in allbytes: if c == six.b(' ') or c == six.b('\n'): output.write(buf[::-1]) output.write(c) buf = bytearray() else: buf.append(c) a = InMemoryDROP('oid:A', 'uid:A') b = GrepResult('oid:B', 'uid:B', substring="a") c = InMemoryDROP('oid:C', 'uid:C') d = SortResult('oid:D', 'uid:D') e = InMemoryDROP('oid:E', 'uid:E') f = RevResult('oid:F', 'oid:F') g = InMemoryDROP('oid:G', 'uid:G') a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) e.addConsumer(f) f.addOutput(g) # Initial write contents = "first line\nwe have an a here\nand another one\nnoone knows me" cResExpected = "we have an a here\nand another one\n" eResExpected = "and another one\nwe have an a here\n" gResExpected = "dna rehtona eno\new evah na a ereh\n" with DROPWaiterCtx(self, g): a.write(contents) a.setCompleted() # Get intermediate and final results and compare actualRes = [] for i in [c, e, g]: actualRes.append(droputils.allDropContents(i)) map(lambda x, y: self.assertEqual(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
def _createGraph(self): """ Creates the following graph of DROPs: A |--> B ----> D --> G --> I --| |--> C -|--> E --------------|-> H --> J |--> F B, C, G and H are AppDOs. The names have been given in breadth-first order (although H has a dependency on I) """ a = InMemoryDROP('a', 'a') b = BarrierAppDROP('b', 'b') c = BarrierAppDROP('c', 'c') d = InMemoryDROP('d', 'd') e = InMemoryDROP('e', 'e') f = InMemoryDROP('f', 'f') g = BarrierAppDROP('g', 'g') h = BarrierAppDROP('h', 'h') i = InMemoryDROP('i', 'i') j = InMemoryDROP('j', 'j') a.addConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) c.addOutput(f) d.addConsumer(g) e.addConsumer(h) g.addOutput(i) i.addConsumer(h) h.addOutput(j) return a, b, c, d, e, f, g, h, i, j
def lofar_standard_pip_pg(): """ Lofar simple imaging pipeline based on Ger's email and https://github.com/lofar-astron/lofar-profiling """ roots = [] num_time_slice = 3 total_num_subband = 8 num_subb_per_image = 2 Cal_model_oid = "A-team/Calibrator" dob_cal_model = InMemoryDROP(Cal_model_oid, Cal_model_oid, lifespan=lifespan) dob_cal_model.location = "catalogue.Groningen" roots.append(dob_cal_model) make_source_app_oid = "makesourcedb" make_source_app = SleepAndCopyApp(make_source_app_oid, make_source_app_oid) make_source_app.location = dob_cal_model.location dob_cal_model.addConsumer(make_source_app) source_db_oid = "CalSource.db" dob_source_db = InMemoryDROP(source_db_oid, source_db_oid, lifespan=lifespan) dob_source_db.location = dob_cal_model.location make_source_app.addOutput(dob_source_db) GSM_oid = "GlobalSkyModel" dob_gsm = InMemoryDROP(GSM_oid, GSM_oid, lifespan=lifespan) dob_gsm.location = "catalogue.Groningen" roots.append(dob_gsm) dob_img_dict = dict() sb_chunks = chunks(range(1, total_num_subband + 1), num_subb_per_image) for k, img_list in enumerate(sb_chunks): imger_oid = "AWImager_SB_{0}~{1}".format(img_list[0], img_list[-1]) dob_img = SleepAndCopyApp(imger_oid, imger_oid, lifespan=lifespan) kk = k + 1 #print k, img_list, kk img_prod_oid = "Image_SB_{0}~{1}".format(img_list[0], img_list[-1]) dob_img_prod = InMemoryDROP(img_prod_oid, img_prod_oid, lifespan=lifespan) dob_img.addOutput(dob_img_prod) dob_img_dict[kk] = (dob_img, dob_img_prod) for j in range(1, num_time_slice + 1): sli = "T%02d" % j for i in range(1, total_num_subband + 1): # for LBA, assume each subband should have two beams # one for calibration pipeline, the other for target / imaging pipeline stri = "%s_SB%02d" % (sli, i) time_slice_oid = "Socket_{0}".format(stri) dob_time_slice_sl = SocketListenerApp(time_slice_oid + "_SL", time_slice_oid + "_SL", lifespan=lifespan) dob_time_slice = InMemoryDROP(time_slice_oid, time_slice_oid, lifespan=lifespan) dob_time_slice_sl.addOutput(dob_time_slice) # all the same sub-bands (i) will be on the same node regardless of its time slice j dob_time_slice.location = "Node%03d.Groningen" % i roots.append(dob_time_slice_sl) oid_data_writer = "DataWriter_{0}".format(stri) dob_ingest = SleepAndCopyApp(oid_data_writer, oid_data_writer, lifespan=lifespan) dob_ingest.location = dob_time_slice.location dob_time_slice.addConsumer(dob_ingest) # assume a single measuremenset has multiple beams oid = "CalNTgt.MS_{0}".format(stri) dob_input = InMemoryDROP(oid, oid, lifespan=lifespan) dob_ingest.addOutput(dob_input) dob_input.location = dob_ingest.location #split by beam - calibrator beam and target beam appOid = "Split_{0}".format(stri) splitApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan) splitApp.location = dob_ingest.location dob_input.addConsumer(splitApp) oid = "Cal_BEAM_{0}".format(stri) dob_cal_beam = InMemoryDROP(oid, oid, lifespan=lifespan) dob_cal_beam.location = dob_ingest.location splitApp.addOutput(dob_cal_beam) oid = "Tgt_BEAM_{0}".format(stri) dob_tgt_beam = InMemoryDROP(oid, oid, lifespan=lifespan) dob_tgt_beam.location = dob_ingest.location splitApp.addOutput(dob_tgt_beam) # flag the cal beam oid = "FlgCal.MS_{0}".format(stri) appOid = "NDPPP_Cal" + stri dob_cal_flagged = InMemoryDROP(oid, oid, lifespan=lifespan) flagCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan) dob_cal_beam.addConsumer(flagCalApp) dob_source_db.addConsumer(flagCalApp) flagCalApp.addOutput(dob_cal_flagged) flagCalApp.location = dob_ingest.location dob_cal_flagged.location = dob_ingest.location #"C%03d.Groningen" % i # flag the target beam oid = "FlgTgt.MS_{0}".format(stri) appOid = "NDPPP_Tgt" + stri dob_tgt_flagged = InMemoryDROP(oid, oid, lifespan=lifespan) flagTgtApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan) dob_tgt_beam.addConsumer(flagTgtApp) dob_source_db.addConsumer(flagTgtApp) flagTgtApp.addOutput(dob_tgt_flagged) flagTgtApp.location = dob_ingest.location dob_tgt_flagged.location = dob_ingest.location # For calibration, each time slice is calibrated independently, and # the result is an output MS per subband # solve the gain oid = "Gain.TBL_{0}".format(stri) appOid = "BBS_GainCal_{0}".format(stri) dob_param_tbl = InMemoryDROP(oid, oid, lifespan=lifespan) gainCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan) dob_cal_flagged.addConsumer(gainCalApp) gainCalApp.addOutput(dob_param_tbl) gainCalApp.location = dob_ingest.location dob_param_tbl.location = dob_ingest.location # or apply the gain to the dataset oid = "CAL.MS_{0}".format(stri) appOid = "BBS_ApplyCal_{0}".format(stri) dob_calibrated = InMemoryDROP(oid, oid, lifespan=lifespan) applyCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan) dob_tgt_flagged.addConsumer(applyCalApp) dob_param_tbl.addConsumer(applyCalApp) applyCalApp.addOutput(dob_calibrated) applyCalApp.location = dob_ingest.location dob_calibrated.location = dob_ingest.location # extract local sky model (LSM) from GSM appOid = "Extract_LSM_{0}".format(stri) extractLSMApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan) extractLSMApp.location = dob_ingest.location dob_gsm.addConsumer(extractLSMApp) LSM_oid = "LSM_{0}".format(stri) dob_lsm = InMemoryDROP(LSM_oid, LSM_oid, lifespan=lifespan) dob_lsm.location = dob_ingest.location extractLSMApp.addOutput(dob_lsm) # convert LSM to source.db makesource_app_oid = "makesourcedb_{0}".format(stri) makesource_app = SleepAndCopyApp(makesource_app_oid, makesource_app_oid) makesource_app.location = dob_ingest.location dob_lsm.addConsumer(makesource_app) sourcedb_oid = "LSM_db_{0}".format(stri) dob_lsmdb = InMemoryDROP(sourcedb_oid, sourcedb_oid, lifespan=lifespan) dob_lsmdb.location = dob_ingest.location makesource_app.addOutput(dob_lsmdb) # add awimager img_k = (i - 1) / num_subb_per_image + 1 dob_img, dob_img_prod = dob_img_dict[img_k] dob_calibrated.addConsumer(dob_img) dob_lsmdb.addConsumer(dob_img) dob_img.location = dob_calibrated.location #overwrite to the last i dob_img_prod.location = dob_img.location return roots
def branch_failure(self, tooManyFailures): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ #create file data objects a1 = InMemoryDROP('oid:A1', 'uid:A1') a2 = InMemoryDROP('oid:A2', 'uid:A2') a3 = InMemoryDROP('oid:A3', 'uid:A3') # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum('oid:B1', 'uid:B1') b2 = SumupContainerChecksum('oid:B2', 'uid:B2') b3 = SumupContainerChecksum('oid:B3', 'uid:B3') c1 = InMemoryDROP('oid:C1', 'uid:C1') c2 = InMemoryDROP('oid:C2', 'uid:C2') c3 = InMemoryDROP('oid:C3', 'uid:C3') # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum('oid:D', 'uid:D', input_error_threshold = 33) e = InMemoryDROP('oid:E', 'uid:E') # Wire together dropAList = [a1,a2,a3] dropBList = [b1,b2,b3] dropCList = [c1,c2,c3] for dropA,dropB in zip(dropAList, dropBList): dropA.addConsumer(dropB) for dropB,dropC in zip(dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): #for dropA in dropAList: # this should be parallel for a1.write(' '); a1.setCompleted() if tooManyFailures: a2.setError() else: a2.write(' '); a2.setCompleted() a3.setError() if tooManyFailures: completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1] errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e] else: completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e] errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:] for drop in completedDrops: self.assertEqual(drop.status, DROPStates.COMPLETED) for drop in errorDrops: self.assertEqual(drop.status, DROPStates.ERROR) # The results we want to compare # (only in case that at least two branches executed) if not tooManyFailures: sum_crc = c1.checksum + c2.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEqual(sum_crc, 0) self.assertEqual(sum_crc, dropEData)
def pip_cont_img_pg(): """ PIP continuum imaging pipeline """ num_beam = 1 num_time = 2 num_freq = 2 num_facet = 2 num_grid = 4 stokes = ['I', 'Q', 'U', 'V'] dob_root_sl = SocketListenerApp("FullDataset_SL", "FullDataset_SL", lifespan=lifespan) dob_root = InMemoryDROP("FullDataset", "FullDataset", lifespan=lifespan) dob_root_sl.addOutput(dob_root) dob_root.location = "Buf01" adob_sp_beam = SleepAndCopyApp("SPLT_BEAM", "SPLT_BEAM", lifespan=lifespan) adob_sp_beam.location = "Buf01" dob_root.addConsumer(adob_sp_beam) for i in range(1, num_beam + 1): uid = i dob_beam = InMemoryDROP("BEAM_{0}".format(uid), "BEAM_{0}".format(uid), lifespan=lifespan) adob_sp_beam.addOutput(dob_beam) adob_sp_time = SleepAndCopyApp("SPLT_TIME_{0}".format(uid), "SPLT_TIME_{0}".format(uid), lifespan=lifespan) dob_beam.addConsumer(adob_sp_time) for j in range(1, num_time + 1): uid = "%d-%d" % (i, j) dob_time = InMemoryDROP("TIME_{0}".format(uid), "TIME_{0}".format(uid), lifespan=lifespan) adob_sp_time.addOutput(dob_time) adob_sp_freq = SleepAndCopyApp("SPLT_FREQ_{0}".format(uid), "SPLT_FREQ_{0}".format(uid), lifespan=lifespan) dob_time.addConsumer(adob_sp_freq) for k in range(1, num_freq + 1): uid = "%d-%d-%d" % (i, j, k) dob_freq = InMemoryDROP("FREQ_{0}".format(uid), "FREQ_{0}".format(uid), lifespan=lifespan) adob_sp_freq.addOutput(dob_freq) adob_sp_facet = SleepAndCopyApp("SPLT_FACET_{0}".format(uid), "SPLT_FACET_{0}".format(uid), lifespan=lifespan) dob_freq.addConsumer(adob_sp_facet) for l in range(1, num_facet + 1): uid = "%d-%d-%d-%d" % (i, j, k, l) dob_facet = InMemoryDROP("FACET_{0}".format(uid), "FACET_{0}".format(uid), lifespan=lifespan) adob_sp_facet.addOutput(dob_facet) adob_ph_rot = SleepAndCopyApp("PH_ROTATN_{0}".format(uid), "PH_ROTATN_{0}".format(uid), lifespan=lifespan) dob_facet.addConsumer(adob_ph_rot) dob_ph_rot = InMemoryDROP("PH_ROTD_{0}".format(uid), "PH_ROTD_{0}".format(uid), lifespan=lifespan) adob_ph_rot.addOutput(dob_ph_rot) adob_sp_stokes = SleepAndCopyApp("SPLT_STOKES_{0}".format(uid), "SPLT_STOKES_{0}".format(uid), lifespan=lifespan) dob_ph_rot.addConsumer(adob_sp_stokes) adob_w_kernel = SleepAndCopyApp("CACL_W_Knl_{0}".format(uid), "CACL_W_Knl_{0}".format(uid), lifespan=lifespan) dob_facet.addConsumer(adob_w_kernel) dob_w_knl = InMemoryDROP("W_Knl_{0}".format(uid), "W_Knl_{0}".format(uid), lifespan=lifespan) adob_w_kernel.addOutput(dob_w_knl) adob_a_kernel = SleepAndCopyApp("CACL_A_Knl_{0}".format(uid), "CACL_A_Knl_{0}".format(uid), lifespan=lifespan) dob_facet.addConsumer(adob_a_kernel) dob_a_knl = InMemoryDROP("A_Knl_{0}".format(uid), "A_Knl_{0}".format(uid), lifespan=lifespan) adob_a_kernel.addOutput(dob_a_knl) adob_create_kernel = SleepAndCopyApp("CREATE_Knl_{0}".format(uid), "CREATE_Knl_{0}".format(uid), lifespan=lifespan) dob_w_knl.addConsumer(adob_create_kernel) dob_a_knl.addConsumer(adob_create_kernel) for stoke in stokes: uid = "%s-%d-%d-%d-%d" % (stoke, i, j, k, l) #print "uid = {0}".format(uid) dob_stoke = InMemoryDROP("STOKE_{0}".format(uid), "STOKE_{0}".format(uid), lifespan=lifespan) adob_sp_stokes.addOutput(dob_stoke) dob_stoke.addConsumer(adob_create_kernel) dob_aw = InMemoryDROP("A_{0}".format(uid), "A_{0}".format(uid), lifespan=lifespan) adob_create_kernel.addOutput(dob_aw) # we do not do loop yet griders = [] for m in range(1, num_grid + 1): gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m) adob_gridding = SleepAndCopyApp("Gridding_{0}".format(gid), "Gridding_{0}".format(gid), lifespan=lifespan) dob_stoke.addConsumer(adob_gridding) dob_gridded_cell = InMemoryDROP("Grided_Cell_{0}".format(gid), "Grided_Cell_{0}".format(gid), lifespan=lifespan) adob_gridding.addOutput(dob_gridded_cell) griders.append(dob_gridded_cell) adob_gridded_bar = SleepAndCopyApp("GRIDDED_BARRIER_{0}".format(uid), "GRIDDED_BARRIER_{0}".format(uid), lifespan=lifespan) for grider in griders: grider.addConsumer(adob_gridded_bar) dob_gridded_stoke = InMemoryDROP("GRIDDED_STOKE_{0}".format(uid), "GRIDDED_STOKE_{0}".format(uid), lifespan=lifespan) adob_gridded_bar.addOutput(dob_gridded_stoke) FFTers = [] for m in range(1, num_grid + 1): gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m) adob_fft = SleepAndCopyApp("FFT_{0}".format(gid), "FFT_{0}".format(gid), lifespan=lifespan) dob_gridded_stoke.addConsumer(adob_fft) dob_ffted_cell = InMemoryDROP("FFTed_Cell_{0}".format(gid), "FFTed_Cell_{0}".format(gid), lifespan=lifespan) adob_fft.addOutput(dob_ffted_cell) FFTers.append(dob_ffted_cell) adob_ffted_bar = SleepAndCopyApp("FFTed_BARRIER_{0}".format(uid), "FFTed_BARRIER_{0}".format(uid), lifespan=lifespan) for ffter in FFTers: ffter.addConsumer(adob_ffted_bar) dob_ffted_stoke = InMemoryDROP("FFTed_STOKE_{0}".format(uid), "FFTed_STOKE_{0}".format(uid), lifespan=lifespan) adob_ffted_bar.addOutput(dob_ffted_stoke) cleaners = [] for m in range(1, num_grid + 1): gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m) adob_cleaner = SleepAndCopyApp("DECONV_{0}".format(gid), "DECONV_{0}".format(gid), lifespan=lifespan) dob_ffted_stoke.addConsumer(adob_cleaner) dob_cleaned_cell = InMemoryDROP("CLEANed_Cell_{0}".format(gid), "CLEANed_Cell_{0}".format(gid), lifespan=lifespan) adob_cleaner.addOutput(dob_cleaned_cell) cleaners.append(dob_cleaned_cell) adob_cleaned_bar = SleepAndCopyApp("CLEANed_BARRIER_{0}".format(uid), "CLEANed_BARRIER_{0}".format(uid), lifespan=lifespan) for cleaner in cleaners: cleaner.addConsumer(adob_cleaned_bar) dob_decon_stoke = InMemoryDROP("CLEANed_STOKE_{0}".format(uid), "CLEANed_STOKE_{0}".format(uid), lifespan=lifespan) adob_cleaned_bar.addOutput(dob_decon_stoke) adob_create_prod = SleepAndCopyApp("CRT-PROD_{0}".format(uid), "CRT-PROD_{0}".format(uid), lifespan=lifespan) dob_decon_stoke.addConsumer(adob_create_prod) dob_prod = InMemoryDROP("PRODUCT_{0}".format(uid), "PRODUCT_{0}".format(uid), lifespan=lifespan) adob_create_prod.addOutput(dob_prod) return dob_root_sl
def test_objectAsNormalAndStreamingInput(self): """ A test that checks that a DROP can act as normal and streaming input of different AppDROPs at the same time. We use the following graph: A --|--> B --> D |--> C --> E Here B uses A as a streaming input, while C uses it as a normal input """ class LastCharWriterApp(AppDROP): def initialize(self, **kwargs): super(LastCharWriterApp, self).initialize(**kwargs) self._lastChar = None def dataWritten(self, uid, data): self.execStatus = AppDROPStates.RUNNING outputDrop = self.outputs[0] self._lastChar = data[-1] outputDrop.write(self._lastChar) def dropCompleted(self, uid, status): self.execStatus = AppDROPStates.FINISHED self._notifyAppIsFinished() a = InMemoryDROP("a", "a") b = LastCharWriterApp("b", "b") c = SumupContainerChecksum("c", "c") d = InMemoryDROP("d", "d") e = InMemoryDROP("e", "e") a.addStreamingConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) # Consumer cannot be normal and streaming at the same time self.assertRaises(Exception, lambda: a.addConsumer(b)) self.assertRaises(Exception, lambda: a.addStreamingConsumer(c)) # Write a little, then check the consumers def checkDropStates(aStatus, dStatus, eStatus, lastChar): self.assertEquals(aStatus, a.status) self.assertEquals(dStatus, d.status) self.assertEquals(eStatus, e.status) self.assertEquals(lastChar, b._lastChar) checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None) a.write("abcde") checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "e") a.write("fghij") checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "j") a.write("k") with DROPWaiterCtx(self, [d, e]): a.setCompleted() checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, "k") self.assertEquals("ejk", droputils.allDropContents(d))
def test_stateMachine(self): """ A simple test to check that some transitions are invalid """ # Nice and easy drop = InMemoryDROP("a", "a") self.assertEquals(drop.status, DROPStates.INITIALIZED) drop.write("a") self.assertEquals(drop.status, DROPStates.WRITING) drop.setCompleted() self.assertEquals(drop.status, DROPStates.COMPLETED) # Try to overwrite the DROP's checksum and size self.assertRaises(Exception, lambda: setattr(drop, "checksum", 0)) self.assertRaises(Exception, lambda: setattr(drop, "size", 0)) # Try to write on a DROP that is already COMPLETED self.assertRaises(Exception, drop.write, "") # Invalid reading on a DROP that isn't COMPLETED yet drop = InMemoryDROP("a", "a") self.assertRaises(Exception, drop.open) self.assertRaises(Exception, drop.read, 1) self.assertRaises(Exception, drop.close, 1) # Invalid file descriptors used to read/close drop.setCompleted() fd = drop.open() otherFd = random.SystemRandom().randint(0, 1000) self.assertNotEquals(fd, otherFd) self.assertRaises(Exception, drop.read, otherFd) self.assertRaises(Exception, drop.close, otherFd) # but using the correct one should be OK drop.read(fd) self.assertTrue(drop.isBeingRead()) drop.close(fd) # Expire it, then try to set it as COMPLETED again drop.status = DROPStates.EXPIRED self.assertRaises(Exception, drop.setCompleted)
def chiles_pg(): total_bandwidth = 480 num_obs = 8 # the same as num of data island subband_width = 60 # MHz num_subb = total_bandwidth / subband_width subband_dict = collections.defaultdict(list) # for corner turning img_list = [] start_freq = 940 # this should be removed roots = [] for i in range(1, num_obs + 1): stri = "%02d" % i oid = "Obs_day_{0}".format(stri) appOid = "Receive_" + oid dob_obs_sl = SocketListenerApp(oid + "_SL", oid + "_SL", lifespan=lifespan, port=1110+i) dob_obs = InMemoryDROP(oid, oid, lifespan=lifespan) dob_obs_sl.addOutput(dob_obs) dob_obs.location = "{0}.aws-ec2.sydney".format(i) roots.append(dob_obs_sl) for j in range(1, num_subb + 1): app_oid = "mstransform_{0}_{1}".format(stri, "%02d" % j) adob_split = SleepAndCopyApp(app_oid, app_oid, lifespan=lifespan) adob_split.location = dob_obs.location dob_obs.addConsumer(adob_split) dob_sboid = "Split_{0}_{1}~{2}MHz".format(stri, start_freq + subband_width * j, start_freq + subband_width * (j + 1)) dob_sb = InMemoryDROP(dob_sboid, dob_sboid, lifespan=lifespan) dob_sb.location = dob_obs.location adob_split.addOutput(dob_sb) subband_dict[j].append(dob_sb) for j, v in subband_dict.items(): oid = "Subband_{0}~{1}MHz".format(start_freq + subband_width * j, start_freq + subband_width * (j + 1)) appOid = "Combine_" + oid dob = InMemoryDROP(oid, oid) dob.location = "{0}.aws-ec2.sydney".format(j % num_obs) subbandCombineAppDrop = SleepAndCopyApp(appOid, appOid, lifespan=lifespan) subbandCombineAppDrop.addOutput(dob) for dob_sb in v: subbandCombineAppDrop.addInput(dob_sb) app_oid = oid.replace("Subband_", "Clean_") adob_clean = SleepAndCopyApp(app_oid, app_oid, lifespan=lifespan) adob_clean.location = dob.location dob.addConsumer(adob_clean) img_oid = oid.replace("Subband_", "Image_") dob_img = InMemoryDROP(img_oid, img_oid, lifespan=lifespan) dob_img.location = dob.location adob_clean.addOutput(dob_img) img_list.append(dob_img) #container + app that produces it combineAppDrop = SleepAndCopyApp('Combine', 'Combine') dob_comb_img_oid = "Combined_image" dob_comb_img = InMemoryDROP(dob_comb_img_oid, dob_comb_img_oid, lifespan=lifespan) dob_comb_img.location = "10.1.1.100:7777" combineAppDrop.addOutput(dob_comb_img) for dob_img in img_list: combineAppDrop.addInput(dob_img) #concatenate all images adob_concat_oid = "Concat_image" adob_concat = SleepAndCopyApp(adob_concat_oid, adob_concat_oid, lifespan=lifespan) adob_concat.location = dob_comb_img.location dob_comb_img.addConsumer(adob_concat) # produce cube dob_cube_oid = "Cube_Day{0}~{1}_{2}~{3}MHz".format(1, num_obs, start_freq, start_freq + total_bandwidth) dob_cube = InMemoryDROP(dob_cube_oid, dob_cube_oid, lifespan=lifespan) dob_cube.location = dob_comb_img.location adob_concat.addOutput(dob_cube) return roots
def branch_failure(self, tooManyFailures): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ # create file data objects a1 = InMemoryDROP("oid:A1", "uid:A1") a2 = InMemoryDROP("oid:A2", "uid:A2") a3 = InMemoryDROP("oid:A3", "uid:A3") # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum("oid:B1", "uid:B1") b2 = SumupContainerChecksum("oid:B2", "uid:B2") b3 = SumupContainerChecksum("oid:B3", "uid:B3") c1 = InMemoryDROP("oid:C1", "uid:C1") c2 = InMemoryDROP("oid:C2", "uid:C2") c3 = InMemoryDROP("oid:C3", "uid:C3") # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum("oid:D", "uid:D", input_error_threshold=33) e = InMemoryDROP("oid:E", "uid:E") # Wire together dropAList = [a1, a2, a3] dropBList = [b1, b2, b3] dropCList = [c1, c2, c3] for dropA, dropB in map(lambda a, b: (a, b), dropAList, dropBList): dropA.addConsumer(dropB) for dropB, dropC in map(lambda b, c: (b, c), dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): # for dropA in dropAList: # this should be parallel for a1.write(" ") a1.setCompleted() if tooManyFailures: a2.setError() else: a2.write(" ") a2.setCompleted() a3.setError() if tooManyFailures: completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1] errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e] else: completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e] errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:] for drop in completedDrops: self.assertEquals(drop.status, DROPStates.COMPLETED) for drop in errorDrops: self.assertEquals(drop.status, DROPStates.ERROR) # The results we want to compare # (only in case that at least two branches executed) if not tooManyFailures: sum_crc = c1.checksum + c2.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEquals(sum_crc, 0) self.assertEquals(sum_crc, dropEData)
def test_join(self): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ filelen = self._test_drop_sz * ONE_MB #create file data objects a1 = FileDROP('oid:A1', 'uid:A1', expectedSize=filelen) a2 = FileDROP('oid:A2', 'uid:A2', expectedSize=filelen) a3 = FileDROP('oid:A3', 'uid:A3', expectedSize=filelen) # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum('oid:B1', 'uid:B1') b2 = SumupContainerChecksum('oid:B2', 'uid:B2') b3 = SumupContainerChecksum('oid:B3', 'uid:B3') c1 = InMemoryDROP('oid:C1', 'uid:C1') c2 = InMemoryDROP('oid:C2', 'uid:C2') c3 = InMemoryDROP('oid:C3', 'uid:C3') # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum('oid:D', 'uid:D') e = InMemoryDROP('oid:E', 'uid:E') # Wire together dropAList = [a1,a2,a3] dropBList = [b1,b2,b3] dropCList = [c1,c2,c3] for dropA,dropB in map(lambda a,b: (a,b), dropAList, dropBList): dropA.addConsumer(dropB) for dropB,dropC in map(lambda b,c: (b,c), dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): for dropA in dropAList: # this should be parallel for for _ in range(self._test_num_blocks): dropA.write(self._test_block) # All DROPs are completed now that the chain executed correctly for drop in dropAList + dropBList + dropCList: self.assertEqual(drop.status, DROPStates.COMPLETED) # The results we want to compare sum_crc = c1.checksum + c2.checksum + c3.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEqual(sum_crc, 0) self.assertEqual(sum_crc, dropEData)
def mwa_fornax_pg(): num_coarse_ch = 24 num_split = 3 # number of time splits per channel dob_root_sl = SocketListenerApp("MWA_LTA_SL", "MWA_LTA_SL", lifespan=lifespan) dob_root = InMemoryDROP("MWA_LTA", "MWA_LTA", lifespan=lifespan) dob_root.location = "Pawsey" dob_root_sl.addOutput(dob_root) #container dob_comb_img_oid = "Combined_image" combineImgAppOid = "Combine image" dob_comb_img = InMemoryDROP(dob_comb_img_oid, dob_comb_img_oid, lifespan=lifespan) combineImgApp = SleepAndCopyApp(combineImgAppOid, combineImgAppOid) combineImgApp.addOutput(dob_comb_img) dob_comb_img.location = "f032.fornax" for i in range(1, num_coarse_ch + 1): stri = "%02d" % i oid = "Subband_{0}".format(stri) appOid = "Combine_" + oid dob_obs = InMemoryDROP(oid, oid, lifespan=lifespan) combineSubbandApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan) combineSubbandApp.addOutput(dob_obs) dob_obs.location = "f%03d.fornax" % i oid_ingest = "NGAS_{0}".format(stri) dob_ingest = SleepAndCopyApp(oid_ingest, oid_ingest, lifespan=lifespan) dob_ingest.location = "f%03d.fornax:7777" % i dob_root.addConsumer(dob_ingest) for j in range(1, num_split + 1): strj = "%02d" % j split_oid = "Subband_{0}_Split_{1}".format(stri, strj) dob_split = InMemoryDROP(split_oid, split_oid, lifespan=lifespan) dob_split.location = dob_obs.location dob_ingest.addOutput(dob_split) combineSubbandApp.addInput(dob_split) oid_rts = "RTS_{0}".format(stri) dob_rts = SleepAndCopyApp(oid_rts, oid_rts, lifespan=lifespan) dob_rts.location = dob_obs.location dob_obs.addConsumer(dob_rts) oid_subimg = "Subcube_{0}".format(stri) dob_subimg = InMemoryDROP(oid_subimg, oid_subimg, lifespan=lifespan) dob_subimg.location = dob_obs.location dob_rts.addOutput(dob_subimg) combineImgApp.addInput(dob_subimg) #concatenate all images adob_concat_oid = "Concat_image" adob_concat = SleepAndCopyApp(adob_concat_oid, adob_concat_oid, lifespan=lifespan) adob_concat.location = dob_comb_img.location dob_comb_img.addConsumer(adob_concat) # produce cube dob_cube_oid = "Cube_30.72MHz" dob_cube = InMemoryDROP(dob_cube_oid, dob_cube_oid, lifespan=lifespan) dob_cube.location = dob_comb_img.location adob_concat.addOutput(dob_cube) return dob_root_sl
def createDrop(self, oid, uid, **kwargs): return InMemoryDROP(oid, uid, **kwargs)
self._evt = threading.Event() drop.addConsumer(self) def dropCompleted(self, drop, state): self._evt.set() def wait(self, timeout=None): return self._evt.wait(timeout) if __name__ == '__main__': try: vis_in_a = [] flux_out = InMemoryDROP(uuid.uuid1(), uuid.uuid1()) flux = SourceFlux(uuid.uuid1(), uuid.uuid1(), casapy_path=CASAPY) cl = Clean(uuid.uuid1(), uuid.uuid1(), field='deepfield', mode='frequency', restfreq='1420.405752MHz', nchan=-1, start='', width='', interpolation='nearest', gain=0.1, imsize=[256], cell=['1.0arcsec'],
def test_simple_chain(self): """ Simple test that creates a pipeline-like chain of commands. In this case we simulate a pipeline that does this, holding each intermediate result in memory: cat someFile | grep 'a' | sort | rev """ class GrepResult(BarrierAppDROP): def initialize(self, **kwargs): super(GrepResult, self).initialize(**kwargs) self._substring = kwargs["substring"] def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = StringIO(droputils.allDropContents(drop)).readlines() for line in allLines: if self._substring in line: output.write(line) class SortResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] sortedLines = StringIO(droputils.allDropContents(drop)).readlines() sortedLines.sort() for line in sortedLines: output.write(line) class RevResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = StringIO(droputils.allDropContents(drop)).readlines() for line in allLines: buf = "" for c in line: if c == " " or c == "\n": output.write(buf[::-1]) output.write(c) buf = "" else: buf += c a = InMemoryDROP("oid:A", "uid:A") b = GrepResult("oid:B", "uid:B", substring="a") c = InMemoryDROP("oid:C", "uid:C") d = SortResult("oid:D", "uid:D") e = InMemoryDROP("oid:E", "uid:E") f = RevResult("oid:F", "oid:F") g = InMemoryDROP("oid:G", "uid:G") a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) e.addConsumer(f) f.addOutput(g) # Initial write contents = "first line\nwe have an a here\nand another one\nnoone knows me" cResExpected = "we have an a here\nand another one\n" eResExpected = "and another one\nwe have an a here\n" gResExpected = "dna rehtona eno\new evah na a ereh\n" with DROPWaiterCtx(self, g): a.write(contents) a.setCompleted() # Get intermediate and final results and compare actualRes = [] for i in [c, e, g]: actualRes.append(droputils.allDropContents(i)) map(lambda x, y: self.assertEquals(x, y), [cResExpected, eResExpected, gResExpected], actualRes)