def test_app_multiple_outputs(self): """ A small method that tests that the AppDROPs writing to two different DROPs outputs works The graph constructed by this example looks as follow: |--> E A --> B --> C --> D --| |--> F Here B and D are an AppDROPs, with D writing to two DROPs outputs (E and F) and reading from C. C, in turn, is written by B, which in turns reads the data from A """ # This is used as "B" class NumberWriterApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] output = self.outputs[0] howMany = int(droputils.allDropContents(inputDrop)) for i in range(howMany): output.write(str(i) + " ") # This is used as "D" class OddAndEvenContainerApp(BarrierAppDROP): def run(self): inputDrop = self.inputs[0] outputs = self.outputs numbers = droputils.allDropContents(inputDrop).strip().split() for n in numbers: outputs[int(n) % 2].write(n + six.b(" ")) # Create DROPs a = InMemoryDROP('oid:A', 'uid:A') b = NumberWriterApp('oid:B', 'uid:B') c = InMemoryDROP('oid:A', 'uid:A') d = OddAndEvenContainerApp('oid:D', 'uid:D') e = InMemoryDROP('oid:E', 'uid:E') f = InMemoryDROP('oid:F', 'uid:F') # Wire them together a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) d.addOutput(f) # Start the execution with DROPWaiterCtx(self, [e,f]): a.write('20') a.setCompleted() # Check the final results are correct for drop in [a,b,c,d,e]: self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop)) self.assertEqual(six.b("0 2 4 6 8 10 12 14 16 18"), droputils.allDropContents(e).strip()) self.assertEqual(six.b("1 3 5 7 9 11 13 15 17 19"), droputils.allDropContents(f).strip())
def test_addDropInstances(self): a1 = InMemoryDROP('a', 'a1') a2 = InMemoryDROP('a', 'a2') registry = RDBMSRegistry('sqlite3', DBFILE) registry.addDrop(a1) uids = registry.getDropUids(a1) self.assertEqual(1, len(uids)) self.assertEqual('a1', uids[0]) registry.addDropInstance(a2) uids = registry.getDropUids(a1) uids.sort() self.assertEqual(2, len(uids)) self.assertEqual('a1', uids[0]) self.assertEqual('a2', uids[1]) # Check accessing the database separately conn = sqlite3.connect(DBFILE) # @UndefinedVariable cur = conn.cursor() cur.execute("SELECT uid FROM dfms_dropinstance WHERE oid = 'a'") rows = cur.fetchall() self.assertEqual(2, len(rows)) uids = [r[0] for r in rows] uids.sort() self.assertEqual(2, len(uids)) self.assertEqual('a1', uids[0]) self.assertEqual('a2', uids[1]) cur.close() conn.close()
def _test_graphExecutionDriver(self, mode): """ A small test to check that DROPs executions can be driven externally if required, and not always internally by themselves """ a = InMemoryDROP('a', 'a', executionMode=mode, expectedSize=1) b = SumupContainerChecksum('b', 'b') c = InMemoryDROP('c', 'c') a.addConsumer(b) c.addProducer(b) # Write and check dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c] with DROPWaiterCtx(self, dropsToWaitFor): a.write('1') if mode == ExecutionMode.EXTERNAL: # b hasn't been triggered self.assertEqual(c.status, DROPStates.INITIALIZED) self.assertEqual(b.status, DROPStates.INITIALIZED) self.assertEqual(b.execStatus, AppDROPStates.NOT_RUN) # Now let b consume a with DROPWaiterCtx(self, [c]): b.dropCompleted('a', DROPStates.COMPLETED) self.assertEqual(c.status, DROPStates.COMPLETED) elif mode == ExecutionMode.DROP: # b is already done self.assertEqual(c.status, DROPStates.COMPLETED)
def test_eager_inputFired_app(self): """ Tests that InputFiredApps works as expected """ # No n_effective_inputs given self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a') # Invalid values self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=-2) self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=0) # More effective inputs than inputs a = InMemoryDROP('b', 'b') b = InputFiredAppDROP('a', 'a', n_effective_inputs=2) b.addInput(a) self.assertRaises(Exception, a.setCompleted) # 2 effective inputs, 4 outputs. Trigger 2 inputs and make sure the # app has run a,b,c,d = [InMemoryDROP(str(i), str(i)) for i in range(4)] e = InputFiredAppDROP('e', 'e', n_effective_inputs=2) for x in a,b,c,d: e.addInput(x) with DROPWaiterCtx(self, e, 5): a.setCompleted() b.setCompleted() self.assertEqual(AppDROPStates.FINISHED, e.execStatus) self.assertEqual(DROPStates.COMPLETED, a.status) self.assertEqual(DROPStates.COMPLETED, b.status) self.assertEqual(DROPStates.INITIALIZED, c.status) self.assertEqual(DROPStates.INITIALIZED, d.status)
def test_objectAsNormalAndStreamingInput(self): """ A test that checks that a DROP can act as normal and streaming input of different AppDROPs at the same time. We use the following graph: A --|--> B --> D |--> C --> E Here B uses A as a streaming input, while C uses it as a normal input """ class LastCharWriterApp(AppDROP): def initialize(self, **kwargs): super(LastCharWriterApp, self).initialize(**kwargs) self._lastByte = None def dataWritten(self, uid, data): self.execStatus = AppDROPStates.RUNNING outputDrop = self.outputs[0] self._lastByte = six.indexbytes(data, -1) outputDrop.write(self._lastByte) def dropCompleted(self, uid, status): self.execStatus = AppDROPStates.FINISHED self._notifyAppIsFinished() a = InMemoryDROP('a', 'a') b = LastCharWriterApp('b', 'b') c = SumupContainerChecksum('c', 'c') d = InMemoryDROP('d', 'd') e = InMemoryDROP('e', 'e') a.addStreamingConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) # Consumer cannot be normal and streaming at the same time self.assertRaises(Exception, a.addConsumer, b) self.assertRaises(Exception, a.addStreamingConsumer, c) # Write a little, then check the consumers def checkDropStates(aStatus, dStatus, eStatus, lastByte): self.assertEqual(aStatus, a.status) self.assertEqual(dStatus, d.status) self.assertEqual(eStatus, e.status) if lastByte is not None: self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte)) checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None) a.write('abcde') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e') a.write('fghij') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j') a.write('k') with DROPWaiterCtx(self, [d,e]): a.setCompleted() checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k') self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
def test_errorState(self): a = InMemoryDROP('a', 'a') b = SumupContainerChecksum('b', 'b') c = InMemoryDROP('c', 'c') c.addProducer(b) b.addInput(a) a.setError() self.assertEqual(DROPStates.ERROR, a.status) self.assertEqual(DROPStates.ERROR, b.status) self.assertEqual(DROPStates.ERROR, c.status)
def test_invalid(self): # Shouldn't allow inputs a = SocketListenerApp('a', 'a', port=1) a.addOutput(InMemoryDROP('c', 'c')) self.assertRaises(Exception, a.addInput, InMemoryDROP('b', 'b')) self.assertRaises(Exception, a.addStreamingInput, InMemoryDROP('b', 'b')) # Shouldn't be able to open ports <= 1024 a.execute() self.assertEqual(a.status, DROPStates.ERROR)
def test_two_simultaneous_pipes(self): """ A more complicated test where three bash applications run at the same time. The first streams its output to the second one, while the second one streams *its* output to the third one. ------------- -------------- ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F | | echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | | -----*------- -------------- ----*--*----- -------------- -----*------- ---------- | | | | \-------------|named-pipe|----------\ \-----------|named-pipe|-----------/ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by "cat" (BashApp C). The printed results (a copy of the original) are streamed through D and read by "sort" (BashApp E), which writes the output to F. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp('a', 'a', command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP('b', 'b') c = StreamingInputOutputBashApp('c', 'c', command="cat") d = InMemoryDROP('d', 'd') e = StreamingInputBashApp('e', 'e', command="sort -n > %o0") f = FileDROP('f', 'f', filepath=output_fname) a.addOutput(b) b.addStreamingConsumer(c) c.addOutput(d) d.addStreamingConsumer(e) e.addOutput(f) # Let's fire the app with DROPWaiterCtx(self, f, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d, e, f): self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual([1, 2, 3, 4, 5], [ int(x) for x in droputils.allDropContents(f).strip().split(six.b('\n')) ]) # Clean up and go os.remove(output_fname)
def test_speadApp(self): port = 1111 itemId = 0x2000 thread_pool = spead2.ThreadPool() self._stream = spead2.send.UdpStream( thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7)) a = SpeadReceiverApp('a', 'a', port=port, itemId=itemId) b = InMemoryDROP('b', 'b') a.addOutput(b) size = 1024 threading.Thread(target=lambda: a.execute()).start() time.sleep(1) msg = os.urandom(size) with DROPWaiterCtx(self, b, timeout=1): ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48)) item = ig.add_item(itemId, 'main_data', 'a char array', shape=(size, ), format=[('c', 8)]) item.value = msg self._stream.send_heap(ig.get_heap()) self._stream.send_heap(ig.get_end()) for drop in a, b: self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual(size, b.size) self.assertEqual(msg, droputils.allDropContents(b))
def test_multipleProducers(self): """ A test that checks that multiple-producers correctly drive the state of their shared output """ class App(BarrierAppDROP): pass a,b,c,d,e = [App(chr(ord('A') + i), chr(ord('A') + i)) for i in range(5)] f = InMemoryDROP('F', 'F') for drop in a,b,c,d,e: drop.addOutput(f) self.assertEqual(DROPStates.INITIALIZED, f.status) for drop in a,b,c,d,e: self.assertEqual(AppDROPStates.NOT_RUN, drop.execStatus) # Run the first 4 ones, F should still be in INITIALIZED for drop in a,b,c,d: drop.execute() self.assertEqual(DROPStates.INITIALIZED, f.status) self.assertEqual(AppDROPStates.NOT_RUN, e.execStatus) for drop in a,b,c,d: self.assertEqual(AppDROPStates.FINISHED, drop.execStatus) # Run the final one, now F should be COMPLETED e.execute() self.assertEqual(DROPStates.COMPLETED, f.status) for drop in a,b,c,d,e: self.assertEqual(AppDROPStates.FINISHED, drop.execStatus)
def test_dropAccess(self): a1 = InMemoryDROP('a', 'a1') registry = RDBMSRegistry('sqlite3', DBFILE) registry.addDrop(a1) self.assertEqual(-1, registry.getLastAccess('a')) registry.recordNewAccess('a') self.assertNotEqual(-1, registry.getLastAccess('a'))
def test_stateMachine(self): """ A simple test to check that some transitions are invalid """ # Nice and easy drop = InMemoryDROP('a', 'a') self.assertEqual(drop.status, DROPStates.INITIALIZED) drop.write('a') self.assertEqual(drop.status, DROPStates.WRITING) drop.setCompleted() self.assertEqual(drop.status, DROPStates.COMPLETED) # Try to overwrite the DROP's checksum and size self.assertRaises(Exception, lambda: setattr(drop, 'checksum', 0)) self.assertRaises(Exception, lambda: setattr(drop, 'size', 0)) # Try to write on a DROP that is already COMPLETED self.assertRaises(Exception, drop.write, '') # Invalid reading on a DROP that isn't COMPLETED yet drop = InMemoryDROP('a', 'a') self.assertRaises(Exception, drop.open) self.assertRaises(Exception, drop.read, 1) self.assertRaises(Exception, drop.close, 1) # Invalid file descriptors used to read/close drop.setCompleted() fd = drop.open() otherFd = random.SystemRandom().randint(0, 1000) self.assertNotEqual(fd, otherFd) self.assertRaises(Exception, drop.read, otherFd) self.assertRaises(Exception, drop.close, otherFd) # but using the correct one should be OK drop.read(fd) self.assertTrue(drop.isBeingRead()) drop.close(fd) # Expire it, then try to set it as COMPLETED again drop.status = DROPStates.EXPIRED self.assertRaises(Exception, drop.setCompleted)
def _createGraph(self): """ Creates the following graph of DROPs: A |--> B ----> D --> G --> I --| |--> C -|--> E --------------|-> H --> J |--> F B, C, G and H are AppDOs. The names have been given in breadth-first order (although H has a dependency on I) """ a = InMemoryDROP('a', 'a') b = BarrierAppDROP('b', 'b') c = BarrierAppDROP('c', 'c') d = InMemoryDROP('d', 'd') e = InMemoryDROP('e', 'e') f = InMemoryDROP('f', 'f') g = BarrierAppDROP('g', 'g') h = BarrierAppDROP('h', 'h') i = InMemoryDROP('i', 'i') j = InMemoryDROP('j', 'j') a.addConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) c.addOutput(f) d.addConsumer(g) e.addConsumer(h) g.addOutput(i) i.addConsumer(h) h.addOutput(j) return a, b, c, d, e, f, g, h, i, j
def test_addDrop(self): a = InMemoryDROP('a', 'a') registry = RDBMSRegistry('sqlite3', DBFILE) registry.addDrop(a) conn = sqlite3.connect(DBFILE) # @UndefinedVariable cur = conn.cursor() cur.execute('SELECT oid FROM dfms_drop') r = cur.fetchone() self.assertEqual(1, len(r)) self.assertEqual('a', r[0]) cur.close() conn.close()
def test_socket_listener(self): ''' A simple test to check that SocketListenerApps are indeed working as expected; that is, they write the data they receive into their output, and finish when the connection is closed from the client side The data flow diagram looks like this: A --> B --> C --> D ''' host = 'localhost' port = 9933 data = os.urandom(1025) a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port) b = InMemoryDROP('oid:B', 'uid:B') c = SumupContainerChecksum('oid:C', 'uid:C') d = InMemoryDROP('oid:D', 'uid:D') a.addOutput(b) b.addConsumer(c) c.addOutput(d) # Create the socket, write, and close the connection, allowing # A to move to COMPLETED with DROPWaiterCtx(self, d, 3): # That's plenty of time a.async_execute() utils.write_to(host, port, data, 1) for drop in [a,b,c,d]: self.assertEqual(DROPStates.COMPLETED, drop.status) # Our expectations are fulfilled! bContents = droputils.allDropContents(b) dContents = int(droputils.allDropContents(d)) self.assertEqual(data, bContents) self.assertEqual(crc32(data, 0), dContents)
def test_single_pipe(self): """ A simple test where two bash apps are connected to each other in a streaming fashion. The data flows through a pipe which is created by the framework. The data drop in between acts only as a intermediator to establish the underlying communication channel. ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | File D | | echo | | "/a/pipe" | | dc | | | -----*------- -------------- ------*------ ---------- | | \-------------|named-pipe|------------/ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by cat and redirected to D. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp('a', 'a', command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP('b', 'b') c = StreamingInputBashApp('c', 'c', command="cat > %o0") d = FileDROP('d', 'd', filepath=output_fname) a.addOutput(b) c.addStreamingInput(b) c.addOutput(d) # Let's fire the app with DROPWaiterCtx(self, d, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d): self.assertEqual(DROPStates.COMPLETED, drop.status, "Drop %r not COMPLETED: %d" % (drop, drop.status)) self.assertEqual( [5, 4, 3, 2, 1], [int(x) for x in droputils.allDropContents(d).split(six.b('\n'))]) # Clean up and go os.remove(output_fname)
def test_directoryContainer(self): """ A small, simple test for the DirectoryContainer DROP that checks it allows only valid children to be added """ # Prepare our playground cwd = os.getcwd() os.chdir('/tmp') dirname = ".hidden" dirname2 = ".hidden/inside" if not os.path.exists(dirname2): os.makedirs(dirname2) # DROPs involved a = FileDROP('a', 'a', dirname=dirname) b = FileDROP('b', 'b', dirname=dirname) c = FileDROP('c', 'c', dirname=dirname2) d = FileDROP('d', 'd', dirname=dirname2) cont1 = DirectoryContainer('e', 'e', dirname=dirname) cont2 = DirectoryContainer('f', 'f', dirname=dirname2) # Paths are absolutely reported self.assertEqual(os.path.realpath('/tmp/.hidden'), os.path.realpath(cont1.path)) self.assertEqual(os.path.realpath('/tmp/.hidden/inside'), os.path.realpath(cont2.path)) # Certain children-to-be are rejected self.assertRaises(TypeError, cont1.addChild, NullDROP('g', 'g')) self.assertRaises(TypeError, cont1.addChild, InMemoryDROP('h', 'h')) self.assertRaises(TypeError, cont1.addChild, ContainerDROP('i', 'i')) self.assertRaises(Exception, cont1.addChild, c) self.assertRaises(Exception, cont1.addChild, d) self.assertRaises(Exception, cont2.addChild, a) self.assertRaises(Exception, cont2.addChild, b) # These children are correct cont1.addChild(a) cont1.addChild(b) cont2.addChild(c) cont2.addChild(d) # Revert to previous state shutil.rmtree(dirname, True) os.chdir(cwd)
def _test_write_withDropType(self, dropType): """ Test an AbstractDROP and a simple AppDROP (for checksum calculation) """ a = dropType('oid:A', 'uid:A', expectedSize = self._test_drop_sz * ONE_MB) b = SumupContainerChecksum('oid:B', 'uid:B') c = InMemoryDROP('oid:C', 'uid:C') b.addInput(a) b.addOutput(c) test_crc = 0 with DROPWaiterCtx(self, c): for _ in range(self._test_num_blocks): a.write(self._test_block) test_crc = crc32(self._test_block, test_crc) # Read the checksum from c cChecksum = int(droputils.allDropContents(c)) self.assertNotEqual(a.checksum, 0) self.assertEqual(a.checksum, test_crc) self.assertEqual(cChecksum, test_crc)
def test_sql_create(self): sqlite01 = get_oid('sqlite') sqlite_drop = FileDROP( sqlite01, get_uuid(), precious=False, dirname=os.path.join(TestSqlite._temp, sqlite01), check_exists=False, ) initialize_sqlite = InitializeSqliteApp( get_oid('app'), get_uuid(), user='******', ) sqlite_in_memory = InMemoryDROP( get_oid('memory'), get_uuid(), precious=False, ) initialize_sqlite.addInput(sqlite_drop) initialize_sqlite.addOutput(sqlite_in_memory) with droputils.DROPWaiterCtx(self, sqlite_in_memory, 50000): sqlite_drop.setCompleted()
def test_join(self): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ filelen = self._test_drop_sz * ONE_MB #create file data objects a1 = FileDROP('oid:A1', 'uid:A1', expectedSize=filelen) a2 = FileDROP('oid:A2', 'uid:A2', expectedSize=filelen) a3 = FileDROP('oid:A3', 'uid:A3', expectedSize=filelen) # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum('oid:B1', 'uid:B1') b2 = SumupContainerChecksum('oid:B2', 'uid:B2') b3 = SumupContainerChecksum('oid:B3', 'uid:B3') c1 = InMemoryDROP('oid:C1', 'uid:C1') c2 = InMemoryDROP('oid:C2', 'uid:C2') c3 = InMemoryDROP('oid:C3', 'uid:C3') # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum('oid:D', 'uid:D') e = InMemoryDROP('oid:E', 'uid:E') # Wire together dropAList = [a1,a2,a3] dropBList = [b1,b2,b3] dropCList = [c1,c2,c3] for dropA,dropB in map(lambda a,b: (a,b), dropAList, dropBList): dropA.addConsumer(dropB) for dropB,dropC in map(lambda b,c: (b,c), dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): for dropA in dropAList: # this should be parallel for for _ in range(self._test_num_blocks): dropA.write(self._test_block) # All DROPs are completed now that the chain executed correctly for drop in dropAList + dropBList + dropCList: self.assertEqual(drop.status, DROPStates.COMPLETED) # The results we want to compare sum_crc = c1.checksum + c2.checksum + c3.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEqual(sum_crc, 0) self.assertEqual(sum_crc, dropEData)
def branch_failure(self, tooManyFailures): """ Using the container data object to implement a join/barrier dataflow. A1, A2 and A3 are FileDROPs B1, B2 and B3 are SumupContainerChecksum C1, C2 and C3 are InMemoryDROPs D is a SumupContainerChecksum E is a InMemoryDROP --> A1 --> B1 --> C1 --| --> A2 --> B2 --> C2 --|--> D --> E --> A3 --> B3 --> C3 --| Upon writing all A* DROPs, the execution of B* DROPs should be triggered, after which "C" will transition to COMPLETE. Once all "C"s have moved to COMPLETED "D"'s execution will also be triggered, and finally E will hold the sum of B1, B2 and B3's checksums """ #create file data objects a1 = InMemoryDROP('oid:A1', 'uid:A1') a2 = InMemoryDROP('oid:A2', 'uid:A2') a3 = InMemoryDROP('oid:A3', 'uid:A3') # CRC Result DROPs, storing the result in memory b1 = SumupContainerChecksum('oid:B1', 'uid:B1') b2 = SumupContainerChecksum('oid:B2', 'uid:B2') b3 = SumupContainerChecksum('oid:B3', 'uid:B3') c1 = InMemoryDROP('oid:C1', 'uid:C1') c2 = InMemoryDROP('oid:C2', 'uid:C2') c3 = InMemoryDROP('oid:C3', 'uid:C3') # The final DROP that sums up the CRCs from the container DROP d = SumupContainerChecksum('oid:D', 'uid:D', input_error_threshold = 33) e = InMemoryDROP('oid:E', 'uid:E') # Wire together dropAList = [a1,a2,a3] dropBList = [b1,b2,b3] dropCList = [c1,c2,c3] for dropA,dropB in zip(dropAList, dropBList): dropA.addConsumer(dropB) for dropB,dropC in zip(dropBList, dropCList): dropB.addOutput(dropC) for dropC in dropCList: dropC.addConsumer(d) d.addOutput(e) # Write data into the initial "A" DROPs, which should trigger # the whole chain explained above with DROPWaiterCtx(self, e): #for dropA in dropAList: # this should be parallel for a1.write(' '); a1.setCompleted() if tooManyFailures: a2.setError() else: a2.write(' '); a2.setCompleted() a3.setError() if tooManyFailures: completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1] errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e] else: completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e] errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:] for drop in completedDrops: self.assertEqual(drop.status, DROPStates.COMPLETED) for drop in errorDrops: self.assertEqual(drop.status, DROPStates.ERROR) # The results we want to compare # (only in case that at least two branches executed) if not tooManyFailures: sum_crc = c1.checksum + c2.checksum dropEData = int(droputils.allDropContents(e)) self.assertNotEqual(sum_crc, 0) self.assertEqual(sum_crc, dropEData)
def test_simple_chain(self): ''' Simple test that creates a pipeline-like chain of commands. In this case we simulate a pipeline that does this, holding each intermediate result in memory: cat someFile | grep 'a' | sort | rev ''' class GrepResult(BarrierAppDROP): def initialize(self, **kwargs): super(GrepResult, self).initialize(**kwargs) self._substring = six.b(kwargs['substring']) def run(self): drop = self.inputs[0] output = self.outputs[0] allLines = BytesIO(droputils.allDropContents(drop)).readlines() for line in allLines: if self._substring in line: output.write(line) class SortResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] sortedLines = BytesIO(droputils.allDropContents(drop)).readlines() sortedLines.sort() for line in sortedLines: output.write(line) class RevResult(BarrierAppDROP): def run(self): drop = self.inputs[0] output = self.outputs[0] allbytes = droputils.allDropContents(drop) buf = bytearray() for c in allbytes: if c == six.b(' ') or c == six.b('\n'): output.write(buf[::-1]) output.write(c) buf = bytearray() else: buf.append(c) a = InMemoryDROP('oid:A', 'uid:A') b = GrepResult('oid:B', 'uid:B', substring="a") c = InMemoryDROP('oid:C', 'uid:C') d = SortResult('oid:D', 'uid:D') e = InMemoryDROP('oid:E', 'uid:E') f = RevResult('oid:F', 'oid:F') g = InMemoryDROP('oid:G', 'uid:G') a.addConsumer(b) b.addOutput(c) c.addConsumer(d) d.addOutput(e) e.addConsumer(f) f.addOutput(g) # Initial write contents = "first line\nwe have an a here\nand another one\nnoone knows me" cResExpected = "we have an a here\nand another one\n" eResExpected = "and another one\nwe have an a here\n" gResExpected = "dna rehtona eno\new evah na a ereh\n" with DROPWaiterCtx(self, g): a.write(contents) a.setCompleted() # Get intermediate and final results and compare actualRes = [] for i in [c, e, g]: actualRes.append(droputils.allDropContents(i)) map(lambda x, y: self.assertEqual(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
self._evt = threading.Event() drop.addConsumer(self) def dropCompleted(self, drop, state): self._evt.set() def wait(self, timeout=None): return self._evt.wait(timeout) if __name__ == '__main__': try: vis_in_a = [] flux_out = InMemoryDROP(uuid.uuid1(), uuid.uuid1()) flux = SourceFlux(uuid.uuid1(), uuid.uuid1(), casapy_path=CASAPY) cl = Clean(uuid.uuid1(), uuid.uuid1(), field='deepfield', mode='frequency', restfreq='1420.405752MHz', nchan=-1, start='', width='', interpolation='nearest', gain=0.1, imsize=[256], cell=['1.0arcsec'],
def createDrop(self, oid, uid, **kwargs): return InMemoryDROP(oid, uid, **kwargs)