def test_basic_run(self): signal = SignalGenerateAndAverageDrop('1', '1', internal_port=internal_port, stream_port=stream_port, start_freq=int(os.environ.get('START_FREQ', 45991200)), freq_step=int(os.environ.get('FREQ_STEP', 6400)), use_gpus=int(os.environ.get('USE_GPUS', 0)), num_freq_steps=int(os.environ.get('NUM_CHANNELS', 1)), telescope_model_path='./conf/%s.tm' % tm, sky_model_file_path="./conf/eor_model_list.csv", num_time_steps=int(os.environ.get('NUM_TIME_STEPS', 1))) sink = AveragerSinkDrop('2', '2', stream_listen_port_start=stream_port, use_adios2=int(os.environ.get('USE_ADIOS2', 0)), baseline_exclusion_map_path='./conf/%s_baselines.csv' % tm, node='127.0.0.1') drop = InMemoryDROP('3', '3') drop.addStreamingConsumer(sink) signal.addOutput(drop) ms = FileDROP('4', '4', filepath=output) sink.addOutput(ms) with droputils.DROPWaiterCtx(self, ms, 1000): signal.async_execute()
def test_eager_inputFired_app(self): """ Tests that InputFiredApps works as expected """ # No n_effective_inputs given self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a') # Invalid values self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=-2) self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=0) # More effective inputs than inputs a = InMemoryDROP('b', 'b') b = InputFiredAppDROP('a', 'a', n_effective_inputs=2) b.addInput(a) self.assertRaises(Exception, a.setCompleted) # 2 effective inputs, 4 outputs. Trigger 2 inputs and make sure the # app has run a, b, c, d = [InMemoryDROP(str(i), str(i)) for i in range(4)] e = InputFiredAppDROP('e', 'e', n_effective_inputs=2) for x in a, b, c, d: e.addInput(x) with DROPWaiterCtx(self, e, 5): a.setCompleted() b.setCompleted() self.assertEqual(AppDROPStates.FINISHED, e.execStatus) self.assertEqual(DROPStates.COMPLETED, a.status) self.assertEqual(DROPStates.COMPLETED, b.status) self.assertEqual(DROPStates.INITIALIZED, c.status) self.assertEqual(DROPStates.INITIALIZED, d.status)
def _test_graphExecutionDriver(self, mode): """ A small test to check that DROPs executions can be driven externally if required, and not always internally by themselves """ a = InMemoryDROP("a", "a", executionMode=mode, expectedSize=1) b = SumupContainerChecksum("b", "b") c = InMemoryDROP("c", "c") a.addConsumer(b) c.addProducer(b) # Write and check dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c] with DROPWaiterCtx(self, dropsToWaitFor): a.write(b"1") if mode == ExecutionMode.EXTERNAL: # b hasn't been triggered self.assertEqual(c.status, DROPStates.INITIALIZED) self.assertEqual(b.status, DROPStates.INITIALIZED) self.assertEqual(b.execStatus, AppDROPStates.NOT_RUN) # Now let b consume a with DROPWaiterCtx(self, [c]): b.dropCompleted("a", DROPStates.COMPLETED) self.assertEqual(c.status, DROPStates.COMPLETED) elif mode == ExecutionMode.DROP: # b is already done self.assertEqual(c.status, DROPStates.COMPLETED)
def test_namedPorts(self): """ Use a graph with named ports and check whether it is runnning """ init_oid = "2022-03-20T04:33:27_-2_0" # first drop in graph sessionId = "lalo" with pkg_resources.resource_stream( "test", "graphs/funcTestPG_namedPorts.graph" ) as f: # @UndefinedVariable graphSpec = json.load(f) # dropSpecs = graph_loader.loadDropSpecs(graphSpec) self.createSessionAndAddGraph(sessionId, graphSpec=graphSpec) # Deploy now and get OIDs self.dim.deploySession(sessionId) fd = self.dm._sessions[sessionId].drops["2022-03-20T04:33:27_-1_0"] init_drop = self.dm._sessions[sessionId].drops[init_oid] a = InMemoryDROP("a", "a") init_drop.addInput(a) logger.debug(f"PyfuncAPPDrop: {dir(fd)}") for i in fd.parameters["inputs"]: logger.debug(f"PyfuncAPPDrop input names:{i}") with droputils.DROPWaiterCtx(self, init_drop, 3): a.setCompleted()
def test_addDropInstances(self): a1 = InMemoryDROP('a', 'a1') a2 = InMemoryDROP('a', 'a2') registry = RDBMSRegistry('sqlite3', DBFILE) registry.addDrop(a1) uids = registry.getDropUids(a1) self.assertEqual(1, len(uids)) self.assertEqual('a1', uids[0]) registry.addDropInstance(a2) uids = registry.getDropUids(a1) uids.sort() self.assertEqual(2, len(uids)) self.assertEqual('a1', uids[0]) self.assertEqual('a2', uids[1]) # Check accessing the database separately conn = sqlite3.connect(DBFILE) # @UndefinedVariable cur = conn.cursor() cur.execute("SELECT uid FROM dlg_dropinstance WHERE oid = 'a'"); rows = cur.fetchall() self.assertEqual(2, len(rows)) uids = [r[0] for r in rows] uids.sort() self.assertEqual(2, len(uids)) self.assertEqual('a1', uids[0]) self.assertEqual('a2', uids[1]) cur.close() conn.close()
def test_listappendthrashing(self, size=1000): a = InMemoryDROP("a", "a") b = ListAppendThrashingApp("b", "b", size=size) self.assertEqual(b.size, size) c = InMemoryDROP("c", "c") b.addInput(a) b.addOutput(c) self._test_graph_runs((a, b, c), a, c, timeout=4) data_out = pickle.loads(droputils.allDropContents(c)) self.assertEqual(b.marray, data_out)
def test_invalid(self): # Shouldn't allow inputs a = SocketListenerApp("a", "a", port=64 * 1024) a.addOutput(InMemoryDROP("c", "c")) self.assertRaises(Exception, a.addInput, InMemoryDROP("b", "b")) self.assertRaises(Exception, a.addStreamingInput, InMemoryDROP("b", "b")) # Shouldn't be able to open ports > 64k - 1 a.execute() self.assertEqual(a.status, DROPStates.ERROR)
def test_invalid(self): # Shouldn't allow inputs a = SocketListenerApp('a', 'a', port=1) a.addOutput(InMemoryDROP('c', 'c')) self.assertRaises(Exception, a.addInput, InMemoryDROP('b', 'b')) self.assertRaises(Exception, a.addStreamingInput, InMemoryDROP('b', 'b')) # Shouldn't be able to open ports <= 1024 a.execute() self.assertEqual(a.status, DROPStates.ERROR)
def test_speadApp(self): port = 1111 itemId = 0x2000 thread_pool = spead2.ThreadPool() self._stream = spead2.send.UdpStream( thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7)) a = SpeadReceiverApp('a', 'a', port=port, itemId=itemId) b = InMemoryDROP('b', 'b') a.addOutput(b) size = 1024 threading.Thread(target=lambda: a.execute()).start() time.sleep(1) msg = os.urandom(size) with DROPWaiterCtx(self, b, timeout=1): ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48)) item = ig.add_item(itemId, 'main_data', 'a char array', shape=(size, ), format=[('c', 8)]) item.value = msg self._stream.send_heap(ig.get_heap()) self._stream.send_heap(ig.get_end()) for drop in a, b: self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual(size, b.size) self.assertEqual(msg, droputils.allDropContents(b))
def test_multipleProducers(self): """ A test that checks that multiple-producers correctly drive the state of their shared output """ class App(BarrierAppDROP): pass a, b, c, d, e = [App(chr(ord("A") + i), chr(ord("A") + i)) for i in range(5)] f = InMemoryDROP("F", "F") for drop in a, b, c, d, e: drop.addOutput(f) self.assertEqual(DROPStates.INITIALIZED, f.status) for drop in a, b, c, d, e: self.assertEqual(AppDROPStates.NOT_RUN, drop.execStatus) # Run the first 4 ones, F should still be in INITIALIZED for drop in a, b, c, d: drop.execute() self.assertEqual(DROPStates.INITIALIZED, f.status) self.assertEqual(AppDROPStates.NOT_RUN, e.execStatus) for drop in a, b, c, d: self.assertEqual(AppDROPStates.FINISHED, drop.execStatus) # Run the final one, now F should be COMPLETED e.execute() self.assertEqual(DROPStates.COMPLETED, f.status) for drop in a, b, c, d, e: self.assertEqual(AppDROPStates.FINISHED, drop.execStatus)
def test_namedPorts_with_kwonlyargs(self): """ Use a graph with named ports and check whether it is runnning """ init_oids = [ "2022-03-30T03:46:01_-2_0", "2022-03-30T03:46:01_-6_0", ] # first drops in graph sessionId = "lalo" with pkg_resources.resource_stream( "test", "graphs/pyfunc_glob_testPG.graph") as f: # @UndefinedVariable graphSpec = json.load(f) # dropSpecs = graph_loader.loadDropSpecs(graphSpec) self.createSessionAndAddGraph(sessionId, graphSpec=graphSpec) # Deploy now and get OIDs self.dim.deploySession(sessionId) fd = self.dm._sessions[sessionId].drops["2022-03-30T03:46:01_-1_0"] i = 0 start_drops = [InMemoryDROP(x, x) for x in ("a", "b")] for oid in init_oids: init_drop = self.dm._sessions[sessionId].drops[oid] init_drop.addInput(start_drops[i]) i += 1 logger.debug(f"PyfuncAPPDrop: {dir(fd)}") for i in fd.parameters["inputs"]: logger.debug(f"PyfuncAPPDrop input names:{i}") with droputils.DROPWaiterCtx(self, init_drop, 3): [a.setCompleted() for a in start_drops]
def _test_dynamic_write_withDropType(self, dropType): """ Test an AbstractDROP and a simple AppDROP (for checksum calculation) without an expected drop size (for app compatibility and not recommended in production) """ # NOTE: use_staging required for multiple writes to plasma drops a = dropType("oid:A", "uid:A", expectedSize=-1, use_staging=True) b = SumupContainerChecksum("oid:B", "uid:B") c = InMemoryDROP("oid:C", "uid:C") b.addInput(a) b.addOutput(c) test_crc = 0 with DROPWaiterCtx(self, c): for _ in range(self._test_num_blocks): a.write(self._test_block) test_crc = crc32c(self._test_block, test_crc) a.setCompleted() # Read the checksum from c cChecksum = int(droputils.allDropContents(c)) self.assertNotEqual(a.checksum, 0) self.assertEqual(a.checksum, test_crc) self.assertEqual(cChecksum, test_crc)
def test_func3_multioutput(self): """ Checks that func3 in this module works when wrapped as an application with multiple outputs. """ output_drops = [InMemoryDROP(x, x) for x in ('b', 'c', 'd')] self._test_func3(output_drops, ('b', 'c', 'd'))
def test_genericNpyScatter(self): data_in = random.rand(100, 100) b = InMemoryDROP("b", "b") droputils.save_numpy(b, data_in) s = GenericNpyScatterApp("s", "s", num_of_copies=2) s.addInput(b) o1 = InMemoryDROP("o1", "o1") o2 = InMemoryDROP("o2", "o2") for x in o1, o2: s.addOutput(x) self._test_graph_runs((b, s, o1, o2), b, (o1, o2), timeout=4) data1 = droputils.load_numpy(o1) data2 = droputils.load_numpy(o2) data_out = concatenate([data1, data2]) self.assertEqual(data_in.all(), data_out.all())
def test_errorState(self): a = InMemoryDROP("a", "a") b = SumupContainerChecksum("b", "b") c = InMemoryDROP("c", "c") c.addProducer(b) b.addInput(a) a.setError() self.assertEqual(DROPStates.ERROR, a.status) self.assertEqual(DROPStates.ERROR, b.status) self.assertEqual(DROPStates.ERROR, c.status)
def test_errorState(self): a = InMemoryDROP('a', 'a') b = SumupContainerChecksum('b', 'b') c = InMemoryDROP('c', 'c') c.addProducer(b) b.addInput(a) a.setError() self.assertEqual(DROPStates.ERROR, a.status) self.assertEqual(DROPStates.ERROR, b.status) self.assertEqual(DROPStates.ERROR, c.status)
def _do_test(func, expected_out, *args, **kwargs): # List with (drop, value) elements arg_inputs = [] # dict with name: (drop, value) items kwarg_inputs = {} translate = lambda x: base64.b64encode(pickle.dumps(x)) i = 0 for arg in args: si = 'uid_%d' % i arg_inputs.append(InMemoryDROP(si, si, pydata=translate(arg))) i += 1 for name, kwarg in kwargs.items(): si = 'uid_%d' % i kwarg_inputs[name] = (si, InMemoryDROP(si, si, pydata=translate(kwarg))) i += 1 a = InMemoryDROP('a', 'a', pydata=translate(1)) output = InMemoryDROP('o', 'o') app = _PyFuncApp('f', 'f', func, func_arg_mapping={name: vals[0] for name, vals in kwarg_inputs.items()}) app.addInput(a) app.addOutput(output) for drop in arg_inputs + [x[1] for x in kwarg_inputs.values()]: app.addInput(drop) with droputils.DROPWaiterCtx(self, output): a.setCompleted() for i in arg_inputs + [x[1] for x in kwarg_inputs.values()]: i.setCompleted() self.assertEqual(expected_out, pickle.loads(droputils.allDropContents(output))) # @UndefinedVariable
def test_dropAccess(self): a1 = InMemoryDROP('a', 'a1') registry = RDBMSRegistry('sqlite3', DBFILE) registry.addDrop(a1) self.assertEqual(-1, registry.getLastAccess('a')) registry.recordNewAccess('a') self.assertNotEqual(-1, registry.getLastAccess('a'))
def test_no_write_to_file_drop(self): """Check that FileDrops can be *not* written""" a = FileDROP("a", "a") b = SleepAndCopyApp("b", "b") c = InMemoryDROP("c", "c") a.addConsumer(b) b.addOutput(c) with DROPWaiterCtx(self, c): a.setCompleted() self.assertEqual(droputils.allDropContents(c), b"")
def test_ngasio(self): nd_in = NgasDROP("HelloWorld.txt", "HelloWorld.txt") nd_in.ngasSrv = "ngas.ddns.net" b = CopyApp("b", "b") did = "HelloWorld-%f" % time.time() nd_out = NgasDROP(did, did, len=11) nd_out.ngasSrv = "ngas.ddns.net" nd_out.len = nd_in.size d = CopyApp("d", "d") i = InMemoryDROP("i", "i") # b.addInput(nd_in) # b.addOutput(nd_out) nd_in.addConsumer(b) nd_out.addProducer(b) d.addInput(nd_out) i.addProducer(d) # b.addOutput(i) self._test_graph_runs((nd_in, b, nd_out, i, d), nd_in, i, timeout=10) self.assertEqual(b"Hello World", droputils.allDropContents(i))
def test_dropAccess(self): a1 = InMemoryDROP("a", "a1") registry = RDBMSRegistry("sqlite3", DBFILE) registry.addDrop(a1) self.assertEqual(-1, registry.getLastAccess("a")) registry.recordNewAccess("a") self.assertNotEqual(-1, registry.getLastAccess("a"))
def test_randomarrayapp(self): i = NullDROP("i", "i") c = RandomArrayApp("c", "c", keep_array=True) o = InMemoryDROP("o", "o") c.addInput(i) c.addOutput(o) self._test_graph_runs((i, c, o), i, o) marray = c._getArray() data = pickle.loads(droputils.allDropContents(o)) v = marray == data self.assertEqual(v.all(), True)
def test_genericNpyScatter_multi(self): data1_in = random.rand(100, 100) data2_in = random.rand(100, 100) b = InMemoryDROP("b", "b") c = InMemoryDROP("c", "c") droputils.save_numpy(b, data1_in) droputils.save_numpy(c, data2_in) s = GenericNpyScatterApp("s", "s", num_of_copies=2, scatter_axes="[0,0]") s.addInput(b) s.addInput(c) o1 = InMemoryDROP("o1", "o1") o2 = InMemoryDROP("o2", "o2") o3 = InMemoryDROP("o3", "o3") o4 = InMemoryDROP("o4", "o4") for x in o1, o2, o3, o4: s.addOutput(x) self._test_graph_runs((b, s, o1, o2, o3, o4), (b, c), (o1, o2, o3, o4), timeout=4) data11 = droputils.load_numpy(o1) data12 = droputils.load_numpy(o2) data1_out = concatenate([data11, data12]) self.assertEqual(data1_out.shape, data1_in.shape) testing.assert_array_equal(data1_out, data1_in) data21 = droputils.load_numpy(o3) data22 = droputils.load_numpy(o4) data2_out = concatenate([data21, data22]) testing.assert_array_equal(data2_out, data2_in)
def test_two_simultaneous_pipes(self): """ A more complicated test where three bash applications run at the same time. The first streams its output to the second one, while the second one streams *its* output to the third one. ------------- -------------- ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F | | echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | | -----*------- -------------- ----*--*----- -------------- -----*------- ---------- | | | | +-------------|named-pipe|----------+ +-----------|named-pipe|-----------+ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by "cat" (BashApp C). The printed results (a copy of the original) are streamed through D and read by "sort" (BashApp E), which writes the output to F. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp('a', 'a', command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP('b', 'b') c = StreamingInputOutputBashApp('c', 'c', command="cat") d = InMemoryDROP('d', 'd') e = StreamingInputBashApp('e', 'e', command="sort -n > %o0") f = FileDROP('f', 'f', filepath=output_fname) a.addOutput(b) b.addStreamingConsumer(c) c.addOutput(d) d.addStreamingConsumer(e) e.addOutput(f) # Let's fire the app with DROPWaiterCtx(self, f, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d, e, f): self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual([1, 2, 3, 4, 5], [ int(x) for x in droputils.allDropContents(f).strip().split(six.b('\n')) ]) # Clean up and go os.remove(output_fname)
def _test_copyapp_simple(self, app): # Again, not foo fancy, simple apps require simple tests a, c = (InMemoryDROP(x, x) for x in ("a", "c")) b = app("b", "b") b.addInput(a) b.addOutput(c) data = os.urandom(32) a.write(data) self._test_graph_runs((a, b, c), a, c) self.assertEqual(data, droputils.allDropContents(c))
def test_addDrop(self): a = InMemoryDROP('a', 'a') registry = RDBMSRegistry('sqlite3', DBFILE) registry.addDrop(a) conn = sqlite3.connect(DBFILE) # @UndefinedVariable cur = conn.cursor() cur.execute('SELECT oid FROM dlg_drop'); r = cur.fetchone() self.assertEqual(1, len(r)) self.assertEqual('a', r[0]) cur.close() conn.close()
def test_multi_listappendthrashing(self, size=1000, parallel=True): max_threads = cpu_count(logical=False) drop_ids = [chr(97 + x) for x in range(max_threads)] threadpool = ThreadPool(processes=max_threads) memory_manager = DlgSharedMemoryManager() session_id = 1 memory_manager.register_session(session_id) S = InMemoryDROP("S", "S") X = AverageArraysApp("X", "X") Z = InMemoryDROP("Z", "Z") drops = [ListAppendThrashingApp(x, x, size=size) for x in drop_ids] mdrops = [ InMemoryDROP(chr(65 + x), chr(65 + x)) for x in range(max_threads) ] if parallel: # a bit of magic to get the app drops using the processes _ = [drop.__setattr__("_tp", threadpool) for drop in drops] _ = [drop.__setattr__("_tp", threadpool) for drop in mdrops] _ = [drop.__setattr__("_sessID", session_id) for drop in mdrops] _ = [ memory_manager.register_drop(drop.uid, session_id) for drop in mdrops ] X.__setattr__("_tp", threadpool) Z.__setattr__("_tp", threadpool) Z.__setattr__("_sessID", session_id) memory_manager.register_drop(Z.uid, session_id) _ = [d.addInput(S) for d in drops] _ = [d.addOutput(m) for d, m in zip(drops, mdrops)] _ = [X.addInput(m) for m in mdrops] X.addOutput(Z) logger.info( f"Number of inputs/outputs: {len(X.inputs)}, {len(X.outputs)}") self._test_graph_runs([S, X, Z] + drops + mdrops, S, Z, timeout=200) # Need to run our 'copy' of the averaging APP num_array = [] for drop in mdrops: buf = droputils.allDropContents(drop) num_array.extend(pickle.loads(buf)) X.marray = num_array average = X.averageArray() # Load actual results graph_result = droputils.allDropContents(Z) graph_result = pickle.loads(graph_result) self.assertEqual(graph_result, average) # Must be called to unlink all shared memory memory_manager.shutdown_all()
def test_ArrayLoop(self): """ Use a graph with compile function to test positional only arguments """ sessionId = "lalo" start_drop = InMemoryDROP('a', 'a') with pkg_resources.resource_stream( "test", "graphs/ArrayLoopPG.graph") as f: # @UndefinedVariable graphSpec = json.load(f) # dropSpecs = graph_loader.loadDropSpecs(graphSpec) self.createSessionAndAddGraph(sessionId, graphSpec=graphSpec) # Deploy now and get OIDs self.dim.deploySession(sessionId) sd = self.dm._sessions[sessionId].drops["2022-06-22T09:13:53_-1_0"] sd.addInput(start_drop) fd = self.dm._sessions[sessionId].drops["2022-06-22T09:13:53_-4_0/0/0"] with droputils.DROPWaiterCtx(self, fd, 3): start_drop.setCompleted() #logger.debug(f'PyfuncAPPDrop signature: {dir(fd)}') logger.debug(f'PyfuncAPPDrop status: {fd.status}') self.assertEqual(2, fd.status)
def _test_simple_functions(self, f, input_data, output_data): a, c = [InMemoryDROP(x, x) for x in ('a', 'c')] b = _PyFuncApp('b', 'b', f) b.addInput(a) b.addOutput(c) with DROPWaiterCtx(self, c, 5): a.write(pickle.dumps(input_data)) # @UndefinedVariable a.setCompleted() for drop in a, b, c: self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual(output_data, pickle.loads(droputils.allDropContents(c))) # @UndefinedVariable
def _test_socket_listener(self, **kwargs): ''' A simple test to check that SocketListenerApps are indeed working as expected; that is, they write the data they receive into their output, and finish when the connection is closed from the client side The data flow diagram looks like this: A --> B --> C --> D ''' host = 'localhost' port = 9933 data = os.urandom(1025) a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port, **kwargs) b = InMemoryDROP('oid:B', 'uid:B') c = SumupContainerChecksum('oid:C', 'uid:C') d = InMemoryDROP('oid:D', 'uid:D') a.addOutput(b) b.addConsumer(c) c.addOutput(d) # Create the socket, write, and close the connection, allowing # A to move to COMPLETED with DROPWaiterCtx(self, d, 3): # That's plenty of time a.async_execute() utils.write_to(host, port, data, 1) for drop in [a, b, c, d]: self.assertEqual(DROPStates.COMPLETED, drop.status) # Our expectations are fulfilled! bContents = droputils.allDropContents(b) dContents = int(droputils.allDropContents(d)) self.assertEqual(data, bContents) self.assertEqual(crc32(data, 0), dContents)