def test_basic_run(self): signal = SignalGenerateAndAverageDrop('1', '1', internal_port=internal_port, stream_port=stream_port, start_freq=int(os.environ.get('START_FREQ', 45991200)), freq_step=int(os.environ.get('FREQ_STEP', 6400)), use_gpus=int(os.environ.get('USE_GPUS', 0)), num_freq_steps=int(os.environ.get('NUM_CHANNELS', 1)), telescope_model_path='./conf/%s.tm' % tm, sky_model_file_path="./conf/eor_model_list.csv", num_time_steps=int(os.environ.get('NUM_TIME_STEPS', 1))) sink = AveragerSinkDrop('2', '2', stream_listen_port_start=stream_port, use_adios2=int(os.environ.get('USE_ADIOS2', 0)), baseline_exclusion_map_path='./conf/%s_baselines.csv' % tm, node='127.0.0.1') drop = InMemoryDROP('3', '3') drop.addStreamingConsumer(sink) signal.addOutput(drop) ms = FileDROP('4', '4', filepath=output) sink.addOutput(ms) with droputils.DROPWaiterCtx(self, ms, 1000): signal.async_execute()
def test_objectAsNormalAndStreamingInput(self): """ A test that checks that a DROP can act as normal and streaming input of different AppDROPs at the same time. We use the following graph: A --|--> B --> D |--> C --> E Here B uses A as a streaming input, while C uses it as a normal input """ class LastCharWriterApp(AppDROP): def initialize(self, **kwargs): super(LastCharWriterApp, self).initialize(**kwargs) self._lastByte = None def dataWritten(self, uid, data): self.execStatus = AppDROPStates.RUNNING outputDrop = self.outputs[0] self._lastByte = data[-1:] outputDrop.write(self._lastByte) def dropCompleted(self, uid, status): self.execStatus = AppDROPStates.FINISHED self._notifyAppIsFinished() a = InMemoryDROP('a', 'a') b = LastCharWriterApp('b', 'b') c = SumupContainerChecksum('c', 'c') d = InMemoryDROP('d', 'd') e = InMemoryDROP('e', 'e') a.addStreamingConsumer(b) a.addConsumer(c) b.addOutput(d) c.addOutput(e) # Consumer cannot be normal and streaming at the same time self.assertRaises(Exception, a.addConsumer, b) self.assertRaises(Exception, a.addStreamingConsumer, c) # Write a little, then check the consumers def checkDropStates(aStatus, dStatus, eStatus, lastByte): self.assertEqual(aStatus, a.status) self.assertEqual(dStatus, d.status) self.assertEqual(eStatus, e.status) if lastByte is not None: self.assertEqual(lastByte, b._lastByte) checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None) a.write(b'abcde') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, b'e') a.write(b'fghij') checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, b'j') a.write(b'k') with DROPWaiterCtx(self, [d, e]): a.setCompleted() checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, b'k') self.assertEqual(b'ejk', droputils.allDropContents(d))
def test_two_simultaneous_pipes(self): """ A more complicated test where three bash applications run at the same time. The first streams its output to the second one, while the second one streams *its* output to the third one. ------------- -------------- ------------- -------------- ------------- ---------- | BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F | | echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | | -----*------- -------------- ----*--*----- -------------- -----*------- ---------- | | | | +-------------|named-pipe|----------+ +-----------|named-pipe|-----------+ BashApp A writes "5 4 3 2 1" (each on a new line), which is read by "cat" (BashApp C). The printed results (a copy of the original) are streamed through D and read by "sort" (BashApp E), which writes the output to F. """ output_fname = tempfile.mktemp() a = StreamingOutputBashApp('a', 'a', command=r"echo -en '5\n4\n3\n2\n1'") b = InMemoryDROP('b', 'b') c = StreamingInputOutputBashApp('c', 'c', command="cat") d = InMemoryDROP('d', 'd') e = StreamingInputBashApp('e', 'e', command="sort -n > %o0") f = FileDROP('f', 'f', filepath=output_fname) a.addOutput(b) b.addStreamingConsumer(c) c.addOutput(d) d.addStreamingConsumer(e) e.addOutput(f) # Let's fire the app with DROPWaiterCtx(self, f, 2): a.async_execute() # The application executed, finished, and its output was recorded for drop in (a, b, c, d, e, f): self.assertEqual(DROPStates.COMPLETED, drop.status) self.assertEqual([1, 2, 3, 4, 5], [ int(x) for x in droputils.allDropContents(f).strip().split(six.b('\n')) ]) # Clean up and go os.remove(output_fname)