def test_basic_run(self):
signal = SignalGenerateAndAverageDrop('1', '1',
internal_port=internal_port,
stream_port=stream_port,
start_freq=int(os.environ.get('START_FREQ', 45991200)),
freq_step=int(os.environ.get('FREQ_STEP', 6400)),
use_gpus=int(os.environ.get('USE_GPUS', 0)),
num_freq_steps=int(os.environ.get('NUM_CHANNELS', 1)),
telescope_model_path='./conf/%s.tm' % tm,
sky_model_file_path="./conf/eor_model_list.csv",
num_time_steps=int(os.environ.get('NUM_TIME_STEPS', 1)))
sink = AveragerSinkDrop('2', '2',
stream_listen_port_start=stream_port,
use_adios2=int(os.environ.get('USE_ADIOS2', 0)),
baseline_exclusion_map_path='./conf/%s_baselines.csv' % tm,
node='127.0.0.1')
drop = InMemoryDROP('3', '3')
drop.addStreamingConsumer(sink)
signal.addOutput(drop)
ms = FileDROP('4', '4', filepath=output)
sink.addOutput(ms)
with droputils.DROPWaiterCtx(self, ms, 1000):
signal.async_execute()
def test_objectAsNormalAndStreamingInput(self):
"""
A test that checks that a DROP can act as normal and streaming input of
different AppDROPs at the same time. We use the following graph:
A --|--> B --> D
|--> C --> E
Here B uses A as a streaming input, while C uses it as a normal
input
"""
class LastCharWriterApp(AppDROP):
def initialize(self, **kwargs):
super(LastCharWriterApp, self).initialize(**kwargs)
self._lastByte = None
def dataWritten(self, uid, data):
self.execStatus = AppDROPStates.RUNNING
outputDrop = self.outputs[0]
self._lastByte = data[-1:]
outputDrop.write(self._lastByte)
def dropCompleted(self, uid, status):
self.execStatus = AppDROPStates.FINISHED
self._notifyAppIsFinished()
a = InMemoryDROP('a', 'a')
b = LastCharWriterApp('b', 'b')
c = SumupContainerChecksum('c', 'c')
d = InMemoryDROP('d', 'd')
e = InMemoryDROP('e', 'e')
a.addStreamingConsumer(b)
a.addConsumer(c)
b.addOutput(d)
c.addOutput(e)
# Consumer cannot be normal and streaming at the same time
self.assertRaises(Exception, a.addConsumer, b)
self.assertRaises(Exception, a.addStreamingConsumer, c)
# Write a little, then check the consumers
def checkDropStates(aStatus, dStatus, eStatus, lastByte):
self.assertEqual(aStatus, a.status)
self.assertEqual(dStatus, d.status)
self.assertEqual(eStatus, e.status)
if lastByte is not None:
self.assertEqual(lastByte, b._lastByte)
checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
a.write(b'abcde')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, b'e')
a.write(b'fghij')
checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, b'j')
a.write(b'k')
with DROPWaiterCtx(self, [d, e]):
a.setCompleted()
checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, b'k')
self.assertEqual(b'ejk', droputils.allDropContents(d))
def test_two_simultaneous_pipes(self):
"""
A more complicated test where three bash applications run at the same
time. The first streams its output to the second one, while the second
one streams *its* output to the third one.
------------- -------------- ------------- -------------- ------------- ----------
| BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F |
| echo | | "/pipe1" | | dc | | "/pipe2" | | sort | | |
-----*------- -------------- ----*--*----- -------------- -----*------- ----------
| | | |
+-------------|named-pipe|----------+ +-----------|named-pipe|-----------+
BashApp A writes "5 4 3 2 1" (each on a new line), which is read
by "cat" (BashApp C). The printed results (a copy of the original) are
streamed through D and read by "sort" (BashApp E), which writes the
output to F.
"""
output_fname = tempfile.mktemp()
a = StreamingOutputBashApp('a',
'a',
command=r"echo -en '5\n4\n3\n2\n1'")
b = InMemoryDROP('b', 'b')
c = StreamingInputOutputBashApp('c', 'c', command="cat")
d = InMemoryDROP('d', 'd')
e = StreamingInputBashApp('e', 'e', command="sort -n > %o0")
f = FileDROP('f', 'f', filepath=output_fname)
a.addOutput(b)
b.addStreamingConsumer(c)
c.addOutput(d)
d.addStreamingConsumer(e)
e.addOutput(f)
# Let's fire the app
with DROPWaiterCtx(self, f, 2):
a.async_execute()
# The application executed, finished, and its output was recorded
for drop in (a, b, c, d, e, f):
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual([1, 2, 3, 4, 5], [
int(x)
for x in droputils.allDropContents(f).strip().split(six.b('\n'))
])
# Clean up and go
os.remove(output_fname)