async def build_fd_pipes(pipe_args: str,
node: BaseNode) -> Tuple[Pipes, Pipes]:
try:
pipe_json = json.loads(json.loads(pipe_args))
# if debugging, pycharm escapes the outer JSON
# pipe_json = json.loads(pipe_args.encode('utf-8').decode('unicode_escape'))
dest_args = pipe_json['outputs']
src_args = pipe_json['inputs']
except (KeyError, json.JSONDecodeError):
raise errors.ConfigurationError("invalid pipes argument: [%s]" %
pipe_args)
pipes_out = {}
pipes_in = {}
for name, arg in dest_args.items():
wf = pipes.writer_factory(arg['fd'])
dest_stream = None
if arg['id'] is not None: # used in testing when no API is available
dest_stream = await node.data_stream_get(arg['id'])
pipes_out[name] = pipes.OutputPipe(stream=dest_stream,
layout=arg['layout'],
writer_factory=wf)
for name, arg in src_args.items():
rf = pipes.reader_factory(arg['fd'])
src_stream = None
if arg['id'] is not None: # used in testing when no API is available
src_stream = await node.data_stream_get(arg['id'])
pipes_in[name] = pipes.InputPipe(stream=src_stream,
layout=arg['layout'],
reader_factory=rf)
return pipes_in, pipes_out
async def _spawn_outputs(self) -> asyncio.Task:
tasks: List[asyncio.Task] = []
# configure output pipes [module]==>[worker]
for (name, stream) in self.module.outputs.items():
(r, w) = os.pipe()
rf = pipes.reader_factory(r)
os.set_inheritable(w, True)
pipe = pipes.InputPipe(name=name, stream=stream, reader_factory=rf)
self.output_connections.append(
DataConnection(name, w, stream, pipe))
t = asyncio.create_task(
self._output_handler(pipe, self.subscribers[stream]))
t.set_name("worker [%s]: output [%s]" %
(self.module.name, stream.name))
tasks.append(t)
return asyncio.gather(*tasks)
async def _build_pipes_new(self, interval, input_streams, output_streams,
pipe_args) -> Tuple[Pipes, Pipes]:
input_pipes = {}
output_pipes = {}
# use network sockets for connection to inputs and outputs
if pipe_args == 'unset':
for (name, stream) in input_streams.items():
if interval is None: # subscribe to live data
input_pipes[name] = await self.node.data_subscribe(stream)
else:
input_pipes[name] = await self.node.data_read(
stream, interval[0], interval[1])
for (name, stream) in output_streams.items():
if interval is None:
output_pipes[name] = await self.node.data_write(stream)
else:
output_pipes[name] = await self.node.data_write(
stream, interval[0], interval[1])
# use file descriptors provided by joule for connection to inputs and outputs
else:
try:
pipe_json = json.loads(json.loads(pipe_args))
# if debugging, pycharm escapes the outer JSON
# pipe_json = json.loads(pipe_args.encode('utf-8').decode('unicode_escape'))
output_args = pipe_json['outputs']
input_args = pipe_json['inputs']
except (KeyError, json.JSONDecodeError):
raise ConfigurationError(
f"invalid pipes argument: {pipe_args}")
for name, arg in output_args.items():
wf = pipes.writer_factory(arg['fd'])
output_pipes[name] = pipes.OutputPipe(
stream=output_streams[name],
layout=arg['layout'],
writer_factory=wf)
for name, arg in input_args.items():
rf = pipes.reader_factory(arg['fd'])
input_pipes[name] = pipes.InputPipe(stream=input_streams[name],
layout=arg['layout'],
reader_factory=rf)
# keep track of the pipes so they can be closed
self.pipes = list(input_pipes.values()) + list(output_pipes.values())
return input_pipes, output_pipes
def test_writes_to_pipes(self):
module = SimpleReader()
(r, w) = os.pipe()
loop = asyncio.get_event_loop()
rf = pipes.reader_factory(r)
pipe = pipes.InputPipe(name="output",
stream=self.stream,
reader_factory=rf)
pipe_arg = json.dumps(
json.dumps({
"outputs": {
'output': {
'fd': w,
'id': None,
'layout': self.stream.layout
}
},
"inputs": {}
}))
data = helpers.create_data(self.stream.layout)
args = argparse.Namespace(pipes=pipe_arg,
socket="unset",
url='http://localhost:8080',
node="",
api_socket="",
mock_data=data)
# run the reader module
loop = asyncio.new_event_loop()
loop.set_debug(True)
asyncio.set_event_loop(loop)
module.start(args)
asyncio.set_event_loop(self.loop)
# check the output
received_data = self.loop.run_until_complete(pipe.read())
np.testing.assert_array_equal(data, received_data)
self.loop.run_until_complete(pipe.close())
if not loop.is_closed():
loop.close()
def test_runs_composited_modules(self):
module = SimpleComposite()
(r, w) = os.pipe()
module_loop = asyncio.new_event_loop()
rf = pipes.reader_factory(r)
pipe = pipes.InputPipe(name="output", stream=self.stream, reader_factory=rf)
pipe_arg = json.dumps(json.dumps({"outputs": {'output': {'fd': w, 'id': None, 'layout': self.stream.layout}},
"inputs": {}}))
data = helpers.create_data(self.stream.layout)
args = argparse.Namespace(pipes=pipe_arg, socket="unset",
node="", api_socket="",
url='http://localhost:8080',
mock_data=data)
# run the composite module
asyncio.set_event_loop(module_loop)
module.start(args)
asyncio.set_event_loop(self.loop)
# check the output
received_data = self.loop.run_until_complete(pipe.read())
np.testing.assert_array_equal(data['timestamp'], received_data['timestamp'])
np.testing.assert_array_almost_equal(data['data'] * 2, received_data['data'])
self.loop.run_until_complete(pipe.close())
self.loop.close()
def test_writes_to_pipes(self):
module = SimpleFilter()
(r, w_module) = os.pipe()
loop = asyncio.get_event_loop()
rf = pipes.reader_factory(r)
from_filter = pipes.InputPipe(name="from_filter",
stream=self.output,
reader_factory=rf)
(r_module, w) = os.pipe()
loop = asyncio.get_event_loop()
wf = pipes.writer_factory(w)
to_filter = pipes.OutputPipe(name="to_filter",
stream=self.input,
writer_factory=wf)
pipe_arg = json.dumps(
json.dumps({
"outputs": {
'from_filter': {
'fd': w_module,
'id': 2,
'layout': self.output.layout
}
},
"inputs": {
'to_filter': {
'fd': r_module,
'id': 3,
'layout': self.input.layout
}
}
}))
data = helpers.create_data(self.input.layout)
self.loop.run_until_complete(to_filter.write(data))
self.loop.run_until_complete(to_filter.close())
args = argparse.Namespace(pipes=pipe_arg,
socket="unset",
node="",
api_socket="",
live=False,
url='http://localhost:8080')
# run the reader module
loop = asyncio.new_event_loop()
loop.set_debug(True)
asyncio.set_event_loop(loop)
class MockNode(BaseNode):
def __init__(self):
self.session = mock.Mock()
self.session.close = asynctest.CoroutineMock()
pass
@property
def loop(self):
return asyncio.get_event_loop()
with mock.patch('joule.client.base_module.node') as mock_node_pkg:
node = MockNode()
node.data_stream_get = asynctest.CoroutineMock(
return_value=self.output)
mock_node_pkg.UnixNode = mock.Mock(return_value=node)
module.start(args)
# make sure the API was used to retrieve stream objects
self.assertEqual(node.data_stream_get.await_count, 2)
asyncio.set_event_loop(self.loop)
# check the output
received_data = self.loop.run_until_complete(from_filter.read())
np.testing.assert_array_equal(data['timestamp'],
received_data['timestamp'])
np.testing.assert_array_almost_equal(data['data'] * 2,
received_data['data'])
self.loop.run_until_complete(from_filter.close())
if not loop.is_closed():
loop.close()