def test_different_format_writes(self):
LAYOUT = "int8_2"
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT, name="testpipe")
test_data = helpers.create_data(LAYOUT, length=4, step=1, start=106)
async def write():
# write unstructured numpy arrays
await my_pipe.write(np.array([[1, 1, 1]]))
await my_pipe.write(np.array([[2, 2, 2], [3, 3, 3]]))
# logs empty writes
with self.assertLogs(level="INFO") as logs:
await my_pipe.write(np.array([[]]))
# errors on invalid write types
bad_data = [[100, 1, 2], 'invalid', 4, None, np.array([4, 8])]
for data in bad_data:
with self.assertRaises(PipeError):
await my_pipe.write(data)
# write structured numpy arrays
await my_pipe.write(test_data)
loop = asyncio.get_event_loop()
loop.run_until_complete(write())
result = my_pipe.read_nowait(flatten=True)
np.testing.assert_array_equal(result[:3],
[[1, 1, 1], [2, 2, 2], [3, 3, 3]])
my_pipe.consume(3)
result = my_pipe.read_nowait()
np.testing.assert_array_equal(result, test_data)
def test_nowait_read_does_not_block(self):
LAYOUT = "int8_2"
LENGTH = 50
my_pipe = LocalPipe(LAYOUT)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data)
self.assertEqual(len(my_pipe.read_nowait()), LENGTH)
# another read executes immediately because there is unconsumed data
self.assertEqual(len(my_pipe.read_nowait()), LENGTH)
# now consume the data and the next call to read execute immediately
# even though there is no data to be returned
my_pipe.consume(LENGTH)
self.assertEqual(0, len(my_pipe.read_nowait(flatten=True)))
return
def test_sends_data_to_subscribers(self):
LAYOUT = "float32_2"
(fd_r, fd_w) = os.pipe()
loop = asyncio.get_event_loop()
output_cb = CoroutineMock()
input_cb = CoroutineMock()
subscriber_cb = CoroutineMock()
npipe_out = OutputPipe(layout=LAYOUT, writer_factory=writer_factory(fd_w),
close_cb=output_cb)
subscriber = LocalPipe(layout=LAYOUT, close_cb=subscriber_cb)
npipe_out.subscribe(subscriber)
test_data = helpers.create_data(LAYOUT)
loop.run_until_complete(npipe_out.write(test_data))
# data should be available on the InputPipe side
npipe_in = InputPipe(layout=LAYOUT, reader_factory=reader_factory(fd_r),
close_cb=input_cb)
rx_data = loop.run_until_complete(npipe_in.read())
np.testing.assert_array_equal(test_data, rx_data)
# data should also be available on the Subscriber output
rx_data = subscriber.read_nowait()
np.testing.assert_array_equal(test_data, rx_data)
loop.run_until_complete(asyncio.gather(npipe_in.close(),
npipe_out.close()))
# subscriber should be closed
self.assertTrue(subscriber.closed)
# make sure all of the callbacks have been executed
self.assertEqual(output_cb.call_count, 1)
self.assertEqual(input_cb.call_count, 1)
self.assertEqual(subscriber_cb.call_count, 1)
def test_nowait_read_writes(self):
LAYOUT = "int8_2"
LENGTH = 500
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT)
my_subscriber = LocalPipe(LAYOUT)
my_pipe.subscribe(my_subscriber)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data)
result = my_pipe.read_nowait()
# read data should match written data
np.testing.assert_array_almost_equal(test_data['data'], result['data'])
np.testing.assert_array_almost_equal(test_data['timestamp'],
result['timestamp'])
# subscriber should have a copy too
result = my_subscriber.read_nowait()
np.testing.assert_array_almost_equal(test_data['data'], result['data'])
np.testing.assert_array_almost_equal(test_data['timestamp'],
result['timestamp'])
def test_raises_consume_errors(self):
LAYOUT = "int32_3"
LENGTH = 1000
my_pipe = LocalPipe(LAYOUT)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data)
read_data = my_pipe.read_nowait()
# can't consume more than was read
with self.assertRaises(PipeError) as e:
my_pipe.consume(len(read_data) + 1)
self.assertTrue('consume' in str(e.exception))
# can't consume less than zero
with self.assertRaises(PipeError) as e:
my_pipe.consume(-1)
self.assertTrue('negative' in str(e.exception))
def test_nowait_read_empties_queue(self):
LAYOUT = "int8_2"
LENGTH = 50
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
async def writer():
for row in test_data:
await my_pipe.write(np.array([row]))
loop = asyncio.get_event_loop()
loop.run_until_complete(writer())
result = my_pipe.read_nowait()
np.testing.assert_array_almost_equal(test_data['data'], result['data'])
np.testing.assert_array_almost_equal(test_data['timestamp'],
result['timestamp'])
def test_pipes_numpy_arrays(self):
"""writes to pipe sends data to reader and any subscribers"""
LAYOUT = "int8_2"
LENGTH = 1003
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT, name="pipe")
# test timeouts, since the writer is slow
my_pipe.TIMEOUT_INTERVAL = 0.05
subscriber_pipe = LocalPipe(LAYOUT)
subscriber_pipe2 = LocalPipe(LAYOUT)
my_pipe.subscribe(subscriber_pipe)
my_pipe.subscribe(subscriber_pipe2)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
reader_rx_data = np.empty(test_data.shape, test_data.dtype)
subscriber_rx_data = np.empty(test_data.shape, test_data.dtype)
# print(test_data['data'][:,1])
async def writer():
for block in helpers.to_chunks(test_data, 270):
await asyncio.sleep(0.1)
await my_pipe.write(block)
await my_pipe.close()
async def reader():
# note this is kind of delicate, you can only read data twice
# if the pipe is closed so it might look like you didn't finish all
# the data if the last read ignores data past blk_size
blk_size = 357
rx_idx = 0
while not my_pipe.is_empty():
# consume data in pipe
data_chunk = await my_pipe.read()
rows_used = min(len(data_chunk), blk_size)
reader_rx_data[rx_idx:rx_idx +
rows_used] = data_chunk[:rows_used]
rx_idx += rows_used
my_pipe.consume(rows_used)
async def subscriber():
rx_idx = 0
while not subscriber_pipe.is_empty():
# consume data in pipe
data_chunk = await subscriber_pipe.read()
subscriber_rx_data[rx_idx:rx_idx +
len(data_chunk)] = data_chunk
rx_idx += len(data_chunk)
subscriber_pipe.consume(len(data_chunk))
loop = asyncio.get_event_loop()
tasks = [
asyncio.ensure_future(writer()),
asyncio.ensure_future(reader()),
asyncio.ensure_future(subscriber())
]
loop.run_until_complete(asyncio.gather(*tasks))
data = subscriber_pipe2.read_nowait()
np.testing.assert_array_equal(test_data, data)
np.testing.assert_array_equal(test_data, reader_rx_data)
np.testing.assert_array_equal(test_data, subscriber_rx_data)