def test_handles_interval_breaks(self):
LAYOUT = "int32_3"
LENGTH = 1000
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
async def reader():
# read the first interval
read_data = await my_pipe.read()
self.assertTrue(my_pipe.end_of_interval)
my_pipe.consume(len(read_data) - 20)
np.testing.assert_array_equal(test_data1, read_data)
# read the second interval
read_data = await my_pipe.read()
self.assertFalse(my_pipe.end_of_interval)
self.assertEqual(len(read_data), len(test_data2) + 20)
my_pipe.consume(len(read_data))
np.testing.assert_array_equal(test_data2, read_data[20:])
loop = asyncio.get_event_loop()
loop.run_until_complete(reader())
def test_invalid_write_nowait_inputs(self):
LAYOUT = "int8_2"
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT, name="testpipe")
with self.assertLogs(level="INFO"):
my_pipe.write_nowait(np.array([[]]))
with self.assertRaises(PipeError):
my_pipe.write_nowait([1, 2, 3])
def test_nowait_read_does_not_block(self):
LAYOUT = "int8_2"
LENGTH = 50
my_pipe = LocalPipe(LAYOUT)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data)
self.assertEqual(len(my_pipe.read_nowait()), LENGTH)
# another read executes immediately because there is unconsumed data
self.assertEqual(len(my_pipe.read_nowait()), LENGTH)
# now consume the data and the next call to read execute immediately
# even though there is no data to be returned
my_pipe.consume(LENGTH)
self.assertEqual(0, len(my_pipe.read_nowait(flatten=True)))
return
def test_raises_consume_errors(self):
LAYOUT = "int32_3"
LENGTH = 1000
my_pipe = LocalPipe(LAYOUT)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data)
read_data = my_pipe.read_nowait()
# can't consume more than was read
with self.assertRaises(PipeError) as e:
my_pipe.consume(len(read_data) + 1)
self.assertTrue('consume' in str(e.exception))
# can't consume less than zero
with self.assertRaises(PipeError) as e:
my_pipe.consume(-1)
self.assertTrue('negative' in str(e.exception))
def test_read_data_must_be_consumed(self):
#writes to pipe sends data to reader and any subscribers
LAYOUT = "float32_2"
LENGTH = 500
UNCONSUMED_ROWS = 4
my_pipe = LocalPipe(LAYOUT)
chunk1 = helpers.create_data(LAYOUT, length=LENGTH)
chunk2 = helpers.create_data(LAYOUT, length=LENGTH)
chunk3 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(chunk1)
async def reader():
await my_pipe.read()
my_pipe.consume(0)
# should get the same data back on the next read
# add a second copy of the test data
await my_pipe.write(chunk2)
rx_data = await my_pipe.read()
# two copies of the data now
np.testing.assert_array_equal(chunk1, rx_data[:len(chunk1)])
np.testing.assert_array_equal(chunk2, rx_data[len(chunk1):])
# write another copy but consume the first
my_pipe.consume(len(chunk1))
await my_pipe.write(chunk3)
rx_data = await my_pipe.read()
# two copies of the data now
np.testing.assert_array_equal(chunk2, rx_data[:len(chunk2)])
np.testing.assert_array_equal(chunk3, rx_data[len(chunk2):])
my_pipe.consume(len(chunk2))
await my_pipe.close()
# now a read should return immediately with the unconsumed data
rx_data = await my_pipe.read()
np.testing.assert_array_equal(chunk3, rx_data)
# the pipe should be empty but still return the old data
rx_data = await my_pipe.read()
np.testing.assert_array_equal(chunk3, rx_data)
# only after consuming the remaining data does it raise an exception
my_pipe.consume(len(rx_data))
# another read should cause an exception (even if the data hasn't been consumed)
with self.assertRaises(EmptyPipeError):
await my_pipe.read()
# the pipe should be empty
self.assertTrue(my_pipe.is_empty())
asyncio.run(reader())
def subscribe(self, stream: DataStream, pipe: pipes.LocalPipe):
if self.raise_error:
raise SubscriptionError()
self.subscribed_stream = stream
while True:
try:
data = self.subscription_pipe.read_nowait()
self.subscription_pipe.consume(len(data))
pipe.write_nowait(data)
except pipes.EmptyPipe:
if not self.hang_pipe:
pipe.close_nowait()
break
if self.subscription_pipe.end_of_interval:
pipe.close_interval_nowait()
# return a mock as the unsubscribe callback
return self.unsubscribe
def test_nowait_read_writes(self):
LAYOUT = "int8_2"
LENGTH = 500
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT)
my_subscriber = LocalPipe(LAYOUT)
my_pipe.subscribe(my_subscriber)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data)
result = my_pipe.read_nowait()
# read data should match written data
np.testing.assert_array_almost_equal(test_data['data'], result['data'])
np.testing.assert_array_almost_equal(test_data['timestamp'],
result['timestamp'])
# subscriber should have a copy too
result = my_subscriber.read_nowait()
np.testing.assert_array_almost_equal(test_data['data'], result['data'])
np.testing.assert_array_almost_equal(test_data['timestamp'],
result['timestamp'])
def test_nowait_function_exceptions(self):
pipe = LocalPipe(layout="float32_3")
subscriber = LocalPipe(layout="float32_3", close_cb=mock.Mock())
fd_subscriber = OutputPipe()
# cannot close_nowait if there is a callback
with self.assertRaises(PipeError):
subscriber.close_nowait()
# can write_nowait if only localpipe subscribers
pipe.subscribe(subscriber)
pipe.write_nowait(np.ones((100, 4)))
# cannot if any subscribers are not localpipes
pipe.subscribe(fd_subscriber)
with self.assertRaises(PipeError):
pipe.write_nowait(np.ones((100, 4)))
# cannot close_nowait if there are subscribers
pipe.subscribe(subscriber)
with self.assertRaises(PipeError):
pipe.close_nowait()
def test_read_all_no_flatten(self):
LAYOUT = "int32_3"
LENGTH = 1000
loop = asyncio.get_event_loop()
# raises exception if the pipe is empty
my_pipe = LocalPipe(LAYOUT, name="pipe")
my_pipe.close_nowait()
with self.assertRaises(PipeError):
loop.run_until_complete(my_pipe.read_all())
# read_all empties pipe and closes it, regardless of intervals
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
my_pipe.close_nowait()
actual_data = loop.run_until_complete(my_pipe.read_all())
expected_data = np.hstack((test_data1, test_data2))
np.testing.assert_array_equal(actual_data, expected_data)
self.assertTrue(my_pipe.closed)
# read_all only add maxrows to the pipe
# (less than one read)
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
actual_data = loop.run_until_complete(my_pipe.read_all(maxrows=103))
expected_data = test_data1[:103]
np.testing.assert_array_equal(actual_data, expected_data)
self.assertTrue(my_pipe.closed)
# (more than one read)
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
actual_data = loop.run_until_complete(
my_pipe.read_all(maxrows=LENGTH + 101))
expected_data = np.hstack((test_data1, test_data2[:101]))
np.testing.assert_array_equal(expected_data, actual_data)
self.assertTrue(my_pipe.closed)
# read_all raises an exception if the pipe has more than maxrows
# (less than one read)
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
with self.assertRaises(PipeError):
loop.run_until_complete(
my_pipe.read_all(maxrows=LENGTH + 101, error_on_overflow=True))
self.assertTrue(my_pipe.closed)
# (more than one read)
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
my_pipe.close_nowait()
with self.assertRaises(PipeError):
loop.run_until_complete(
my_pipe.read_all(maxrows=LENGTH + 101, error_on_overflow=True))
self.assertTrue(my_pipe.closed)
