def test_handles_interval_breaks(self): LAYOUT = "int32_3" LENGTH = 1000 my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) async def reader(): # read the first interval read_data = await my_pipe.read() self.assertTrue(my_pipe.end_of_interval) my_pipe.consume(len(read_data) - 20) np.testing.assert_array_equal(test_data1, read_data) # read the second interval read_data = await my_pipe.read() self.assertFalse(my_pipe.end_of_interval) self.assertEqual(len(read_data), len(test_data2) + 20) my_pipe.consume(len(read_data)) np.testing.assert_array_equal(test_data2, read_data[20:]) loop = asyncio.get_event_loop() loop.run_until_complete(reader())
def test_invalid_write_nowait_inputs(self): LAYOUT = "int8_2" loop = asyncio.get_event_loop() my_pipe = LocalPipe(LAYOUT, name="testpipe") with self.assertLogs(level="INFO"): my_pipe.write_nowait(np.array([[]])) with self.assertRaises(PipeError): my_pipe.write_nowait([1, 2, 3])
def test_nowait_read_does_not_block(self): LAYOUT = "int8_2" LENGTH = 50 my_pipe = LocalPipe(LAYOUT) test_data = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data) self.assertEqual(len(my_pipe.read_nowait()), LENGTH) # another read executes immediately because there is unconsumed data self.assertEqual(len(my_pipe.read_nowait()), LENGTH) # now consume the data and the next call to read execute immediately # even though there is no data to be returned my_pipe.consume(LENGTH) self.assertEqual(0, len(my_pipe.read_nowait(flatten=True))) return
def test_raises_consume_errors(self): LAYOUT = "int32_3" LENGTH = 1000 my_pipe = LocalPipe(LAYOUT) test_data = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data) read_data = my_pipe.read_nowait() # can't consume more than was read with self.assertRaises(PipeError) as e: my_pipe.consume(len(read_data) + 1) self.assertTrue('consume' in str(e.exception)) # can't consume less than zero with self.assertRaises(PipeError) as e: my_pipe.consume(-1) self.assertTrue('negative' in str(e.exception))
def test_read_data_must_be_consumed(self): #writes to pipe sends data to reader and any subscribers LAYOUT = "float32_2" LENGTH = 500 UNCONSUMED_ROWS = 4 my_pipe = LocalPipe(LAYOUT) chunk1 = helpers.create_data(LAYOUT, length=LENGTH) chunk2 = helpers.create_data(LAYOUT, length=LENGTH) chunk3 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(chunk1) async def reader(): await my_pipe.read() my_pipe.consume(0) # should get the same data back on the next read # add a second copy of the test data await my_pipe.write(chunk2) rx_data = await my_pipe.read() # two copies of the data now np.testing.assert_array_equal(chunk1, rx_data[:len(chunk1)]) np.testing.assert_array_equal(chunk2, rx_data[len(chunk1):]) # write another copy but consume the first my_pipe.consume(len(chunk1)) await my_pipe.write(chunk3) rx_data = await my_pipe.read() # two copies of the data now np.testing.assert_array_equal(chunk2, rx_data[:len(chunk2)]) np.testing.assert_array_equal(chunk3, rx_data[len(chunk2):]) my_pipe.consume(len(chunk2)) await my_pipe.close() # now a read should return immediately with the unconsumed data rx_data = await my_pipe.read() np.testing.assert_array_equal(chunk3, rx_data) # the pipe should be empty but still return the old data rx_data = await my_pipe.read() np.testing.assert_array_equal(chunk3, rx_data) # only after consuming the remaining data does it raise an exception my_pipe.consume(len(rx_data)) # another read should cause an exception (even if the data hasn't been consumed) with self.assertRaises(EmptyPipeError): await my_pipe.read() # the pipe should be empty self.assertTrue(my_pipe.is_empty()) asyncio.run(reader())
def subscribe(self, stream: DataStream, pipe: pipes.LocalPipe): if self.raise_error: raise SubscriptionError() self.subscribed_stream = stream while True: try: data = self.subscription_pipe.read_nowait() self.subscription_pipe.consume(len(data)) pipe.write_nowait(data) except pipes.EmptyPipe: if not self.hang_pipe: pipe.close_nowait() break if self.subscription_pipe.end_of_interval: pipe.close_interval_nowait() # return a mock as the unsubscribe callback return self.unsubscribe
def test_nowait_read_writes(self): LAYOUT = "int8_2" LENGTH = 500 loop = asyncio.get_event_loop() my_pipe = LocalPipe(LAYOUT) my_subscriber = LocalPipe(LAYOUT) my_pipe.subscribe(my_subscriber) test_data = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data) result = my_pipe.read_nowait() # read data should match written data np.testing.assert_array_almost_equal(test_data['data'], result['data']) np.testing.assert_array_almost_equal(test_data['timestamp'], result['timestamp']) # subscriber should have a copy too result = my_subscriber.read_nowait() np.testing.assert_array_almost_equal(test_data['data'], result['data']) np.testing.assert_array_almost_equal(test_data['timestamp'], result['timestamp'])
def test_nowait_function_exceptions(self): pipe = LocalPipe(layout="float32_3") subscriber = LocalPipe(layout="float32_3", close_cb=mock.Mock()) fd_subscriber = OutputPipe() # cannot close_nowait if there is a callback with self.assertRaises(PipeError): subscriber.close_nowait() # can write_nowait if only localpipe subscribers pipe.subscribe(subscriber) pipe.write_nowait(np.ones((100, 4))) # cannot if any subscribers are not localpipes pipe.subscribe(fd_subscriber) with self.assertRaises(PipeError): pipe.write_nowait(np.ones((100, 4))) # cannot close_nowait if there are subscribers pipe.subscribe(subscriber) with self.assertRaises(PipeError): pipe.close_nowait()
def test_read_all_no_flatten(self): LAYOUT = "int32_3" LENGTH = 1000 loop = asyncio.get_event_loop() # raises exception if the pipe is empty my_pipe = LocalPipe(LAYOUT, name="pipe") my_pipe.close_nowait() with self.assertRaises(PipeError): loop.run_until_complete(my_pipe.read_all()) # read_all empties pipe and closes it, regardless of intervals my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) my_pipe.close_nowait() actual_data = loop.run_until_complete(my_pipe.read_all()) expected_data = np.hstack((test_data1, test_data2)) np.testing.assert_array_equal(actual_data, expected_data) self.assertTrue(my_pipe.closed) # read_all only add maxrows to the pipe # (less than one read) my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) actual_data = loop.run_until_complete(my_pipe.read_all(maxrows=103)) expected_data = test_data1[:103] np.testing.assert_array_equal(actual_data, expected_data) self.assertTrue(my_pipe.closed) # (more than one read) my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) actual_data = loop.run_until_complete( my_pipe.read_all(maxrows=LENGTH + 101)) expected_data = np.hstack((test_data1, test_data2[:101])) np.testing.assert_array_equal(expected_data, actual_data) self.assertTrue(my_pipe.closed) # read_all raises an exception if the pipe has more than maxrows # (less than one read) my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) with self.assertRaises(PipeError): loop.run_until_complete( my_pipe.read_all(maxrows=LENGTH + 101, error_on_overflow=True)) self.assertTrue(my_pipe.closed) # (more than one read) my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) my_pipe.close_nowait() with self.assertRaises(PipeError): loop.run_until_complete( my_pipe.read_all(maxrows=LENGTH + 101, error_on_overflow=True)) self.assertTrue(my_pipe.closed)