async def data_read(session: BaseSession, stream: Union[DataStream, str, int], start: Optional[int] = None, end: Optional[int] = None, max_rows: Optional[int] = None) -> Pipe: # make sure the input is compatible src_stream = await data_stream_get(session, stream) if type(stream) is DataStream: if src_stream.layout != src_stream.layout: raise errors.ApiError( "Input [%s] configured for [%s] but source is [%s]" % (stream, stream.layout, src_stream.layout)) # replace the stub stream (from config file) with actual stream pipe = LocalPipe(src_stream.layout, name=src_stream.name, stream=src_stream, write_limit=5) pipe.stream = src_stream task = asyncio.create_task( _historic_reader(session, src_stream, pipe, start, end, max_rows)) async def close(): task.cancel() try: await task except asyncio.CancelledError: pass pipe.close_cb = close return pipe
async def data_subscribe(session: BaseSession, stream: Union[DataStream, str, int]) -> Pipe: # make sure the input is compatible src_stream = await data_stream_get(session, stream) if type(stream) is DataStream: if src_stream.layout != src_stream.layout: raise errors.ApiError( "Input [%s] configured for [%s] but source is [%s]" % (stream, stream.layout, src_stream.layout)) # make sure the stream is being produced if not src_stream.is_destination: raise errors.ApiError( "DataStream [%s] is not being produced, specify time bounds for historic execution" % src_stream.name) # replace the stub stream (from config file) with actual stream # do not let the buffer grow beyond 5 server chunks pipe = LocalPipe(src_stream.layout, name=src_stream.name, stream=src_stream, write_limit=5) pipe.stream = src_stream task = asyncio.create_task(_live_reader(session, src_stream, pipe)) async def close(): task.cancel() try: await task except asyncio.CancelledError: pass pipe.close_cb = close return pipe
def test_sends_data_to_subscribers(self): LAYOUT = "float32_2" (fd_r, fd_w) = os.pipe() loop = asyncio.get_event_loop() output_cb = CoroutineMock() input_cb = CoroutineMock() subscriber_cb = CoroutineMock() npipe_out = OutputPipe(layout=LAYOUT, writer_factory=writer_factory(fd_w), close_cb=output_cb) subscriber = LocalPipe(layout=LAYOUT, close_cb=subscriber_cb) npipe_out.subscribe(subscriber) test_data = helpers.create_data(LAYOUT) loop.run_until_complete(npipe_out.write(test_data)) # data should be available on the InputPipe side npipe_in = InputPipe(layout=LAYOUT, reader_factory=reader_factory(fd_r), close_cb=input_cb) rx_data = loop.run_until_complete(npipe_in.read()) np.testing.assert_array_equal(test_data, rx_data) # data should also be available on the Subscriber output rx_data = subscriber.read_nowait() np.testing.assert_array_equal(test_data, rx_data) loop.run_until_complete(asyncio.gather(npipe_in.close(), npipe_out.close())) # subscriber should be closed self.assertTrue(subscriber.closed) # make sure all of the callbacks have been executed self.assertEqual(output_cb.call_count, 1) self.assertEqual(input_cb.call_count, 1) self.assertEqual(subscriber_cb.call_count, 1)
class TestingPipe(Pipe): # reads come out in the same blocks as the writes # ...wraps LocalPipe def __init__(self, layout: str, name: str = None, stream=None): super().__init__(name=name, layout=layout, stream=stream) self._pipe = LocalPipe(layout, name, stream) self._closed = False self.data_blocks: deque = deque() async def write(self, data): if self._closed: raise PipeError("Cannot write to a closed pipe") self.data_blocks.append(data) def write_nowait(self, data): if self._closed: raise PipeError("Cannot write to a closed pipe") self.data_blocks.append(data) def close_interval_no_wait(self): self.data_blocks.append(None) async def close_interval(self): self.data_blocks.append(None) def consume(self, rows): return self._pipe.consume(rows) def reread_last(self): self._pipe.reread_last() @property def end_of_interval(self): return self._pipe.end_of_interval async def read(self, flatten=False): # first write a block to the pipe if there are any waiting if len(self.data_blocks) > 0: block = self.data_blocks.popleft() if block is None: await self._pipe.close_interval() else: await self._pipe.write(block) elif self._closed: await self._pipe.close() # now return the result of the inner pipe's read return await self._pipe.read(flatten) def is_empty(self): return self._pipe.is_empty() async def close(self): self._closed = True
def test_handles_interval_breaks(self): LAYOUT = "int32_3" LENGTH = 1000 my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) async def reader(): # read the first interval read_data = await my_pipe.read() self.assertTrue(my_pipe.end_of_interval) my_pipe.consume(len(read_data) - 20) np.testing.assert_array_equal(test_data1, read_data) # read the second interval read_data = await my_pipe.read() self.assertFalse(my_pipe.end_of_interval) self.assertEqual(len(read_data), len(test_data2) + 20) my_pipe.consume(len(read_data)) np.testing.assert_array_equal(test_data2, read_data[20:]) loop = asyncio.get_event_loop() loop.run_until_complete(reader())
def test_read_data_must_be_consumed(self): #writes to pipe sends data to reader and any subscribers LAYOUT = "float32_2" LENGTH = 500 UNCONSUMED_ROWS = 4 my_pipe = LocalPipe(LAYOUT) chunk1 = helpers.create_data(LAYOUT, length=LENGTH) chunk2 = helpers.create_data(LAYOUT, length=LENGTH) chunk3 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(chunk1) async def reader(): await my_pipe.read() my_pipe.consume(0) # should get the same data back on the next read # add a second copy of the test data await my_pipe.write(chunk2) rx_data = await my_pipe.read() # two copies of the data now np.testing.assert_array_equal(chunk1, rx_data[:len(chunk1)]) np.testing.assert_array_equal(chunk2, rx_data[len(chunk1):]) # write another copy but consume the first my_pipe.consume(len(chunk1)) await my_pipe.write(chunk3) rx_data = await my_pipe.read() # two copies of the data now np.testing.assert_array_equal(chunk2, rx_data[:len(chunk2)]) np.testing.assert_array_equal(chunk3, rx_data[len(chunk2):]) my_pipe.consume(len(chunk2)) await my_pipe.close() # now a read should return immediately with the unconsumed data rx_data = await my_pipe.read() np.testing.assert_array_equal(chunk3, rx_data) # the pipe should be empty but still return the old data rx_data = await my_pipe.read() np.testing.assert_array_equal(chunk3, rx_data) # only after consuming the remaining data does it raise an exception my_pipe.consume(len(rx_data)) # another read should cause an exception (even if the data hasn't been consumed) with self.assertRaises(EmptyPipeError): await my_pipe.read() # the pipe should be empty self.assertTrue(my_pipe.is_empty()) asyncio.run(reader())
def test_nowait_read_does_not_block(self): LAYOUT = "int8_2" LENGTH = 50 my_pipe = LocalPipe(LAYOUT) test_data = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data) self.assertEqual(len(my_pipe.read_nowait()), LENGTH) # another read executes immediately because there is unconsumed data self.assertEqual(len(my_pipe.read_nowait()), LENGTH) # now consume the data and the next call to read execute immediately # even though there is no data to be returned my_pipe.consume(LENGTH) self.assertEqual(0, len(my_pipe.read_nowait(flatten=True))) return
def test_raises_consume_errors(self): LAYOUT = "int32_3" LENGTH = 1000 my_pipe = LocalPipe(LAYOUT) test_data = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data) read_data = my_pipe.read_nowait() # can't consume more than was read with self.assertRaises(PipeError) as e: my_pipe.consume(len(read_data) + 1) self.assertTrue('consume' in str(e.exception)) # can't consume less than zero with self.assertRaises(PipeError) as e: my_pipe.consume(-1) self.assertTrue('negative' in str(e.exception))
def test_nowait_read_writes(self): LAYOUT = "int8_2" LENGTH = 500 loop = asyncio.get_event_loop() my_pipe = LocalPipe(LAYOUT) my_subscriber = LocalPipe(LAYOUT) my_pipe.subscribe(my_subscriber) test_data = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data) result = my_pipe.read_nowait() # read data should match written data np.testing.assert_array_almost_equal(test_data['data'], result['data']) np.testing.assert_array_almost_equal(test_data['timestamp'], result['timestamp']) # subscriber should have a copy too result = my_subscriber.read_nowait() np.testing.assert_array_almost_equal(test_data['data'], result['data']) np.testing.assert_array_almost_equal(test_data['timestamp'], result['timestamp'])
def test_nowait_read_empties_queue(self): LAYOUT = "int8_2" LENGTH = 50 loop = asyncio.get_event_loop() my_pipe = LocalPipe(LAYOUT) test_data = helpers.create_data(LAYOUT, length=LENGTH) async def writer(): for row in test_data: await my_pipe.write(np.array([row])) loop = asyncio.get_event_loop() loop.run_until_complete(writer()) result = my_pipe.read_nowait() np.testing.assert_array_almost_equal(test_data['data'], result['data']) np.testing.assert_array_almost_equal(test_data['timestamp'], result['timestamp'])
async def _run(): LAYOUT = "float32_3" LENGTH = 100 pipe = LocalPipe(layout="float32_3") tx_data = helpers.create_data(LAYOUT, length=LENGTH) # write a block and close off the pipe await pipe.write(tx_data) await pipe.close() # read it back rx_data = await pipe.read() np.testing.assert_array_equal(tx_data, rx_data) # don't consume it so you can read it agai pipe.consume(len(rx_data[:500])) # read it back again rx_data = await pipe.read() np.testing.assert_array_equal(tx_data[500:], rx_data) # the pipe should be empty now self.assertTrue(pipe.is_empty()) with self.assertRaises(EmptyPipeError): await pipe.read()
def test_caching(self): LAYOUT = "float32_2" LENGTH = 128 CACHE_SIZE = 40 my_pipe = LocalPipe(LAYOUT) my_pipe.enable_cache(CACHE_SIZE) test_data = helpers.create_data(LAYOUT, length=LENGTH, start=1, step=1) async def writer(): for block in helpers.to_chunks(test_data, 4): await asyncio.sleep(.1) await my_pipe.write(block) # closing the interval should flush the data await my_pipe.close_interval() # add a dummy section after the interval break await my_pipe.write(np.ones((35, 3))) await my_pipe.flush_cache() num_reads = 0 async def reader(): nonlocal num_reads index = 0 while True: data = await my_pipe.read() my_pipe.consume(len(data)) # make sure the data is correct np.testing.assert_array_equal( test_data[index:index + len(data)], data) num_reads += 1 index += len(data) if index == len(test_data): break # now get the dummy section after the interval break data = await my_pipe.read(flatten=True) np.testing.assert_array_equal(data, np.ones((35, 3))) loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.gather(reader(), writer())) self.assertLessEqual(num_reads, np.ceil(LENGTH / CACHE_SIZE))
def test_different_format_writes(self): LAYOUT = "int8_2" loop = asyncio.get_event_loop() my_pipe = LocalPipe(LAYOUT, name="testpipe") test_data = helpers.create_data(LAYOUT, length=4, step=1, start=106) async def write(): # write unstructured numpy arrays await my_pipe.write(np.array([[1, 1, 1]])) await my_pipe.write(np.array([[2, 2, 2], [3, 3, 3]])) # logs empty writes with self.assertLogs(level="INFO") as logs: await my_pipe.write(np.array([[]])) # errors on invalid write types bad_data = [[100, 1, 2], 'invalid', 4, None, np.array([4, 8])] for data in bad_data: with self.assertRaises(PipeError): await my_pipe.write(data) # write structured numpy arrays await my_pipe.write(test_data) loop = asyncio.get_event_loop() loop.run_until_complete(write()) result = my_pipe.read_nowait(flatten=True) np.testing.assert_array_equal(result[:3], [[1, 1, 1], [2, 2, 2], [3, 3, 3]]) my_pipe.consume(3) result = my_pipe.read_nowait() np.testing.assert_array_equal(result, test_data)
def test_handles_flat_and_structured_arrays(self): # converts flat arrays to structured arrays and returns # either flat or structured arrays depending on [flatten] parameter LAYOUT = "float64_1" LENGTH = 1000 my_pipe = LocalPipe(LAYOUT) test_data = helpers.create_data(LAYOUT, length=LENGTH) flat_data = np.c_[test_data['timestamp'][:, None], test_data['data']] async def reader(): sdata = await my_pipe.read() fdata = await my_pipe.read(flatten=True) np.testing.assert_array_almost_equal(sdata['timestamp'], test_data['timestamp']) np.testing.assert_array_almost_equal(sdata['data'], test_data['data']) np.testing.assert_array_almost_equal(fdata, flat_data) np.testing.assert_array_almost_equal(fdata, flat_data) asyncio.run(my_pipe.write(flat_data)) asyncio.run(my_pipe.close()) asyncio.run(reader())
def test_invalid_write_nowait_inputs(self): LAYOUT = "int8_2" loop = asyncio.get_event_loop() my_pipe = LocalPipe(LAYOUT, name="testpipe") with self.assertLogs(level="INFO"): my_pipe.write_nowait(np.array([[]])) with self.assertRaises(PipeError): my_pipe.write_nowait([1, 2, 3])
async def data_write( session: BaseSession, stream: Union[DataStream, str, int], start_time: Optional[int] = None, end_time: Optional[int] = None, ) -> Pipe: # stream must exist, does not automatically create a stream # retrieve the destination stream object dest_stream = await data_stream_get(session, stream) if start_time is not None or end_time is not None: await data_delete(session, dest_stream, start_time, end_time) if type(stream) is DataStream: if dest_stream.layout != stream.layout: raise errors.ApiError( "DataStream [%s] configured for [%s] but destination is [%s]" % (stream.name, stream.layout, dest_stream.layout)) # raise a warning if the element names do not match actual_names = [e.name for e in dest_stream.elements] requested_names = [e.name for e in stream.elements] if actual_names != requested_names: # pragma: no cover log.warning("[%s] elements do not match the existing stream" % stream.name) # make sure the stream is not currently produced if dest_stream.is_destination: raise errors.ApiError("DataStream [%s] is already being produced" % dest_stream.name) # all checks passed, subscribe to the output async def close(): await task pipe = LocalPipe(dest_stream.layout, name=dest_stream.name, stream=dest_stream, close_cb=close, debug=False, write_limit=5) task = asyncio.create_task(_send_data(session, dest_stream, pipe)) return pipe
def subscribe(self, stream: DataStream, pipe: pipes.LocalPipe): if self.raise_error: raise SubscriptionError() self.subscribed_stream = stream while True: try: data = self.subscription_pipe.read_nowait() self.subscription_pipe.consume(len(data)) pipe.write_nowait(data) except pipes.EmptyPipe: if not self.hang_pipe: pipe.close_nowait() break if self.subscription_pipe.end_of_interval: pipe.close_interval_nowait() # return a mock as the unsubscribe callback return self.unsubscribe
def test_nowait_function_exceptions(self): pipe = LocalPipe(layout="float32_3") subscriber = LocalPipe(layout="float32_3", close_cb=mock.Mock()) fd_subscriber = OutputPipe() # cannot close_nowait if there is a callback with self.assertRaises(PipeError): subscriber.close_nowait() # can write_nowait if only localpipe subscribers pipe.subscribe(subscriber) pipe.write_nowait(np.ones((100, 4))) # cannot if any subscribers are not localpipes pipe.subscribe(fd_subscriber) with self.assertRaises(PipeError): pipe.write_nowait(np.ones((100, 4))) # cannot close_nowait if there are subscribers pipe.subscribe(subscriber) with self.assertRaises(PipeError): pipe.close_nowait()
def test_read_all_no_flatten(self): LAYOUT = "int32_3" LENGTH = 1000 loop = asyncio.get_event_loop() # raises exception if the pipe is empty my_pipe = LocalPipe(LAYOUT, name="pipe") my_pipe.close_nowait() with self.assertRaises(PipeError): loop.run_until_complete(my_pipe.read_all()) # read_all empties pipe and closes it, regardless of intervals my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) my_pipe.close_nowait() actual_data = loop.run_until_complete(my_pipe.read_all()) expected_data = np.hstack((test_data1, test_data2)) np.testing.assert_array_equal(actual_data, expected_data) self.assertTrue(my_pipe.closed) # read_all only add maxrows to the pipe # (less than one read) my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) actual_data = loop.run_until_complete(my_pipe.read_all(maxrows=103)) expected_data = test_data1[:103] np.testing.assert_array_equal(actual_data, expected_data) self.assertTrue(my_pipe.closed) # (more than one read) my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) actual_data = loop.run_until_complete( my_pipe.read_all(maxrows=LENGTH + 101)) expected_data = np.hstack((test_data1, test_data2[:101])) np.testing.assert_array_equal(expected_data, actual_data) self.assertTrue(my_pipe.closed) # read_all raises an exception if the pipe has more than maxrows # (less than one read) my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) with self.assertRaises(PipeError): loop.run_until_complete( my_pipe.read_all(maxrows=LENGTH + 101, error_on_overflow=True)) self.assertTrue(my_pipe.closed) # (more than one read) my_pipe = LocalPipe(LAYOUT, name="pipe") test_data1 = helpers.create_data(LAYOUT, length=LENGTH) test_data2 = helpers.create_data(LAYOUT, length=LENGTH) my_pipe.write_nowait(test_data1) my_pipe.close_interval_nowait() my_pipe.write_nowait(test_data2) my_pipe.close_nowait() with self.assertRaises(PipeError): loop.run_until_complete( my_pipe.read_all(maxrows=LENGTH + 101, error_on_overflow=True)) self.assertTrue(my_pipe.closed)
def __init__(self, layout: str, name: str = None, stream=None): super().__init__(name=name, layout=layout, stream=stream) self._pipe = LocalPipe(layout, name, stream) self._closed = False self.data_blocks: deque = deque()
def test_pipes_numpy_arrays(self): """writes to pipe sends data to reader and any subscribers""" LAYOUT = "int8_2" LENGTH = 1003 loop = asyncio.get_event_loop() my_pipe = LocalPipe(LAYOUT, name="pipe") # test timeouts, since the writer is slow my_pipe.TIMEOUT_INTERVAL = 0.05 subscriber_pipe = LocalPipe(LAYOUT) subscriber_pipe2 = LocalPipe(LAYOUT) my_pipe.subscribe(subscriber_pipe) my_pipe.subscribe(subscriber_pipe2) test_data = helpers.create_data(LAYOUT, length=LENGTH) reader_rx_data = np.empty(test_data.shape, test_data.dtype) subscriber_rx_data = np.empty(test_data.shape, test_data.dtype) # print(test_data['data'][:,1]) async def writer(): for block in helpers.to_chunks(test_data, 270): await asyncio.sleep(0.1) await my_pipe.write(block) await my_pipe.close() async def reader(): # note this is kind of delicate, you can only read data twice # if the pipe is closed so it might look like you didn't finish all # the data if the last read ignores data past blk_size blk_size = 357 rx_idx = 0 while not my_pipe.is_empty(): # consume data in pipe data_chunk = await my_pipe.read() rows_used = min(len(data_chunk), blk_size) reader_rx_data[rx_idx:rx_idx + rows_used] = data_chunk[:rows_used] rx_idx += rows_used my_pipe.consume(rows_used) async def subscriber(): rx_idx = 0 while not subscriber_pipe.is_empty(): # consume data in pipe data_chunk = await subscriber_pipe.read() subscriber_rx_data[rx_idx:rx_idx + len(data_chunk)] = data_chunk rx_idx += len(data_chunk) subscriber_pipe.consume(len(data_chunk)) loop = asyncio.get_event_loop() tasks = [ asyncio.ensure_future(writer()), asyncio.ensure_future(reader()), asyncio.ensure_future(subscriber()) ] loop.run_until_complete(asyncio.gather(*tasks)) data = subscriber_pipe2.read_nowait() np.testing.assert_array_equal(test_data, data) np.testing.assert_array_equal(test_data, reader_rx_data) np.testing.assert_array_equal(test_data, subscriber_rx_data)