async def data_read(session: BaseSession,
stream: Union[DataStream, str, int],
start: Optional[int] = None,
end: Optional[int] = None,
max_rows: Optional[int] = None) -> Pipe:
# make sure the input is compatible
src_stream = await data_stream_get(session, stream)
if type(stream) is DataStream:
if src_stream.layout != src_stream.layout:
raise errors.ApiError(
"Input [%s] configured for [%s] but source is [%s]" %
(stream, stream.layout, src_stream.layout))
# replace the stub stream (from config file) with actual stream
pipe = LocalPipe(src_stream.layout,
name=src_stream.name,
stream=src_stream,
write_limit=5)
pipe.stream = src_stream
task = asyncio.create_task(
_historic_reader(session, src_stream, pipe, start, end, max_rows))
async def close():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
pipe.close_cb = close
return pipe
async def data_subscribe(session: BaseSession, stream: Union[DataStream, str,
int]) -> Pipe:
# make sure the input is compatible
src_stream = await data_stream_get(session, stream)
if type(stream) is DataStream:
if src_stream.layout != src_stream.layout:
raise errors.ApiError(
"Input [%s] configured for [%s] but source is [%s]" %
(stream, stream.layout, src_stream.layout))
# make sure the stream is being produced
if not src_stream.is_destination:
raise errors.ApiError(
"DataStream [%s] is not being produced, specify time bounds for historic execution"
% src_stream.name)
# replace the stub stream (from config file) with actual stream
# do not let the buffer grow beyond 5 server chunks
pipe = LocalPipe(src_stream.layout,
name=src_stream.name,
stream=src_stream,
write_limit=5)
pipe.stream = src_stream
task = asyncio.create_task(_live_reader(session, src_stream, pipe))
async def close():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
pipe.close_cb = close
return pipe
def test_sends_data_to_subscribers(self):
LAYOUT = "float32_2"
(fd_r, fd_w) = os.pipe()
loop = asyncio.get_event_loop()
output_cb = CoroutineMock()
input_cb = CoroutineMock()
subscriber_cb = CoroutineMock()
npipe_out = OutputPipe(layout=LAYOUT, writer_factory=writer_factory(fd_w),
close_cb=output_cb)
subscriber = LocalPipe(layout=LAYOUT, close_cb=subscriber_cb)
npipe_out.subscribe(subscriber)
test_data = helpers.create_data(LAYOUT)
loop.run_until_complete(npipe_out.write(test_data))
# data should be available on the InputPipe side
npipe_in = InputPipe(layout=LAYOUT, reader_factory=reader_factory(fd_r),
close_cb=input_cb)
rx_data = loop.run_until_complete(npipe_in.read())
np.testing.assert_array_equal(test_data, rx_data)
# data should also be available on the Subscriber output
rx_data = subscriber.read_nowait()
np.testing.assert_array_equal(test_data, rx_data)
loop.run_until_complete(asyncio.gather(npipe_in.close(),
npipe_out.close()))
# subscriber should be closed
self.assertTrue(subscriber.closed)
# make sure all of the callbacks have been executed
self.assertEqual(output_cb.call_count, 1)
self.assertEqual(input_cb.call_count, 1)
self.assertEqual(subscriber_cb.call_count, 1)
class TestingPipe(Pipe):
# reads come out in the same blocks as the writes
# ...wraps LocalPipe
def __init__(self, layout: str, name: str = None, stream=None):
super().__init__(name=name, layout=layout, stream=stream)
self._pipe = LocalPipe(layout, name, stream)
self._closed = False
self.data_blocks: deque = deque()
async def write(self, data):
if self._closed:
raise PipeError("Cannot write to a closed pipe")
self.data_blocks.append(data)
def write_nowait(self, data):
if self._closed:
raise PipeError("Cannot write to a closed pipe")
self.data_blocks.append(data)
def close_interval_no_wait(self):
self.data_blocks.append(None)
async def close_interval(self):
self.data_blocks.append(None)
def consume(self, rows):
return self._pipe.consume(rows)
def reread_last(self):
self._pipe.reread_last()
@property
def end_of_interval(self):
return self._pipe.end_of_interval
async def read(self, flatten=False):
# first write a block to the pipe if there are any waiting
if len(self.data_blocks) > 0:
block = self.data_blocks.popleft()
if block is None:
await self._pipe.close_interval()
else:
await self._pipe.write(block)
elif self._closed:
await self._pipe.close()
# now return the result of the inner pipe's read
return await self._pipe.read(flatten)
def is_empty(self):
return self._pipe.is_empty()
async def close(self):
self._closed = True
def test_handles_interval_breaks(self):
LAYOUT = "int32_3"
LENGTH = 1000
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
async def reader():
# read the first interval
read_data = await my_pipe.read()
self.assertTrue(my_pipe.end_of_interval)
my_pipe.consume(len(read_data) - 20)
np.testing.assert_array_equal(test_data1, read_data)
# read the second interval
read_data = await my_pipe.read()
self.assertFalse(my_pipe.end_of_interval)
self.assertEqual(len(read_data), len(test_data2) + 20)
my_pipe.consume(len(read_data))
np.testing.assert_array_equal(test_data2, read_data[20:])
loop = asyncio.get_event_loop()
loop.run_until_complete(reader())
def test_read_data_must_be_consumed(self):
#writes to pipe sends data to reader and any subscribers
LAYOUT = "float32_2"
LENGTH = 500
UNCONSUMED_ROWS = 4
my_pipe = LocalPipe(LAYOUT)
chunk1 = helpers.create_data(LAYOUT, length=LENGTH)
chunk2 = helpers.create_data(LAYOUT, length=LENGTH)
chunk3 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(chunk1)
async def reader():
await my_pipe.read()
my_pipe.consume(0)
# should get the same data back on the next read
# add a second copy of the test data
await my_pipe.write(chunk2)
rx_data = await my_pipe.read()
# two copies of the data now
np.testing.assert_array_equal(chunk1, rx_data[:len(chunk1)])
np.testing.assert_array_equal(chunk2, rx_data[len(chunk1):])
# write another copy but consume the first
my_pipe.consume(len(chunk1))
await my_pipe.write(chunk3)
rx_data = await my_pipe.read()
# two copies of the data now
np.testing.assert_array_equal(chunk2, rx_data[:len(chunk2)])
np.testing.assert_array_equal(chunk3, rx_data[len(chunk2):])
my_pipe.consume(len(chunk2))
await my_pipe.close()
# now a read should return immediately with the unconsumed data
rx_data = await my_pipe.read()
np.testing.assert_array_equal(chunk3, rx_data)
# the pipe should be empty but still return the old data
rx_data = await my_pipe.read()
np.testing.assert_array_equal(chunk3, rx_data)
# only after consuming the remaining data does it raise an exception
my_pipe.consume(len(rx_data))
# another read should cause an exception (even if the data hasn't been consumed)
with self.assertRaises(EmptyPipeError):
await my_pipe.read()
# the pipe should be empty
self.assertTrue(my_pipe.is_empty())
asyncio.run(reader())
def test_nowait_read_does_not_block(self):
LAYOUT = "int8_2"
LENGTH = 50
my_pipe = LocalPipe(LAYOUT)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data)
self.assertEqual(len(my_pipe.read_nowait()), LENGTH)
# another read executes immediately because there is unconsumed data
self.assertEqual(len(my_pipe.read_nowait()), LENGTH)
# now consume the data and the next call to read execute immediately
# even though there is no data to be returned
my_pipe.consume(LENGTH)
self.assertEqual(0, len(my_pipe.read_nowait(flatten=True)))
return
def test_raises_consume_errors(self):
LAYOUT = "int32_3"
LENGTH = 1000
my_pipe = LocalPipe(LAYOUT)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data)
read_data = my_pipe.read_nowait()
# can't consume more than was read
with self.assertRaises(PipeError) as e:
my_pipe.consume(len(read_data) + 1)
self.assertTrue('consume' in str(e.exception))
# can't consume less than zero
with self.assertRaises(PipeError) as e:
my_pipe.consume(-1)
self.assertTrue('negative' in str(e.exception))
def test_nowait_read_writes(self):
LAYOUT = "int8_2"
LENGTH = 500
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT)
my_subscriber = LocalPipe(LAYOUT)
my_pipe.subscribe(my_subscriber)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data)
result = my_pipe.read_nowait()
# read data should match written data
np.testing.assert_array_almost_equal(test_data['data'], result['data'])
np.testing.assert_array_almost_equal(test_data['timestamp'],
result['timestamp'])
# subscriber should have a copy too
result = my_subscriber.read_nowait()
np.testing.assert_array_almost_equal(test_data['data'], result['data'])
np.testing.assert_array_almost_equal(test_data['timestamp'],
result['timestamp'])
def test_nowait_read_empties_queue(self):
LAYOUT = "int8_2"
LENGTH = 50
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
async def writer():
for row in test_data:
await my_pipe.write(np.array([row]))
loop = asyncio.get_event_loop()
loop.run_until_complete(writer())
result = my_pipe.read_nowait()
np.testing.assert_array_almost_equal(test_data['data'], result['data'])
np.testing.assert_array_almost_equal(test_data['timestamp'],
result['timestamp'])
async def _run():
LAYOUT = "float32_3"
LENGTH = 100
pipe = LocalPipe(layout="float32_3")
tx_data = helpers.create_data(LAYOUT, length=LENGTH)
# write a block and close off the pipe
await pipe.write(tx_data)
await pipe.close()
# read it back
rx_data = await pipe.read()
np.testing.assert_array_equal(tx_data, rx_data)
# don't consume it so you can read it agai
pipe.consume(len(rx_data[:500]))
# read it back again
rx_data = await pipe.read()
np.testing.assert_array_equal(tx_data[500:], rx_data)
# the pipe should be empty now
self.assertTrue(pipe.is_empty())
with self.assertRaises(EmptyPipeError):
await pipe.read()
def test_caching(self):
LAYOUT = "float32_2"
LENGTH = 128
CACHE_SIZE = 40
my_pipe = LocalPipe(LAYOUT)
my_pipe.enable_cache(CACHE_SIZE)
test_data = helpers.create_data(LAYOUT, length=LENGTH, start=1, step=1)
async def writer():
for block in helpers.to_chunks(test_data, 4):
await asyncio.sleep(.1)
await my_pipe.write(block)
# closing the interval should flush the data
await my_pipe.close_interval()
# add a dummy section after the interval break
await my_pipe.write(np.ones((35, 3)))
await my_pipe.flush_cache()
num_reads = 0
async def reader():
nonlocal num_reads
index = 0
while True:
data = await my_pipe.read()
my_pipe.consume(len(data))
# make sure the data is correct
np.testing.assert_array_equal(
test_data[index:index + len(data)], data)
num_reads += 1
index += len(data)
if index == len(test_data):
break
# now get the dummy section after the interval break
data = await my_pipe.read(flatten=True)
np.testing.assert_array_equal(data, np.ones((35, 3)))
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(reader(), writer()))
self.assertLessEqual(num_reads, np.ceil(LENGTH / CACHE_SIZE))
def test_different_format_writes(self):
LAYOUT = "int8_2"
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT, name="testpipe")
test_data = helpers.create_data(LAYOUT, length=4, step=1, start=106)
async def write():
# write unstructured numpy arrays
await my_pipe.write(np.array([[1, 1, 1]]))
await my_pipe.write(np.array([[2, 2, 2], [3, 3, 3]]))
# logs empty writes
with self.assertLogs(level="INFO") as logs:
await my_pipe.write(np.array([[]]))
# errors on invalid write types
bad_data = [[100, 1, 2], 'invalid', 4, None, np.array([4, 8])]
for data in bad_data:
with self.assertRaises(PipeError):
await my_pipe.write(data)
# write structured numpy arrays
await my_pipe.write(test_data)
loop = asyncio.get_event_loop()
loop.run_until_complete(write())
result = my_pipe.read_nowait(flatten=True)
np.testing.assert_array_equal(result[:3],
[[1, 1, 1], [2, 2, 2], [3, 3, 3]])
my_pipe.consume(3)
result = my_pipe.read_nowait()
np.testing.assert_array_equal(result, test_data)
def test_handles_flat_and_structured_arrays(self):
# converts flat arrays to structured arrays and returns
# either flat or structured arrays depending on [flatten] parameter
LAYOUT = "float64_1"
LENGTH = 1000
my_pipe = LocalPipe(LAYOUT)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
flat_data = np.c_[test_data['timestamp'][:, None], test_data['data']]
async def reader():
sdata = await my_pipe.read()
fdata = await my_pipe.read(flatten=True)
np.testing.assert_array_almost_equal(sdata['timestamp'],
test_data['timestamp'])
np.testing.assert_array_almost_equal(sdata['data'],
test_data['data'])
np.testing.assert_array_almost_equal(fdata, flat_data)
np.testing.assert_array_almost_equal(fdata, flat_data)
asyncio.run(my_pipe.write(flat_data))
asyncio.run(my_pipe.close())
asyncio.run(reader())
def test_invalid_write_nowait_inputs(self):
LAYOUT = "int8_2"
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT, name="testpipe")
with self.assertLogs(level="INFO"):
my_pipe.write_nowait(np.array([[]]))
with self.assertRaises(PipeError):
my_pipe.write_nowait([1, 2, 3])
async def data_write(
session: BaseSession,
stream: Union[DataStream, str, int],
start_time: Optional[int] = None,
end_time: Optional[int] = None,
) -> Pipe:
# stream must exist, does not automatically create a stream
# retrieve the destination stream object
dest_stream = await data_stream_get(session, stream)
if start_time is not None or end_time is not None:
await data_delete(session, dest_stream, start_time, end_time)
if type(stream) is DataStream:
if dest_stream.layout != stream.layout:
raise errors.ApiError(
"DataStream [%s] configured for [%s] but destination is [%s]" %
(stream.name, stream.layout, dest_stream.layout))
# raise a warning if the element names do not match
actual_names = [e.name for e in dest_stream.elements]
requested_names = [e.name for e in stream.elements]
if actual_names != requested_names: # pragma: no cover
log.warning("[%s] elements do not match the existing stream" %
stream.name)
# make sure the stream is not currently produced
if dest_stream.is_destination:
raise errors.ApiError("DataStream [%s] is already being produced" %
dest_stream.name)
# all checks passed, subscribe to the output
async def close():
await task
pipe = LocalPipe(dest_stream.layout,
name=dest_stream.name,
stream=dest_stream,
close_cb=close,
debug=False,
write_limit=5)
task = asyncio.create_task(_send_data(session, dest_stream, pipe))
return pipe
def subscribe(self, stream: DataStream, pipe: pipes.LocalPipe):
if self.raise_error:
raise SubscriptionError()
self.subscribed_stream = stream
while True:
try:
data = self.subscription_pipe.read_nowait()
self.subscription_pipe.consume(len(data))
pipe.write_nowait(data)
except pipes.EmptyPipe:
if not self.hang_pipe:
pipe.close_nowait()
break
if self.subscription_pipe.end_of_interval:
pipe.close_interval_nowait()
# return a mock as the unsubscribe callback
return self.unsubscribe
def test_nowait_function_exceptions(self):
pipe = LocalPipe(layout="float32_3")
subscriber = LocalPipe(layout="float32_3", close_cb=mock.Mock())
fd_subscriber = OutputPipe()
# cannot close_nowait if there is a callback
with self.assertRaises(PipeError):
subscriber.close_nowait()
# can write_nowait if only localpipe subscribers
pipe.subscribe(subscriber)
pipe.write_nowait(np.ones((100, 4)))
# cannot if any subscribers are not localpipes
pipe.subscribe(fd_subscriber)
with self.assertRaises(PipeError):
pipe.write_nowait(np.ones((100, 4)))
# cannot close_nowait if there are subscribers
pipe.subscribe(subscriber)
with self.assertRaises(PipeError):
pipe.close_nowait()
def test_read_all_no_flatten(self):
LAYOUT = "int32_3"
LENGTH = 1000
loop = asyncio.get_event_loop()
# raises exception if the pipe is empty
my_pipe = LocalPipe(LAYOUT, name="pipe")
my_pipe.close_nowait()
with self.assertRaises(PipeError):
loop.run_until_complete(my_pipe.read_all())
# read_all empties pipe and closes it, regardless of intervals
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
my_pipe.close_nowait()
actual_data = loop.run_until_complete(my_pipe.read_all())
expected_data = np.hstack((test_data1, test_data2))
np.testing.assert_array_equal(actual_data, expected_data)
self.assertTrue(my_pipe.closed)
# read_all only add maxrows to the pipe
# (less than one read)
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
actual_data = loop.run_until_complete(my_pipe.read_all(maxrows=103))
expected_data = test_data1[:103]
np.testing.assert_array_equal(actual_data, expected_data)
self.assertTrue(my_pipe.closed)
# (more than one read)
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
actual_data = loop.run_until_complete(
my_pipe.read_all(maxrows=LENGTH + 101))
expected_data = np.hstack((test_data1, test_data2[:101]))
np.testing.assert_array_equal(expected_data, actual_data)
self.assertTrue(my_pipe.closed)
# read_all raises an exception if the pipe has more than maxrows
# (less than one read)
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
with self.assertRaises(PipeError):
loop.run_until_complete(
my_pipe.read_all(maxrows=LENGTH + 101, error_on_overflow=True))
self.assertTrue(my_pipe.closed)
# (more than one read)
my_pipe = LocalPipe(LAYOUT, name="pipe")
test_data1 = helpers.create_data(LAYOUT, length=LENGTH)
test_data2 = helpers.create_data(LAYOUT, length=LENGTH)
my_pipe.write_nowait(test_data1)
my_pipe.close_interval_nowait()
my_pipe.write_nowait(test_data2)
my_pipe.close_nowait()
with self.assertRaises(PipeError):
loop.run_until_complete(
my_pipe.read_all(maxrows=LENGTH + 101, error_on_overflow=True))
self.assertTrue(my_pipe.closed)
def __init__(self, layout: str, name: str = None, stream=None):
super().__init__(name=name, layout=layout, stream=stream)
self._pipe = LocalPipe(layout, name, stream)
self._closed = False
self.data_blocks: deque = deque()
def test_pipes_numpy_arrays(self):
"""writes to pipe sends data to reader and any subscribers"""
LAYOUT = "int8_2"
LENGTH = 1003
loop = asyncio.get_event_loop()
my_pipe = LocalPipe(LAYOUT, name="pipe")
# test timeouts, since the writer is slow
my_pipe.TIMEOUT_INTERVAL = 0.05
subscriber_pipe = LocalPipe(LAYOUT)
subscriber_pipe2 = LocalPipe(LAYOUT)
my_pipe.subscribe(subscriber_pipe)
my_pipe.subscribe(subscriber_pipe2)
test_data = helpers.create_data(LAYOUT, length=LENGTH)
reader_rx_data = np.empty(test_data.shape, test_data.dtype)
subscriber_rx_data = np.empty(test_data.shape, test_data.dtype)
# print(test_data['data'][:,1])
async def writer():
for block in helpers.to_chunks(test_data, 270):
await asyncio.sleep(0.1)
await my_pipe.write(block)
await my_pipe.close()
async def reader():
# note this is kind of delicate, you can only read data twice
# if the pipe is closed so it might look like you didn't finish all
# the data if the last read ignores data past blk_size
blk_size = 357
rx_idx = 0
while not my_pipe.is_empty():
# consume data in pipe
data_chunk = await my_pipe.read()
rows_used = min(len(data_chunk), blk_size)
reader_rx_data[rx_idx:rx_idx +
rows_used] = data_chunk[:rows_used]
rx_idx += rows_used
my_pipe.consume(rows_used)
async def subscriber():
rx_idx = 0
while not subscriber_pipe.is_empty():
# consume data in pipe
data_chunk = await subscriber_pipe.read()
subscriber_rx_data[rx_idx:rx_idx +
len(data_chunk)] = data_chunk
rx_idx += len(data_chunk)
subscriber_pipe.consume(len(data_chunk))
loop = asyncio.get_event_loop()
tasks = [
asyncio.ensure_future(writer()),
asyncio.ensure_future(reader()),
asyncio.ensure_future(subscriber())
]
loop.run_until_complete(asyncio.gather(*tasks))
data = subscriber_pipe2.read_nowait()
np.testing.assert_array_equal(test_data, data)
np.testing.assert_array_equal(test_data, reader_rx_data)
np.testing.assert_array_equal(test_data, subscriber_rx_data)