async def _run():
# Primary: y=10x-212
ts = np.arange(0, 500, 1)
primary_data = np.array([ts, 10 * ts - 212]).T
primary = helpers.TestingPipe("float32_1", name="primary")
await primary.write(primary_data[:100])
await primary.write(primary_data[100:200])
await primary.write(primary_data[200:])
await primary.close()
# Secondary1: y=-3.3x+436
ts = np.arange(510, 1000, 1)
secondary1_data = np.array([ts, -5 * ts + 436]).T
secondary1 = helpers.TestingPipe("float32_1", name="secondary1")
await secondary1.write(secondary1_data[:150])
await secondary1.write(secondary1_data[150:350])
await secondary1.write(secondary1_data[350:])
await secondary1.close()
args = argparse.Namespace(primary="primary", pipes="unset")
output = LocalPipe("float32_2", name="output")
# run filter in an event loop
my_filter = MergeFilter()
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1
},
outputs={'output': output})
self.assertTrue(output.is_empty())
with self.assertRaises(EmptyPipeError):
await output.read()
async def _run():
ts = np.arange(0, 1000)
values = np.random.randn(1000, 1)
data = np.hstack((ts[:, None], values))
primary = helpers.TestingPipe("float32_1", name="primary")
secondary1 = helpers.TestingPipe("float32_1", name="secondary1")
secondary2 = helpers.TestingPipe("float32_1", name="secondary2")
# seed the input data
for block in np.split(data, [200, 354, 700, 800, 930]):
primary.write_nowait(block)
for block in np.split(data, [155, 600, 652, 900]):
secondary1.write_nowait(block)
for block in np.split(data, [100, 300, 600]):
secondary2.write_nowait(block)
await primary.close()
await secondary1.close()
await secondary2.close()
# run filter in an event loop
my_filter = MergeFilter()
args = argparse.Namespace(primary="primary", pipes="unset")
output = LocalPipe("float32_3", name="output")
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1,
'secondary2': secondary2
},
outputs={'output': output})
# put together the data_blocks (should not be any interval breaks)
# remove the interval close at the end
result = await output.read()
output.consume(len(result))
self.assertTrue(output.is_empty())
# all elements should match the data
np.testing.assert_array_almost_equal(result['data'][:, 0][:, None],
values)
np.testing.assert_array_almost_equal(result['data'][:, 1][:, None],
values)
np.testing.assert_array_almost_equal(result['data'][:, 2][:, None],
values)
if VISUALIZE:
from matplotlib import pyplot as plt
f, (ax1, ax2) = plt.subplots(2, 1, sharey=True)
ax1.plot(result['timestamp'],
result['data'][:, 0],
linewidth=4)
ax1.plot(result['timestamp'],
result['data'][:, 1],
linewidth=1)
ax1.set_title('Secondary 1 vs primary')
ax2.plot(result['timestamp'],
result['data'][:, 0],
linewidth=4)
ax2.plot(result['timestamp'],
result['data'][:, 2],
linewidth=1)
ax2.set_title('Secondary 2 vs primary')
plt.show()
async def _run():
ts = np.arange(0, 500, 1)
primary_data = np.array([ts, 10 * ts - 212]).T
primary = LocalPipe("float32_1", name="primary")
await primary.write(primary_data[:101])
await primary.close_interval()
await primary.write(primary_data[110:201])
await primary.close_interval()
await primary.write(primary_data[210:301])
await primary.close()
# Secondary1: y=-3x+436
ts = np.arange(0, 500, 1)
secondary_data = np.array([ts, -3 * ts + 436]).T
secondary1 = LocalPipe("float32_1", name="secondary1")
await secondary1.write(secondary_data[50:161])
await secondary1.close_interval()
await secondary1.write(secondary_data[170:231])
await secondary1.close()
# Secondary2: y=30x+210
ts = np.arange(0, 500, 1)
secondary_data = np.array([ts, 30 * ts + 210]).T
secondary2 = LocalPipe("float32_1", name="secondary2")
await secondary2.write(secondary_data[10:191])
await secondary2.close_interval()
await secondary2.write(secondary_data[205:311])
await secondary2.close()
output = LocalPipe("float32_3", name="output")
my_filter = MergeFilter()
args = argparse.Namespace(primary="primary", pipes="unset")
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1,
'secondary2': secondary2
},
outputs={'output': output})
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 50)
self.assertEqual(chunk['timestamp'][-1], 100)
self.assertTrue(output.end_of_interval)
output.consume(len(chunk))
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 110)
self.assertEqual(chunk['timestamp'][-1], 160)
self.assertTrue(output.end_of_interval)
output.consume(len(chunk))
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 170)
self.assertEqual(chunk['timestamp'][-1], 190)
self.assertTrue(output.end_of_interval)
output.consume(len(chunk))
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 210)
self.assertEqual(chunk['timestamp'][-1], 230)
output.consume(len(chunk))
self.assertTrue(output.is_empty())