def test_computes_median(self):
# data is a repeating series of 0,1,2,..N
# the median of this array should be N/2
# timestamps is just an increasing index
my_filter = MedianFilter()
loop = asyncio.get_event_loop()
pipe_in = LocalPipe("float32_%d" % WIDTH, name="input")
pipe_out = LocalPipe("float32_%d" % WIDTH, name="output")
args = argparse.Namespace(window=WINDOW, pipes="unset")
base = np.array([np.arange(x, WINDOW + x) for x in range(WIDTH)]).T
async def writer():
prev_ts = 0
for block in range(NUM_BLOCKS):
data = np.tile(base, (REPS_PER_BLOCK, 1))
ts = np.arange(prev_ts, prev_ts + len(data))
input_block = np.hstack((ts[:, None], data))
pipe_in.write_nowait(input_block)
#await asyncio.sleep(0.1)
prev_ts = ts[-1] + 1
# now put in an extra block after an interval break
await pipe_in.close_interval()
# all 1's (even timestamps)
await pipe_in.write(np.ones((100, WIDTH + 1)))
#await asyncio.sleep(0.2)
await pipe_in.close()
loop.run_until_complete(asyncio.ensure_future(writer()))
loop.run_until_complete(
my_filter.run(args, {"input": pipe_in}, {"output": pipe_out}))
result = pipe_out.read_nowait()
# expect the output to be a constant (WINDOW-1)/2
# expect the output to be a constant (WINDOW-1)/2
base_output = np.ones(
(WINDOW * REPS_PER_BLOCK * NUM_BLOCKS - (WINDOW - 1), WIDTH))
expected = base_output * range(int(WINDOW / 2),
int(WINDOW / 2) + WIDTH)
np.testing.assert_array_equal(expected, result['data'])
self.assertTrue(pipe_out.end_of_interval)
pipe_out.consume(len(result))
# now read the extra block and make sure it has all the data
result = pipe_out.read_nowait(flatten=True)
self.assertEqual(len(result), 100 - WINDOW + 1)
loop.close()
async def _run():
ts = np.arange(0, 1000)
values = np.random.randn(1000, 1)
data = np.hstack((ts[:, None], values))
primary = helpers.TestingPipe("float32_1", name="primary")
secondary1 = helpers.TestingPipe("float32_1", name="secondary1")
secondary2 = helpers.TestingPipe("float32_1", name="secondary2")
# seed the input data
for block in np.split(data, [200, 354, 700, 800, 930]):
primary.write_nowait(block)
for block in np.split(data, [155, 600, 652, 900]):
secondary1.write_nowait(block)
for block in np.split(data, [100, 300, 600]):
secondary2.write_nowait(block)
await primary.close()
await secondary1.close()
await secondary2.close()
# run filter in an event loop
my_filter = MergeFilter()
args = argparse.Namespace(primary="primary", pipes="unset")
output = LocalPipe("float32_3", name="output")
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1,
'secondary2': secondary2
},
outputs={'output': output})
# put together the data_blocks (should not be any interval breaks)
# remove the interval close at the end
result = await output.read()
output.consume(len(result))
self.assertTrue(output.is_empty())
# all elements should match the data
np.testing.assert_array_almost_equal(result['data'][:, 0][:, None],
values)
np.testing.assert_array_almost_equal(result['data'][:, 1][:, None],
values)
np.testing.assert_array_almost_equal(result['data'][:, 2][:, None],
values)
if VISUALIZE:
from matplotlib import pyplot as plt
f, (ax1, ax2) = plt.subplots(2, 1, sharey=True)
ax1.plot(result['timestamp'],
result['data'][:, 0],
linewidth=4)
ax1.plot(result['timestamp'],
result['data'][:, 1],
linewidth=1)
ax1.set_title('Secondary 1 vs primary')
ax2.plot(result['timestamp'],
result['data'][:, 0],
linewidth=4)
ax2.plot(result['timestamp'],
result['data'][:, 2],
linewidth=1)
ax2.set_title('Secondary 2 vs primary')
plt.show()
async def _run():
ts = np.arange(0, 500, 1)
primary_data = np.array([ts, 10 * ts - 212]).T
primary = LocalPipe("float32_1", name="primary")
await primary.write(primary_data[:101])
await primary.close_interval()
await primary.write(primary_data[110:201])
await primary.close_interval()
await primary.write(primary_data[210:301])
await primary.close()
# Secondary1: y=-3x+436
ts = np.arange(0, 500, 1)
secondary_data = np.array([ts, -3 * ts + 436]).T
secondary1 = LocalPipe("float32_1", name="secondary1")
await secondary1.write(secondary_data[50:161])
await secondary1.close_interval()
await secondary1.write(secondary_data[170:231])
await secondary1.close()
# Secondary2: y=30x+210
ts = np.arange(0, 500, 1)
secondary_data = np.array([ts, 30 * ts + 210]).T
secondary2 = LocalPipe("float32_1", name="secondary2")
await secondary2.write(secondary_data[10:191])
await secondary2.close_interval()
await secondary2.write(secondary_data[205:311])
await secondary2.close()
output = LocalPipe("float32_3", name="output")
my_filter = MergeFilter()
args = argparse.Namespace(primary="primary", pipes="unset")
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1,
'secondary2': secondary2
},
outputs={'output': output})
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 50)
self.assertEqual(chunk['timestamp'][-1], 100)
self.assertTrue(output.end_of_interval)
output.consume(len(chunk))
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 110)
self.assertEqual(chunk['timestamp'][-1], 160)
self.assertTrue(output.end_of_interval)
output.consume(len(chunk))
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 170)
self.assertEqual(chunk['timestamp'][-1], 190)
self.assertTrue(output.end_of_interval)
output.consume(len(chunk))
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 210)
self.assertEqual(chunk['timestamp'][-1], 230)
output.consume(len(chunk))
self.assertTrue(output.is_empty())