async def _run():
# Primary: y=10x-212
ts = np.arange(0, 500, 1)
primary_data = np.array([ts, 10 * ts - 212]).T
primary = helpers.TestingPipe("float32_1", name="primary")
await primary.write(primary_data[:100])
await primary.write(primary_data[100:200])
await primary.write(primary_data[200:])
await primary.close()
# Secondary1: y=-3.3x+436
ts = np.arange(510, 1000, 1)
secondary1_data = np.array([ts, -5 * ts + 436]).T
secondary1 = helpers.TestingPipe("float32_1", name="secondary1")
await secondary1.write(secondary1_data[:150])
await secondary1.write(secondary1_data[150:350])
await secondary1.write(secondary1_data[350:])
await secondary1.close()
args = argparse.Namespace(primary="primary", pipes="unset")
output = LocalPipe("float32_2", name="output")
# run filter in an event loop
my_filter = MergeFilter()
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1
},
outputs={'output': output})
self.assertTrue(output.is_empty())
with self.assertRaises(EmptyPipeError):
await output.read()
def test_generates_random_values(self):
my_reader = RandomReader()
loop = asyncio.get_event_loop()
pipe = LocalPipe("float32_%d" % WIDTH, name="output")
args = argparse.Namespace(width=WIDTH, rate=RATE, pipes="unset")
loop.call_later(0.1, my_reader.stop)
loop.run_until_complete(my_reader.run(args, pipe))
loop.close()
# check the results
result = pipe.read_nowait()
diffs = np.diff(result['timestamp'])
self.assertEqual(np.mean(diffs), 1 / RATE * 1e6)
self.assertEqual(np.shape(result['data'])[1], WIDTH)
def test_reads_timestamped_values(self):
data = helpers.create_data("float32_8", length=3)
(data_file, path) = tempfile.mkstemp()
with open(data_file, 'w') as f:
for row in data:
f.write("%d %s\n" % (row['timestamp'], ' '.join(repr(x) for x in row['data'])))
my_reader = FileReader()
loop = asyncio.get_event_loop()
pipe = LocalPipe("float32_8", name="output")
args = argparse.Namespace(file=path, delimiter=" ",
timestamp=False)
loop.run_until_complete(my_reader.run(args, pipe))
# check the results
result = pipe.read_nowait()
np.testing.assert_array_almost_equal(data['timestamp'], result['timestamp'])
np.testing.assert_array_almost_equal(data['data'], result['data'])
os.remove(path)
def test_stops_on_request(self):
data = helpers.create_data("float32_8")
(data_file, path) = tempfile.mkstemp()
with open(data_file, 'w') as f:
for row in data:
f.write("%d %s\n" % (row['timestamp'], ' '.join(repr(x) for x in row['data'])))
my_reader = FileReader()
loop = asyncio.get_event_loop()
pipe = LocalPipe("float32_8", name="output")
args = argparse.Namespace(file=path, delimiter=" ",
timestamp=False)
my_reader.stop()
loop.run_until_complete(my_reader.run(args, pipe))
# check the results
result = pipe.read_nowait()
# output should be shorter than the input
self.assertLess(len(result), len(data))
os.remove(path)
def test_timestamps_raw_values(self):
data = helpers.create_data("float32_8", length=3)
(data_file, path) = tempfile.mkstemp()
with open(data_file, 'w') as f:
for row in data:
f.write("%s\n" % ' '.join(repr(x) for x in row['data']))
my_reader = FileReader()
loop = asyncio.get_event_loop()
pipe = LocalPipe("float32_8", name="output")
args = argparse.Namespace(file=path, delimiter=" ",
timestamp=True)
loop.run_until_complete(my_reader.run(args, pipe))
# check the results
result = pipe.read_nowait()
# timestamps should be close to now
actual = np.average(result['timestamp'])
expected = int(time.time() * 1e6)
# should be within 1 second
np.testing.assert_almost_equal(actual / 1e6, expected / 1e6, decimal=0)
np.testing.assert_array_almost_equal(data['data'], result['data'])
os.remove(path)
def test_computes_median(self):
# data is a repeating series of 0,1,2,..N
# the median of this array should be N/2
# timestamps is just an increasing index
my_filter = MedianFilter()
loop = asyncio.get_event_loop()
pipe_in = LocalPipe("float32_%d" % WIDTH, name="input")
pipe_out = LocalPipe("float32_%d" % WIDTH, name="output")
args = argparse.Namespace(window=WINDOW, pipes="unset")
base = np.array([np.arange(x, WINDOW + x) for x in range(WIDTH)]).T
async def writer():
prev_ts = 0
for block in range(NUM_BLOCKS):
data = np.tile(base, (REPS_PER_BLOCK, 1))
ts = np.arange(prev_ts, prev_ts + len(data))
input_block = np.hstack((ts[:, None], data))
pipe_in.write_nowait(input_block)
#await asyncio.sleep(0.1)
prev_ts = ts[-1] + 1
# now put in an extra block after an interval break
await pipe_in.close_interval()
# all 1's (even timestamps)
await pipe_in.write(np.ones((100, WIDTH + 1)))
#await asyncio.sleep(0.2)
await pipe_in.close()
loop.run_until_complete(asyncio.ensure_future(writer()))
loop.run_until_complete(
my_filter.run(args, {"input": pipe_in}, {"output": pipe_out}))
result = pipe_out.read_nowait()
# expect the output to be a constant (WINDOW-1)/2
# expect the output to be a constant (WINDOW-1)/2
base_output = np.ones(
(WINDOW * REPS_PER_BLOCK * NUM_BLOCKS - (WINDOW - 1), WIDTH))
expected = base_output * range(int(WINDOW / 2),
int(WINDOW / 2) + WIDTH)
np.testing.assert_array_equal(expected, result['data'])
self.assertTrue(pipe_out.end_of_interval)
pipe_out.consume(len(result))
# now read the extra block and make sure it has all the data
result = pipe_out.read_nowait(flatten=True)
self.assertEqual(len(result), 100 - WINDOW + 1)
loop.close()
async def run_case(inputs, expected, primary_width):
my_filter = MergeFilter()
primary_data = np.vstack(
(inputs[0], np.ones((primary_width, len(inputs[0]))))).T
secondary1_data = np.vstack((inputs[1], 2 * np.ones(
(3, len(inputs[1]))))).T
secondary2_data = np.vstack((inputs[2], 3 * np.ones(
(1, len(inputs[2]))))).T
primary = LocalPipe("float32_%d" % primary_width, name="primary")
secondary1 = LocalPipe("float32_3", name="secondary1")
secondary2 = LocalPipe("float32_1", name="secondary2")
output = LocalPipe("float32_%d" % (primary_width + 4),
name="output")
args = argparse.Namespace(primary="primary", pipes="unset")
# seed the input data
await primary.write(primary_data)
await secondary1.write(secondary1_data)
await secondary2.write(secondary2_data)
await secondary1.close()
await secondary2.close()
await primary.close()
# run filter in an event loop
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1,
'secondary2': secondary2
},
outputs={'output': output})
result = output.read_nowait(flatten=True)
expected_data = np.vstack(
(expected, np.ones(
(primary_width, len(expected))), 2 * np.ones(
(3, len(expected))), 3 * np.ones(
(1, len(expected))))).T
np.testing.assert_array_equal(expected_data, result)
async def run() -> np.ndarray:
my_filter = MergeFilter()
primary = LocalPipe("float32_1", name="primary")
secondary1 = LocalPipe("float32_1", name="secondary1")
secondary2 = LocalPipe("float32_1", name="secondary2")
secondary3 = LocalPipe("float32_1", name="secondary3")
secondary4 = LocalPipe("float32_1", name="secondary4")
output = LocalPipe("float32_5", name="output")
args = argparse.Namespace(primary="primary", pipes="unset")
# seed the input data
primary.write_nowait(primary_data)
secondary1.write_nowait(secondary1_data)
secondary2.write_nowait(secondary2_data)
secondary3.write_nowait(secondary3_data)
secondary4.write_nowait(secondary4_data)
[
await pipe.close() for pipe in
[primary, secondary1, secondary2, secondary3, secondary4]
]
# run filter in an event loop
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1,
'secondary2': secondary2,
'secondary3': secondary3,
'secondary4': secondary4
},
outputs={'output': output})
return await output.read_all()
async def _run():
ts = np.arange(0, 1000)
values = np.random.randn(1000, 1)
data = np.hstack((ts[:, None], values))
primary = helpers.TestingPipe("float32_1", name="primary")
secondary1 = helpers.TestingPipe("float32_1", name="secondary1")
secondary2 = helpers.TestingPipe("float32_1", name="secondary2")
# seed the input data
for block in np.split(data, [200, 354, 700, 800, 930]):
primary.write_nowait(block)
for block in np.split(data, [155, 600, 652, 900]):
secondary1.write_nowait(block)
for block in np.split(data, [100, 300, 600]):
secondary2.write_nowait(block)
await primary.close()
await secondary1.close()
await secondary2.close()
# run filter in an event loop
my_filter = MergeFilter()
args = argparse.Namespace(primary="primary", pipes="unset")
output = LocalPipe("float32_3", name="output")
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1,
'secondary2': secondary2
},
outputs={'output': output})
# put together the data_blocks (should not be any interval breaks)
# remove the interval close at the end
result = await output.read()
output.consume(len(result))
self.assertTrue(output.is_empty())
# all elements should match the data
np.testing.assert_array_almost_equal(result['data'][:, 0][:, None],
values)
np.testing.assert_array_almost_equal(result['data'][:, 1][:, None],
values)
np.testing.assert_array_almost_equal(result['data'][:, 2][:, None],
values)
if VISUALIZE:
from matplotlib import pyplot as plt
f, (ax1, ax2) = plt.subplots(2, 1, sharey=True)
ax1.plot(result['timestamp'],
result['data'][:, 0],
linewidth=4)
ax1.plot(result['timestamp'],
result['data'][:, 1],
linewidth=1)
ax1.set_title('Secondary 1 vs primary')
ax2.plot(result['timestamp'],
result['data'][:, 0],
linewidth=4)
ax2.plot(result['timestamp'],
result['data'][:, 2],
linewidth=1)
ax2.set_title('Secondary 2 vs primary')
plt.show()
async def _run():
ts = np.arange(0, 500, 1)
primary_data = np.array([ts, 10 * ts - 212]).T
primary = LocalPipe("float32_1", name="primary")
await primary.write(primary_data[:101])
await primary.close_interval()
await primary.write(primary_data[110:201])
await primary.close_interval()
await primary.write(primary_data[210:301])
await primary.close()
# Secondary1: y=-3x+436
ts = np.arange(0, 500, 1)
secondary_data = np.array([ts, -3 * ts + 436]).T
secondary1 = LocalPipe("float32_1", name="secondary1")
await secondary1.write(secondary_data[50:161])
await secondary1.close_interval()
await secondary1.write(secondary_data[170:231])
await secondary1.close()
# Secondary2: y=30x+210
ts = np.arange(0, 500, 1)
secondary_data = np.array([ts, 30 * ts + 210]).T
secondary2 = LocalPipe("float32_1", name="secondary2")
await secondary2.write(secondary_data[10:191])
await secondary2.close_interval()
await secondary2.write(secondary_data[205:311])
await secondary2.close()
output = LocalPipe("float32_3", name="output")
my_filter = MergeFilter()
args = argparse.Namespace(primary="primary", pipes="unset")
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1,
'secondary2': secondary2
},
outputs={'output': output})
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 50)
self.assertEqual(chunk['timestamp'][-1], 100)
self.assertTrue(output.end_of_interval)
output.consume(len(chunk))
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 110)
self.assertEqual(chunk['timestamp'][-1], 160)
self.assertTrue(output.end_of_interval)
output.consume(len(chunk))
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 170)
self.assertEqual(chunk['timestamp'][-1], 190)
self.assertTrue(output.end_of_interval)
output.consume(len(chunk))
chunk = await output.read()
self.assertEqual(chunk['timestamp'][0], 210)
self.assertEqual(chunk['timestamp'][-1], 230)
output.consume(len(chunk))
self.assertTrue(output.is_empty())
async def _run():
# Primary: y=10x-212
ts = np.arange(0, 500, 1)
primary_data = np.array([ts, 10 * ts - 212]).T
primary = helpers.TestingPipe("float32_1", name="primary")
primary.write_nowait(primary_data[:100])
primary.write_nowait(primary_data[100:200])
primary.write_nowait(primary_data[200:])
await primary.close()
# Secondary1: y=-3.3x+436
ts = np.arange(0, 500, 1)
secondary1_data = np.array([ts, -5 * ts + 436]).T
secondary1 = helpers.TestingPipe("float32_1", name="secondary1")
secondary1.write_nowait(secondary1_data[:150])
secondary1.write_nowait(secondary1_data[150:350])
secondary1.write_nowait(secondary1_data[350:])
await secondary1.close()
args = argparse.Namespace(primary="primary", pipes="unset")
output = LocalPipe("float32_2", name="output")
# run filter in an event loop
my_filter = MergeFilter()
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1
},
outputs={'output': output})
# put together the data_blocks (should not be any interval breaks)
# remove the interval close at the end
result = await output.read_all()
ts = result['timestamp']
# check that the timestamps cover the correct range
self.assertEqual(ts[0], 0)
self.assertEqual(ts[-1], 499)
# no duplicate timestamps
self.assertEqual(len(np.unique(ts)), len(primary_data))
# check the primary
primary_actual = result['data'][:, 0]
residual = (10 * ts - 212) - primary_actual
np.testing.assert_allclose(residual, 0)
# check secondary1
secondary1_actual = result['data'][:, 1]
residual = (-5 * ts + 436) - secondary1_actual
np.testing.assert_allclose(residual, 0)
# NOTE: this test is close but not perfect... hmmm
np.testing.assert_allclose(residual, 0, rtol=1e-5, atol=1e-4)
if VISUALIZE:
from matplotlib import pyplot as plt
for data in [primary_data, secondary1_data]:
plt.plot(data[:, 0], data[:, 1], linewidth=1)
plt.plot(result['timestamp'],
result['data'],
'--',
linewidth=2)
plt.show()
async def _run():
# Primary: y=10x-212
ts = np.arange(0, 1300, 10)
primary_data = np.array([ts, 10 * ts - 212]).T
primary = helpers.TestingPipe("float32_1", name="primary")
await primary.write(primary_data[:10])
await primary.write(primary_data[10:20])
await primary.write(primary_data[20:])
await primary.close()
# Secondary1: y=-3.3x+436
ts = np.arange(500, 1000, 10)
secondary1_data = np.array([ts, -3.3 * ts + 436]).T
secondary1 = helpers.TestingPipe("float32_1", name="secondary1")
await secondary1.write(secondary1_data[:20])
await secondary1.write(secondary1_data[20:30])
await secondary1.write(secondary1_data[30:])
await secondary1.close()
args = argparse.Namespace(primary="primary", pipes="unset")
output = LocalPipe("float32_2", name="output")
# run filter in an event loop
my_filter = MergeFilter()
await my_filter.run(args,
inputs={
'primary': primary,
'secondary1': secondary1
},
outputs={'output': output})
result = await output.read_all()
ts = result['timestamp']
# check that the timestamps cover the correct range
self.assertEqual(ts[0], 500)
self.assertEqual(ts[-1], 990)
# check the primary
primary_actual = result['data'][:, 0]
residual = (10 * ts - 212) - primary_actual
np.testing.assert_allclose(residual, 0)
# check secondary1
secondary1_actual = result['data'][:, 1]
residual = (-3.3 * ts + 436) - secondary1_actual
np.testing.assert_allclose(residual, 0)
# NOTE: this test is close but not perfect... hmmm
np.testing.assert_allclose(residual, 0, rtol=1e-5, atol=1e-4)
if VISUALIZE:
from matplotlib import pyplot as plt
for data in [primary_data, secondary1_data]:
plt.plot(data[:, 0], data[:, 1], linewidth=1)
plt.plot(result['timestamp'],
result['data'],
'--',
linewidth=2)
plt.show()