async def _run(): # Primary: y=10x-212 ts = np.arange(0, 500, 1) primary_data = np.array([ts, 10 * ts - 212]).T primary = helpers.TestingPipe("float32_1", name="primary") await primary.write(primary_data[:100]) await primary.write(primary_data[100:200]) await primary.write(primary_data[200:]) await primary.close() # Secondary1: y=-3.3x+436 ts = np.arange(510, 1000, 1) secondary1_data = np.array([ts, -5 * ts + 436]).T secondary1 = helpers.TestingPipe("float32_1", name="secondary1") await secondary1.write(secondary1_data[:150]) await secondary1.write(secondary1_data[150:350]) await secondary1.write(secondary1_data[350:]) await secondary1.close() args = argparse.Namespace(primary="primary", pipes="unset") output = LocalPipe("float32_2", name="output") # run filter in an event loop my_filter = MergeFilter() await my_filter.run(args, inputs={ 'primary': primary, 'secondary1': secondary1 }, outputs={'output': output}) self.assertTrue(output.is_empty()) with self.assertRaises(EmptyPipeError): await output.read()
async def _run(): ts = np.arange(0, 1000) values = np.random.randn(1000, 1) data = np.hstack((ts[:, None], values)) primary = helpers.TestingPipe("float32_1", name="primary") secondary1 = helpers.TestingPipe("float32_1", name="secondary1") secondary2 = helpers.TestingPipe("float32_1", name="secondary2") # seed the input data for block in np.split(data, [200, 354, 700, 800, 930]): primary.write_nowait(block) for block in np.split(data, [155, 600, 652, 900]): secondary1.write_nowait(block) for block in np.split(data, [100, 300, 600]): secondary2.write_nowait(block) await primary.close() await secondary1.close() await secondary2.close() # run filter in an event loop my_filter = MergeFilter() args = argparse.Namespace(primary="primary", pipes="unset") output = LocalPipe("float32_3", name="output") await my_filter.run(args, inputs={ 'primary': primary, 'secondary1': secondary1, 'secondary2': secondary2 }, outputs={'output': output}) # put together the data_blocks (should not be any interval breaks) # remove the interval close at the end result = await output.read() output.consume(len(result)) self.assertTrue(output.is_empty()) # all elements should match the data np.testing.assert_array_almost_equal(result['data'][:, 0][:, None], values) np.testing.assert_array_almost_equal(result['data'][:, 1][:, None], values) np.testing.assert_array_almost_equal(result['data'][:, 2][:, None], values) if VISUALIZE: from matplotlib import pyplot as plt f, (ax1, ax2) = plt.subplots(2, 1, sharey=True) ax1.plot(result['timestamp'], result['data'][:, 0], linewidth=4) ax1.plot(result['timestamp'], result['data'][:, 1], linewidth=1) ax1.set_title('Secondary 1 vs primary') ax2.plot(result['timestamp'], result['data'][:, 0], linewidth=4) ax2.plot(result['timestamp'], result['data'][:, 2], linewidth=1) ax2.set_title('Secondary 2 vs primary') plt.show()
async def _run(): ts = np.arange(0, 500, 1) primary_data = np.array([ts, 10 * ts - 212]).T primary = LocalPipe("float32_1", name="primary") await primary.write(primary_data[:101]) await primary.close_interval() await primary.write(primary_data[110:201]) await primary.close_interval() await primary.write(primary_data[210:301]) await primary.close() # Secondary1: y=-3x+436 ts = np.arange(0, 500, 1) secondary_data = np.array([ts, -3 * ts + 436]).T secondary1 = LocalPipe("float32_1", name="secondary1") await secondary1.write(secondary_data[50:161]) await secondary1.close_interval() await secondary1.write(secondary_data[170:231]) await secondary1.close() # Secondary2: y=30x+210 ts = np.arange(0, 500, 1) secondary_data = np.array([ts, 30 * ts + 210]).T secondary2 = LocalPipe("float32_1", name="secondary2") await secondary2.write(secondary_data[10:191]) await secondary2.close_interval() await secondary2.write(secondary_data[205:311]) await secondary2.close() output = LocalPipe("float32_3", name="output") my_filter = MergeFilter() args = argparse.Namespace(primary="primary", pipes="unset") await my_filter.run(args, inputs={ 'primary': primary, 'secondary1': secondary1, 'secondary2': secondary2 }, outputs={'output': output}) chunk = await output.read() self.assertEqual(chunk['timestamp'][0], 50) self.assertEqual(chunk['timestamp'][-1], 100) self.assertTrue(output.end_of_interval) output.consume(len(chunk)) chunk = await output.read() self.assertEqual(chunk['timestamp'][0], 110) self.assertEqual(chunk['timestamp'][-1], 160) self.assertTrue(output.end_of_interval) output.consume(len(chunk)) chunk = await output.read() self.assertEqual(chunk['timestamp'][0], 170) self.assertEqual(chunk['timestamp'][-1], 190) self.assertTrue(output.end_of_interval) output.consume(len(chunk)) chunk = await output.read() self.assertEqual(chunk['timestamp'][0], 210) self.assertEqual(chunk['timestamp'][-1], 230) output.consume(len(chunk)) self.assertTrue(output.is_empty())