def dump_ring_and_read(self): """Dump block to ring, read in as histogram""" logfile = ".log.txt" self.blocks.append((WriteAsciiBlock(logfile), [1], [])) Pipeline(self.blocks).main() test_bytes = open(logfile, 'r').read().split(' ') histogram = np.array([np.float(x) for x in test_bytes]) return histogram
def tearDown(self): """Run the pipeline and test the output against the expectation""" Pipeline(self.blocks).main() if np.array(self.expected_result).dtype == 'complex128': result = np.loadtxt('.log.txt', dtype=np.float64).view(np.complex128) else: result = np.loadtxt('.log.txt').astype(np.float32) np.testing.assert_almost_equal(result, self.expected_result)
def test_simple_large_gulp(self): """Test if a large gulp size produces a seg fault""" blocks = [] blocks.append((TestingBlock([1, 2, 3]), [], [0])) blocks.append((ModResizeAsciiBlock('.log.txt', gulp_size=1024), [0], [])) Pipeline(blocks).main() np.testing.assert_almost_equal(np.loadtxt('.log.txt'), [1, 2, 3])
def test_for_bad_ring_definitions(self): """Try to pass bad input and outputs""" blocks = [] blocks.append([TestingBlock([1, 2]), [], [0]]) blocks.append([MultiAddBlock(), {'in_2': 0, 'out_sum': 1}]) blocks.append([WriteAsciiBlock('.log.txt'), [1], []]) with self.assertRaises(AssertionError): Pipeline(blocks).main() blocks[1] = [ MultiAddBlock(), { 'bad_ring_name': 0, 'in_2': 0, 'out_sum': 1 } ] with self.assertRaises(AssertionError): Pipeline(blocks).main()
def test_throughput(self): """Read in data with a small throughput size. Expect all to go through.""" blocks = [] blocks.append((SigprocReadBlock('./data/1chan8bitNoDM.fil', gulp_nframe=4096), [], [0])) blocks.append((WriteAsciiBlock('.log.txt'), [0], [])) Pipeline(blocks).main() log_data = np.loadtxt('.log.txt') self.assertEqual(log_data.size, 12800)
def test_modified_write_ascii(self): """Using a modified WriteAciiBlock, test the late resize. This should fail if ModWriteAscii block does not read the size of the input ring ahead of time, and resize accordingly.""" blocks = [] blocks.append((TestingBlock([1, 2, 3]), [], [0])) blocks.append((ModResizeAsciiBlock('.log.txt'), [0], [])) Pipeline(blocks).main() np.testing.assert_almost_equal(np.loadtxt('.log.txt'), [1, 2, 3])
def test_simple_copy(self): """Test which performs a read of a sigproc file, copy to one ring, and then output as text.""" logfile = '.log.txt' self.blocks.append((CopyBlock(), [0], [1])) self.blocks.append((WriteAsciiBlock(logfile), [1], [])) Pipeline(self.blocks).main() test_byte = open(logfile, 'r').read(1) self.assertEqual(test_byte, '2')
def test_naming_rings(self): """Name the rings instead of numerating them""" blocks = [] blocks.append((TestingBlock([1, 2, 3]), [], ['ring1'])) blocks.append((WriteAsciiBlock('.log.txt', gulp_size=3 * 4), ['ring1'], [])) open('.log.txt', 'w').close() Pipeline(blocks).main() result = np.loadtxt('.log.txt').astype(np.float32) np.testing.assert_almost_equal(result, [1, 2, 3])
def test_multi_dimensional_input(self): """Input a 2 dimensional list, and have this printed""" test_array = [[1, 2], [3, 4]] self.blocks[0] = (WriteAsciiBlock('.log.txt', gulp_size=4*4), [0], []) self.blocks.append((TestingBlock(test_array), [], [0])) self.blocks.append((WriteHeaderBlock('.log2.txt'), [0], [])) Pipeline(self.blocks).main() header = eval(open('.log2.txt').read()) #pylint:disable=eval-used dumped_numbers = np.loadtxt('.log.txt').reshape(header['shape']) np.testing.assert_almost_equal(dumped_numbers, test_array)
def test_two_sequences(self): """Make sure multiple sequences only triggered for different headers""" np.random.seed(44) def generate_two_different_arrays(): """Generate 10 of an array shape, then 10 of a different array shape""" for _ in range(10): yield np.random.rand(4) for _ in range(10): yield np.random.rand(5) self.triggered = False self.monitor_block = None self.sequence_id = "" self.i = 0 #This array holds all of the #starting numbers. If there #are more than two different #numbers, then there is a problem self.all_sequence_starts = [] def monitor_block_sequences(array): """Read the newest sequence, and append the first byte to the all_sequence_starts""" #Avoid reading an empty sequence if self.i > 1 and self.i < 11: with self.monitor_block.rings['out_1'].open_latest_sequence( guarantee=False) as curr_seq: span_gen = curr_seq.read(1) self.all_sequence_starts.append(int( next(span_gen).data[0])) if self.i > 12: with self.monitor_block.rings['out_1'].open_latest_sequence( guarantee=False) as curr_seq: span_gen = curr_seq.read(1) self.all_sequence_starts.append(int( next(span_gen).data[0])) self.i += 1 return array self.monitor_block = NumpyBlock(monitor_block_sequences) blocks = [(NumpySourceBlock(generate_two_different_arrays), { 'out_1': 0 }), (self.monitor_block, { 'in_1': 0, 'out_1': 1 })] Pipeline(blocks).main() unique_starts = len(set(self.all_sequence_starts)) self.assertEqual(unique_starts, 2)
def test_data_sizes(self): """Test that different number of bits give correct throughput size""" for iterate in range(5): nbit = 2**iterate if nbit == 8: continue self.blocks[0] = (SigprocReadBlock('./data/2chan' + str(nbit) + 'bitNoDM.fil'), [], [0]) open(self.logfile, 'w').close() Pipeline(self.blocks).main() number_fftd = np.loadtxt(self.logfile).astype(np.float32).view( np.complex64).size # Compare with simple copy self.blocks[1] = (CopyBlock(), [0], [1]) open(self.logfile, 'w').close() Pipeline(self.blocks).main() number_copied = np.loadtxt(self.logfile).size self.assertEqual(number_fftd, number_copied) # Go back to FFT self.blocks[1] = (FFTBlock(gulp_size=4096 * 8 * 8 * 8), [0], [1])
def test_multi_copy(self): """Test which performs a read of a sigproc file, copy between many rings, and then output as text.""" logfile = '.log.txt' for i in range(10): self.blocks.append((CopyBlock(), [i], [i + 1])) self.blocks.append((WriteAsciiBlock(logfile), [10], [])) Pipeline(self.blocks).main() test_byte = open(logfile, 'r').read(1) self.assertEqual(test_byte, '2')
def test_32bit_copy(self): """Perform a simple test to confirm that 32 bit copying has no information loss""" logfile = '.log.txt' self.blocks = [] self.blocks.append( (SigprocReadBlock('./data/256chan32bitNoDM.fil'), [], [0])) self.blocks.append((CopyBlock(), [0], [1])) self.blocks.append((WriteAsciiBlock(logfile), [1], [])) Pipeline(self.blocks).main() test_bytes = open(logfile, 'r').read(500).split(' ') self.assertAlmostEqual(np.float(test_bytes[0]), 0.72650784254)
def test_data_throughput(self): """Check that data is being put through the block (does this by checking consistency of shape/datatype)""" blocks = [] blocks.append((SigprocReadBlock('./data/1chan8bitNoDM.fil'), [], [0])) blocks.append((KurtosisBlock(), [0], [1])) blocks.append((WriteAsciiBlock('.log.txt'), [1], [])) Pipeline(blocks).main() test_byte = open('.log.txt', 'r').read().split(' ') test_nums = np.array([float(x) for x in test_byte]) self.assertLess(np.max(test_nums), 256) self.assertEqual(test_nums.size, 12800)
def test_single_block_multi_copy(self): """Test which forces one block to do multiple copies at once, and then dumps to two files, checking them both.""" logfiles = ['.log1.txt', '.log2.txt'] self.blocks.append((CopyBlock(), [0], [1, 2])) self.blocks.append((WriteAsciiBlock(logfiles[0]), [1], [])) self.blocks.append((WriteAsciiBlock(logfiles[1]), [2], [])) Pipeline(self.blocks).main() test_bytes = int(open(logfiles[0], 'r').read(1)) + int( open(logfiles[1], 'r').read(1)) self.assertEqual(test_bytes, 4)
def test_equivalent_data_to_copy(self): """Test that the data coming out of this pipeline is equivalent the initial read data""" self.logfile = '.log.txt' self.blocks = [] self.blocks.append(( SigprocReadBlock( './data/1chan8bitNoDM.fil'), [], [0])) self.blocks.append((FFTBlock(gulp_size=4096 * 8 * 8 * 8 * 8), [0], [1])) self.blocks.append((IFFTBlock(gulp_size=4096 * 8 * 8 * 8 * 8), [1], [2])) self.blocks.append((WriteAsciiBlock(self.logfile), [2], [])) open(self.logfile, 'w').close() Pipeline(self.blocks).main() unfft_result = np.loadtxt(self.logfile).astype(np.float32).view(np.complex64) self.blocks[1] = (CopyBlock(), [0], [1]) self.blocks[2] = (WriteAsciiBlock(self.logfile), [1], []) del self.blocks[3] open(self.logfile, 'w').close() Pipeline(self.blocks).main() untouched_result = np.loadtxt(self.logfile).astype(np.float32) np.testing.assert_almost_equal(unfft_result, untouched_result, 2)
def test_simple_half_split(self): """Try to split up a single array in half, and dump to file""" blocks = [] blocks.append([TestingBlock([1, 2]), [], [0]]) blocks.append([SplitterBlock([[0], [1]]), {'in': 0, 'out_1': 1, 'out_2': 2}]) blocks.append([WriteAsciiBlock('.log1.txt', gulp_size=4), [1], []]) blocks.append([WriteAsciiBlock('.log2.txt', gulp_size=4), [2], []]) Pipeline(blocks).main() first_log = np.loadtxt('.log1.txt') second_log = np.loadtxt('.log2.txt') self.assertEqual(first_log.size, 1) self.assertEqual(second_log.size, 1) np.testing.assert_almost_equal(first_log + 1, second_log)
def test_simple_ifft(self): """Put test data through a ring buffer and check correctness""" self.logfile = '.log.txt' self.blocks = [] test_array = [1, 2, 3] self.blocks.append((TestingBlock(test_array), [], [0])) self.blocks.append((IFFTBlock(gulp_size=3 * 4), [0], [1])) self.blocks.append((WriteAsciiBlock(self.logfile), [1], [])) open(self.logfile, 'w').close() Pipeline(self.blocks).main() true_result = np.fft.ifft(test_array) result = np.loadtxt(self.logfile).astype(np.float32).view(np.complex64) np.testing.assert_almost_equal(result, true_result, 2)
def test_simple_single_generation(self): """For single yields, should act like a TestingBlock""" def generate_one_array(): """Put out a single numpy array""" yield np.array([1, 2, 3, 4]).astype(np.float32) def assert_expectation(array): """Assert the array is as expected""" np.testing.assert_almost_equal(array, [1, 2, 3, 4]) self.occurences += 1 blocks = [] blocks.append((NumpySourceBlock(generate_one_array), {'out_1': 0})) blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0})) Pipeline(blocks).main() self.assertEqual(self.occurences, 1)
def test_multiple_yields(self): """Should be able to repeat generation of an array""" def generate_10_arrays(): """Put out 10 numpy arrays""" for _ in range(10): yield np.array([1, 2, 3, 4]).astype(np.float32) def assert_expectation(array): """Assert the array is as expected""" np.testing.assert_almost_equal(array, [1, 2, 3, 4]) self.occurences += 1 blocks = [] blocks.append((NumpySourceBlock(generate_10_arrays), {'out_1': 0})) blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0})) Pipeline(blocks).main() self.assertEqual(self.occurences, 10)
def test_global_variable_capture(self): """Test that we can pull out a number from a ring using NumpyBlock""" self.global_variable = np.array([]) def create_global_variable(array): """Try to append the array to a global variable""" self.global_variable = np.copy(array) self.blocks.append([ NumpyBlock(function=create_global_variable, outputs=0), {'in_1': 0}]) self.blocks.append([ NumpyBlock(function=np.copy), {'in_1': 0, 'out_1': 1}]) Pipeline(self.blocks).main() open('.log.txt', 'w').close() np.testing.assert_almost_equal(self.global_variable, [1, 2, 3, 4]) self.expected_result = [1, 2, 3, 4]
def test_non_linear_multi_copy(self): """Test which reads in a sigproc file, and loads it between different rings in a nonlinear fashion, then outputs to file.""" logfile = '.log.txt' self.blocks.append((CopyBlock(), [0], [1])) self.blocks.append((CopyBlock(), [0], [2])) self.blocks.append((CopyBlock(), [2], [5])) self.blocks.append((CopyBlock(), [0], [3])) self.blocks.append((CopyBlock(), [3], [4])) self.blocks.append((CopyBlock(), [5], [6])) self.blocks.append((WriteAsciiBlock(logfile), [6], [])) Pipeline(self.blocks).main() log_nums = open(logfile, 'r').read(500).split(' ') test_num = np.float(log_nums[8]) self.assertEqual(test_num, 3)
def test_header_output(self): """Output a header for a ring explicitly""" def generate_array_and_header(): """Output the desired header of an array""" header = {'dtype': 'complex128', 'nbit': 128} yield np.array([1, 2, 3, 4]), header def assert_expectation(array): "Assert that the array has a complex datatype" np.testing.assert_almost_equal(array, [1, 2, 3, 4]) self.assertEqual(array.dtype, np.dtype('complex128')) self.occurences += 1 blocks = [] blocks.append(( NumpySourceBlock(generate_array_and_header, grab_headers=True), {'out_1': 0})) blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0})) Pipeline(blocks).main() self.assertEqual(self.occurences, 1)
def test_multiple_sequences(self): """Try to send multiple sequences through a branching pipeline""" def generate_different_arrays(): """Yield four different groups of two arrays""" dtypes = ['float32', 'float64', 'complex64', 'int8'] shapes = [(4, ), (4, 5), (4, 5, 6), (2, ) * 8] for array_index in range(4): yield np.ones(shape=shapes[array_index], dtype=dtypes[array_index]) yield 2 * np.ones(shape=shapes[array_index], dtype=dtypes[array_index]) def switch_types(array): """Return two copies of the array, one with a different type""" return np.copy(array), np.copy(array).astype(np.complex128) self.occurences = 0 def compare_arrays(array1, array2): """Make sure that all arrays coming in are equal""" self.occurences += 1 np.testing.assert_almost_equal(array1, array2) blocks = [(NumpySourceBlock(generate_different_arrays), { 'out_1': 0 }), (NumpyBlock(switch_types, outputs=2), { 'in_1': 0, 'out_1': 1, 'out_2': 2 }), (NumpyBlock(np.fft.fft), { 'in_1': 2, 'out_1': 3 }), (NumpyBlock(np.fft.ifft), { 'in_1': 3, 'out_1': 4 }), (NumpyBlock(compare_arrays, inputs=2, outputs=0), { 'in_1': 1, 'in_2': 4 })] Pipeline(blocks).main() self.assertEqual(self.occurences, 8)
def test_add_block(self): """Try some syntax on an addition block.""" my_ring = Ring() blocks = [] blocks.append([TestingBlock([1, 2]), [], [0]]) blocks.append([TestingBlock([1, 6]), [], [1]]) blocks.append([TestingBlock([9, 2]), [], [2]]) blocks.append([TestingBlock([6, 2]), [], [3]]) blocks.append([TestingBlock([1, 2]), [], [4]]) blocks.append( [MultiAddBlock(), { 'in_1': 0, 'in_2': 1, 'out_sum': 'first_sum' }]) blocks.append( [MultiAddBlock(), { 'in_1': 2, 'in_2': 3, 'out_sum': 'second_sum' }]) blocks.append([ MultiAddBlock(), { 'in_1': 'first_sum', 'in_2': 'second_sum', 'out_sum': 'third_sum' } ]) blocks.append([ MultiAddBlock(), { 'in_1': 'third_sum', 'in_2': 4, 'out_sum': my_ring } ]) def assert_result_of_addition(array): """Make sure that the above arrays add up to what we expect""" np.testing.assert_almost_equal(array, [18, 14]) blocks.append((NumpyBlock(assert_result_of_addition, outputs=0), { 'in_1': my_ring })) Pipeline(blocks).main()
def test_multiple_output_rings(self): """Multiple output ring test.""" def generate_many_arrays(): """Put out 10x10 numpy arrays""" for _ in range(10): yield (np.array([1, 2, 3, 4]).astype(np.float32),) * 10 def assert_expectation(*args): """Assert the arrays are as expected""" assert len(args) == 10 for array in args: np.testing.assert_almost_equal(array, [1, 2, 3, 4]) self.occurences += 1 blocks = [] blocks.append(( NumpySourceBlock(generate_many_arrays, outputs=10), {'out_%d' % (i + 1): i for i in range(10)})) blocks.append(( NumpyBlock(assert_expectation, inputs=10, outputs=0), {'in_%d' % (i + 1): i for i in range(10)})) Pipeline(blocks).main() self.assertEqual(self.occurences, 10)
def test_output_change(self): """Change the output of the source, and expect new sequence""" self.occurences = 0 def generate_different_arrays(): """Yield two different arrays""" yield np.array([1, 2]) yield np.array([1, 2, 3]) def assert_change(array): """Assert the input arrays change""" if self.occurences == 0: np.testing.assert_almost_equal(array, [1, 2]) else: np.testing.assert_almost_equal(array, [1, 2, 3]) self.occurences += 1 blocks = [ (NumpySourceBlock(generate_different_arrays), {'out_1': 0}), (NumpyBlock(np.copy), {'in_1': 0, 'out_1': 1}), (NumpyBlock(assert_change, outputs=0), {'in_1': 1})] Pipeline(blocks).main() self.assertEqual(self.occurences, 2)
def test_simple_dump(self): """Input some numbers, and ensure they are written to a file""" self.blocks.append((TestingBlock([1, 2, 3]), [], [0])) Pipeline(self.blocks).main() dumped_numbers = np.loadtxt('.log.txt') np.testing.assert_almost_equal(dumped_numbers, [1, 2, 3])
def test_throughput(self): """Test that any data is being put through""" Pipeline(self.blocks).main() test_string = open(self.logfile, 'r').read() self.assertGreater(len(test_string), 0)