def dump_ring_and_read(self):
"""Dump block to ring, read in as histogram"""
logfile = ".log.txt"
self.blocks.append((WriteAsciiBlock(logfile), [1], []))
Pipeline(self.blocks).main()
test_bytes = open(logfile, 'r').read().split(' ')
histogram = np.array([np.float(x) for x in test_bytes])
return histogram
def tearDown(self):
"""Run the pipeline and test the output against the expectation"""
Pipeline(self.blocks).main()
if np.array(self.expected_result).dtype == 'complex128':
result = np.loadtxt('.log.txt', dtype=np.float64).view(np.complex128)
else:
result = np.loadtxt('.log.txt').astype(np.float32)
np.testing.assert_almost_equal(result, self.expected_result)
def test_simple_large_gulp(self):
"""Test if a large gulp size produces a seg fault"""
blocks = []
blocks.append((TestingBlock([1, 2, 3]), [], [0]))
blocks.append((ModResizeAsciiBlock('.log.txt',
gulp_size=1024), [0], []))
Pipeline(blocks).main()
np.testing.assert_almost_equal(np.loadtxt('.log.txt'), [1, 2, 3])
def test_for_bad_ring_definitions(self):
"""Try to pass bad input and outputs"""
blocks = []
blocks.append([TestingBlock([1, 2]), [], [0]])
blocks.append([MultiAddBlock(), {'in_2': 0, 'out_sum': 1}])
blocks.append([WriteAsciiBlock('.log.txt'), [1], []])
with self.assertRaises(AssertionError):
Pipeline(blocks).main()
blocks[1] = [
MultiAddBlock(), {
'bad_ring_name': 0,
'in_2': 0,
'out_sum': 1
}
]
with self.assertRaises(AssertionError):
Pipeline(blocks).main()
def test_throughput(self):
"""Read in data with a small throughput size. Expect all to go through."""
blocks = []
blocks.append((SigprocReadBlock('./data/1chan8bitNoDM.fil',
gulp_nframe=4096), [], [0]))
blocks.append((WriteAsciiBlock('.log.txt'), [0], []))
Pipeline(blocks).main()
log_data = np.loadtxt('.log.txt')
self.assertEqual(log_data.size, 12800)
def test_modified_write_ascii(self):
"""Using a modified WriteAciiBlock, test the late resize.
This should fail if ModWriteAscii block does not read the
size of the input ring ahead of time, and resize accordingly."""
blocks = []
blocks.append((TestingBlock([1, 2, 3]), [], [0]))
blocks.append((ModResizeAsciiBlock('.log.txt'), [0], []))
Pipeline(blocks).main()
np.testing.assert_almost_equal(np.loadtxt('.log.txt'), [1, 2, 3])
def test_simple_copy(self):
"""Test which performs a read of a sigproc file,
copy to one ring, and then output as text."""
logfile = '.log.txt'
self.blocks.append((CopyBlock(), [0], [1]))
self.blocks.append((WriteAsciiBlock(logfile), [1], []))
Pipeline(self.blocks).main()
test_byte = open(logfile, 'r').read(1)
self.assertEqual(test_byte, '2')
def test_naming_rings(self):
"""Name the rings instead of numerating them"""
blocks = []
blocks.append((TestingBlock([1, 2, 3]), [], ['ring1']))
blocks.append((WriteAsciiBlock('.log.txt', gulp_size=3 * 4), ['ring1'], []))
open('.log.txt', 'w').close()
Pipeline(blocks).main()
result = np.loadtxt('.log.txt').astype(np.float32)
np.testing.assert_almost_equal(result, [1, 2, 3])
def test_multi_dimensional_input(self):
"""Input a 2 dimensional list, and have this printed"""
test_array = [[1, 2], [3, 4]]
self.blocks[0] = (WriteAsciiBlock('.log.txt', gulp_size=4*4), [0], [])
self.blocks.append((TestingBlock(test_array), [], [0]))
self.blocks.append((WriteHeaderBlock('.log2.txt'), [0], []))
Pipeline(self.blocks).main()
header = eval(open('.log2.txt').read()) #pylint:disable=eval-used
dumped_numbers = np.loadtxt('.log.txt').reshape(header['shape'])
np.testing.assert_almost_equal(dumped_numbers, test_array)
def test_two_sequences(self):
"""Make sure multiple sequences only triggered for different headers"""
np.random.seed(44)
def generate_two_different_arrays():
"""Generate 10 of an array shape, then 10 of a different array shape"""
for _ in range(10):
yield np.random.rand(4)
for _ in range(10):
yield np.random.rand(5)
self.triggered = False
self.monitor_block = None
self.sequence_id = ""
self.i = 0
#This array holds all of the
#starting numbers. If there
#are more than two different
#numbers, then there is a problem
self.all_sequence_starts = []
def monitor_block_sequences(array):
"""Read the newest sequence, and append the first
byte to the all_sequence_starts"""
#Avoid reading an empty sequence
if self.i > 1 and self.i < 11:
with self.monitor_block.rings['out_1'].open_latest_sequence(
guarantee=False) as curr_seq:
span_gen = curr_seq.read(1)
self.all_sequence_starts.append(int(
next(span_gen).data[0]))
if self.i > 12:
with self.monitor_block.rings['out_1'].open_latest_sequence(
guarantee=False) as curr_seq:
span_gen = curr_seq.read(1)
self.all_sequence_starts.append(int(
next(span_gen).data[0]))
self.i += 1
return array
self.monitor_block = NumpyBlock(monitor_block_sequences)
blocks = [(NumpySourceBlock(generate_two_different_arrays), {
'out_1': 0
}), (self.monitor_block, {
'in_1': 0,
'out_1': 1
})]
Pipeline(blocks).main()
unique_starts = len(set(self.all_sequence_starts))
self.assertEqual(unique_starts, 2)
def test_data_sizes(self):
"""Test that different number of bits give correct throughput size"""
for iterate in range(5):
nbit = 2**iterate
if nbit == 8:
continue
self.blocks[0] = (SigprocReadBlock('./data/2chan' + str(nbit) +
'bitNoDM.fil'), [], [0])
open(self.logfile, 'w').close()
Pipeline(self.blocks).main()
number_fftd = np.loadtxt(self.logfile).astype(np.float32).view(
np.complex64).size
# Compare with simple copy
self.blocks[1] = (CopyBlock(), [0], [1])
open(self.logfile, 'w').close()
Pipeline(self.blocks).main()
number_copied = np.loadtxt(self.logfile).size
self.assertEqual(number_fftd, number_copied)
# Go back to FFT
self.blocks[1] = (FFTBlock(gulp_size=4096 * 8 * 8 * 8), [0], [1])
def test_multi_copy(self):
"""Test which performs a read of a sigproc file,
copy between many rings, and then output as
text."""
logfile = '.log.txt'
for i in range(10):
self.blocks.append((CopyBlock(), [i], [i + 1]))
self.blocks.append((WriteAsciiBlock(logfile), [10], []))
Pipeline(self.blocks).main()
test_byte = open(logfile, 'r').read(1)
self.assertEqual(test_byte, '2')
def test_32bit_copy(self):
"""Perform a simple test to confirm that 32 bit
copying has no information loss"""
logfile = '.log.txt'
self.blocks = []
self.blocks.append(
(SigprocReadBlock('./data/256chan32bitNoDM.fil'), [], [0]))
self.blocks.append((CopyBlock(), [0], [1]))
self.blocks.append((WriteAsciiBlock(logfile), [1], []))
Pipeline(self.blocks).main()
test_bytes = open(logfile, 'r').read(500).split(' ')
self.assertAlmostEqual(np.float(test_bytes[0]), 0.72650784254)
def test_data_throughput(self):
"""Check that data is being put through the block
(does this by checking consistency of shape/datatype)"""
blocks = []
blocks.append((SigprocReadBlock('./data/1chan8bitNoDM.fil'), [], [0]))
blocks.append((KurtosisBlock(), [0], [1]))
blocks.append((WriteAsciiBlock('.log.txt'), [1], []))
Pipeline(blocks).main()
test_byte = open('.log.txt', 'r').read().split(' ')
test_nums = np.array([float(x) for x in test_byte])
self.assertLess(np.max(test_nums), 256)
self.assertEqual(test_nums.size, 12800)
def test_single_block_multi_copy(self):
"""Test which forces one block to do multiple
copies at once, and then dumps to two files,
checking them both."""
logfiles = ['.log1.txt', '.log2.txt']
self.blocks.append((CopyBlock(), [0], [1, 2]))
self.blocks.append((WriteAsciiBlock(logfiles[0]), [1], []))
self.blocks.append((WriteAsciiBlock(logfiles[1]), [2], []))
Pipeline(self.blocks).main()
test_bytes = int(open(logfiles[0], 'r').read(1)) + int(
open(logfiles[1], 'r').read(1))
self.assertEqual(test_bytes, 4)
def test_equivalent_data_to_copy(self):
"""Test that the data coming out of this pipeline is equivalent
the initial read data"""
self.logfile = '.log.txt'
self.blocks = []
self.blocks.append((
SigprocReadBlock(
'./data/1chan8bitNoDM.fil'),
[], [0]))
self.blocks.append((FFTBlock(gulp_size=4096 * 8 * 8 * 8 * 8), [0], [1]))
self.blocks.append((IFFTBlock(gulp_size=4096 * 8 * 8 * 8 * 8), [1], [2]))
self.blocks.append((WriteAsciiBlock(self.logfile), [2], []))
open(self.logfile, 'w').close()
Pipeline(self.blocks).main()
unfft_result = np.loadtxt(self.logfile).astype(np.float32).view(np.complex64)
self.blocks[1] = (CopyBlock(), [0], [1])
self.blocks[2] = (WriteAsciiBlock(self.logfile), [1], [])
del self.blocks[3]
open(self.logfile, 'w').close()
Pipeline(self.blocks).main()
untouched_result = np.loadtxt(self.logfile).astype(np.float32)
np.testing.assert_almost_equal(unfft_result, untouched_result, 2)
def test_simple_half_split(self):
"""Try to split up a single array in half, and dump to file"""
blocks = []
blocks.append([TestingBlock([1, 2]), [], [0]])
blocks.append([SplitterBlock([[0], [1]]), {'in': 0, 'out_1': 1, 'out_2': 2}])
blocks.append([WriteAsciiBlock('.log1.txt', gulp_size=4), [1], []])
blocks.append([WriteAsciiBlock('.log2.txt', gulp_size=4), [2], []])
Pipeline(blocks).main()
first_log = np.loadtxt('.log1.txt')
second_log = np.loadtxt('.log2.txt')
self.assertEqual(first_log.size, 1)
self.assertEqual(second_log.size, 1)
np.testing.assert_almost_equal(first_log + 1, second_log)
def test_simple_ifft(self):
"""Put test data through a ring buffer and check correctness"""
self.logfile = '.log.txt'
self.blocks = []
test_array = [1, 2, 3]
self.blocks.append((TestingBlock(test_array), [], [0]))
self.blocks.append((IFFTBlock(gulp_size=3 * 4), [0], [1]))
self.blocks.append((WriteAsciiBlock(self.logfile), [1], []))
open(self.logfile, 'w').close()
Pipeline(self.blocks).main()
true_result = np.fft.ifft(test_array)
result = np.loadtxt(self.logfile).astype(np.float32).view(np.complex64)
np.testing.assert_almost_equal(result, true_result, 2)
def test_simple_single_generation(self):
"""For single yields, should act like a TestingBlock"""
def generate_one_array():
"""Put out a single numpy array"""
yield np.array([1, 2, 3, 4]).astype(np.float32)
def assert_expectation(array):
"""Assert the array is as expected"""
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.occurences += 1
blocks = []
blocks.append((NumpySourceBlock(generate_one_array), {'out_1': 0}))
blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 1)
def test_multiple_yields(self):
"""Should be able to repeat generation of an array"""
def generate_10_arrays():
"""Put out 10 numpy arrays"""
for _ in range(10):
yield np.array([1, 2, 3, 4]).astype(np.float32)
def assert_expectation(array):
"""Assert the array is as expected"""
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.occurences += 1
blocks = []
blocks.append((NumpySourceBlock(generate_10_arrays), {'out_1': 0}))
blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 10)
def test_global_variable_capture(self):
"""Test that we can pull out a number from a ring using NumpyBlock"""
self.global_variable = np.array([])
def create_global_variable(array):
"""Try to append the array to a global variable"""
self.global_variable = np.copy(array)
self.blocks.append([
NumpyBlock(function=create_global_variable, outputs=0),
{'in_1': 0}])
self.blocks.append([
NumpyBlock(function=np.copy),
{'in_1': 0, 'out_1': 1}])
Pipeline(self.blocks).main()
open('.log.txt', 'w').close()
np.testing.assert_almost_equal(self.global_variable, [1, 2, 3, 4])
self.expected_result = [1, 2, 3, 4]
def test_non_linear_multi_copy(self):
"""Test which reads in a sigproc file, and
loads it between different rings in a
nonlinear fashion, then outputs to file."""
logfile = '.log.txt'
self.blocks.append((CopyBlock(), [0], [1]))
self.blocks.append((CopyBlock(), [0], [2]))
self.blocks.append((CopyBlock(), [2], [5]))
self.blocks.append((CopyBlock(), [0], [3]))
self.blocks.append((CopyBlock(), [3], [4]))
self.blocks.append((CopyBlock(), [5], [6]))
self.blocks.append((WriteAsciiBlock(logfile), [6], []))
Pipeline(self.blocks).main()
log_nums = open(logfile, 'r').read(500).split(' ')
test_num = np.float(log_nums[8])
self.assertEqual(test_num, 3)
def test_header_output(self):
"""Output a header for a ring explicitly"""
def generate_array_and_header():
"""Output the desired header of an array"""
header = {'dtype': 'complex128', 'nbit': 128}
yield np.array([1, 2, 3, 4]), header
def assert_expectation(array):
"Assert that the array has a complex datatype"
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.assertEqual(array.dtype, np.dtype('complex128'))
self.occurences += 1
blocks = []
blocks.append((
NumpySourceBlock(generate_array_and_header, grab_headers=True),
{'out_1': 0}))
blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 1)
def test_multiple_sequences(self):
"""Try to send multiple sequences through a branching pipeline"""
def generate_different_arrays():
"""Yield four different groups of two arrays"""
dtypes = ['float32', 'float64', 'complex64', 'int8']
shapes = [(4, ), (4, 5), (4, 5, 6), (2, ) * 8]
for array_index in range(4):
yield np.ones(shape=shapes[array_index],
dtype=dtypes[array_index])
yield 2 * np.ones(shape=shapes[array_index],
dtype=dtypes[array_index])
def switch_types(array):
"""Return two copies of the array, one with a different type"""
return np.copy(array), np.copy(array).astype(np.complex128)
self.occurences = 0
def compare_arrays(array1, array2):
"""Make sure that all arrays coming in are equal"""
self.occurences += 1
np.testing.assert_almost_equal(array1, array2)
blocks = [(NumpySourceBlock(generate_different_arrays), {
'out_1': 0
}),
(NumpyBlock(switch_types, outputs=2), {
'in_1': 0,
'out_1': 1,
'out_2': 2
}), (NumpyBlock(np.fft.fft), {
'in_1': 2,
'out_1': 3
}), (NumpyBlock(np.fft.ifft), {
'in_1': 3,
'out_1': 4
}),
(NumpyBlock(compare_arrays, inputs=2, outputs=0), {
'in_1': 1,
'in_2': 4
})]
Pipeline(blocks).main()
self.assertEqual(self.occurences, 8)
def test_add_block(self):
"""Try some syntax on an addition block."""
my_ring = Ring()
blocks = []
blocks.append([TestingBlock([1, 2]), [], [0]])
blocks.append([TestingBlock([1, 6]), [], [1]])
blocks.append([TestingBlock([9, 2]), [], [2]])
blocks.append([TestingBlock([6, 2]), [], [3]])
blocks.append([TestingBlock([1, 2]), [], [4]])
blocks.append(
[MultiAddBlock(), {
'in_1': 0,
'in_2': 1,
'out_sum': 'first_sum'
}])
blocks.append(
[MultiAddBlock(), {
'in_1': 2,
'in_2': 3,
'out_sum': 'second_sum'
}])
blocks.append([
MultiAddBlock(), {
'in_1': 'first_sum',
'in_2': 'second_sum',
'out_sum': 'third_sum'
}
])
blocks.append([
MultiAddBlock(), {
'in_1': 'third_sum',
'in_2': 4,
'out_sum': my_ring
}
])
def assert_result_of_addition(array):
"""Make sure that the above arrays add up to what we expect"""
np.testing.assert_almost_equal(array, [18, 14])
blocks.append((NumpyBlock(assert_result_of_addition, outputs=0), {
'in_1': my_ring
}))
Pipeline(blocks).main()
def test_multiple_output_rings(self):
"""Multiple output ring test."""
def generate_many_arrays():
"""Put out 10x10 numpy arrays"""
for _ in range(10):
yield (np.array([1, 2, 3, 4]).astype(np.float32),) * 10
def assert_expectation(*args):
"""Assert the arrays are as expected"""
assert len(args) == 10
for array in args:
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.occurences += 1
blocks = []
blocks.append((
NumpySourceBlock(generate_many_arrays, outputs=10),
{'out_%d' % (i + 1): i for i in range(10)}))
blocks.append((
NumpyBlock(assert_expectation, inputs=10, outputs=0),
{'in_%d' % (i + 1): i for i in range(10)}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 10)
def test_output_change(self):
"""Change the output of the source, and expect new sequence"""
self.occurences = 0
def generate_different_arrays():
"""Yield two different arrays"""
yield np.array([1, 2])
yield np.array([1, 2, 3])
def assert_change(array):
"""Assert the input arrays change"""
if self.occurences == 0:
np.testing.assert_almost_equal(array, [1, 2])
else:
np.testing.assert_almost_equal(array, [1, 2, 3])
self.occurences += 1
blocks = [
(NumpySourceBlock(generate_different_arrays), {'out_1': 0}),
(NumpyBlock(np.copy), {'in_1': 0, 'out_1': 1}),
(NumpyBlock(assert_change, outputs=0), {'in_1': 1})]
Pipeline(blocks).main()
self.assertEqual(self.occurences, 2)
def test_simple_dump(self):
"""Input some numbers, and ensure they are written to a file"""
self.blocks.append((TestingBlock([1, 2, 3]), [], [0]))
Pipeline(self.blocks).main()
dumped_numbers = np.loadtxt('.log.txt')
np.testing.assert_almost_equal(dumped_numbers, [1, 2, 3])
def test_throughput(self):
"""Test that any data is being put through"""
Pipeline(self.blocks).main()
test_string = open(self.logfile, 'r').read()
self.assertGreater(len(test_string), 0)
