def test_multiple_sequences(self):
"""Try to send multiple sequences through a branching pipeline"""
def generate_different_arrays():
"""Yield four different groups of two arrays"""
dtypes = ['float32', 'float64', 'complex64', 'int8']
shapes = [(4,), (4, 5), (4, 5, 6), (2,) * 8]
for array_index in range(4):
yield np.ones(
shape=shapes[array_index],
dtype=dtypes[array_index])
yield 2 * np.ones(
shape=shapes[array_index],
dtype=dtypes[array_index])
def switch_types(array):
"""Return two copies of the array, one with a different type"""
return np.copy(array), np.copy(array).astype(np.complex128)
self.occurences = 0
def compare_arrays(array1, array2):
"""Make sure that all arrays coming in are equal"""
self.occurences += 1
np.testing.assert_almost_equal(array1, array2)
blocks = [
(NumpySourceBlock(generate_different_arrays), {'out_1': 0}),
(NumpyBlock(switch_types, outputs=2), {'in_1': 0, 'out_1': 1, 'out_2': 2}),
(NumpyBlock(np.fft.fft), {'in_1': 2, 'out_1': 3}),
(NumpyBlock(np.fft.ifft), {'in_1': 3, 'out_1': 4}),
(NumpyBlock(compare_arrays, inputs=2, outputs=0), {'in_1': 1, 'in_2': 4})]
Pipeline(blocks).main()
self.assertEqual(self.occurences, 8)
def test_multi_header_output(self):
"""Output multiple arrays and headers to fill up rings"""
def generate_array_and_header():
"""Output the desired header of an array"""
header_1 = {'dtype': 'complex128', 'nbit': 128}
header_2 = {'dtype': 'complex64', 'nbit': 64}
yield (
np.array([1, 2, 3, 4]), header_1,
np.array([1, 2]), header_2)
def assert_expectation(array1, array2):
"Assert that the arrays have different complex datatypes"
np.testing.assert_almost_equal(array1, [1, 2, 3, 4])
np.testing.assert_almost_equal(array2, [1, 2])
self.assertEqual(array1.dtype, np.dtype('complex128'))
self.assertEqual(array2.dtype, np.dtype('complex64'))
self.occurences += 1
blocks = []
blocks.append((
NumpySourceBlock(generate_array_and_header, outputs=2, grab_headers=True),
{'out_1': 0, 'out_2': 1}))
blocks.append((
NumpyBlock(assert_expectation, inputs=2, outputs=0),
{'in_1': 0, 'in_2': 1}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 1)
def test_different_types(self):
"""Try to output different type arrays"""
def generate_different_type_arrays():
"""Put out arrays of different types"""
arrays = []
for array_type in ['float32', 'float64', 'int8', 'uint8']:
numpy_type = np.dtype(array_type).type
arrays.append(np.array([1, 2, 3, 4]).astype(numpy_type))
arrays.append(np.array([1 + 10j]))
yield arrays
def assert_expectation(*args):
"""Assert the arrays are as expected"""
self.occurences += 1
self.assertEqual(len(args), 5)
for index, array_type in enumerate(['float32', 'float64', 'int8', 'uint8']):
self.assertTrue(str(args[index].dtype) == array_type)
np.testing.assert_almost_equal(args[index], [1, 2, 3, 4])
np.testing.assert_almost_equal(args[-1], np.array([1 + 10j]))
blocks = []
blocks.append((
NumpySourceBlock(generate_different_type_arrays, outputs=5),
{'out_%d' % (i + 1): i for i in range(5)}))
blocks.append((
NumpyBlock(assert_expectation, inputs=5, outputs=0),
{'in_%d' % (i + 1): i for i in range(5)}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 1)
def test_output_change(self):
"""Change the output of the source, and expect new sequence"""
self.occurences = 0
def generate_different_arrays():
"""Yield two different arrays"""
yield np.array([1, 2])
yield np.array([1, 2, 3])
def assert_change(array):
"""Assert the input arrays change"""
if self.occurences == 0:
np.testing.assert_almost_equal(array, [1, 2])
else:
np.testing.assert_almost_equal(array, [1, 2, 3])
self.occurences += 1
blocks = [(NumpySourceBlock(generate_different_arrays), {
'out_1': 0
}), (NumpyBlock(np.copy), {
'in_1': 0,
'out_1': 1
}), (NumpyBlock(assert_change, outputs=0), {
'in_1': 1
})]
Pipeline(blocks).main()
self.assertEqual(self.occurences, 2)
def test_two_sequences(self):
"""Make sure multiple sequences only triggered for different headers"""
np.random.seed(44)
def generate_two_different_arrays():
"""Generate 10 of an array shape, then 10 of a different array shape"""
for _ in range(10):
yield np.random.rand(4)
for _ in range(10):
yield np.random.rand(5)
self.triggered = False
self.monitor_block = None
self.sequence_id = ""
self.i = 0
#This array holds all of the
#starting numbers. If there
#are more than two different
#numbers, then there is a problem
self.all_sequence_starts = []
def monitor_block_sequences(array):
"""Read the newest sequence, and append the first
byte to the all_sequence_starts"""
#Avoid reading an empty sequence
if self.i > 1 and self.i < 11:
with self.monitor_block.rings['out_1'].open_latest_sequence(
guarantee=False) as curr_seq:
span_gen = curr_seq.read(1)
self.all_sequence_starts.append(int(
next(span_gen).data[0]))
if self.i > 12:
with self.monitor_block.rings['out_1'].open_latest_sequence(
guarantee=False) as curr_seq:
span_gen = curr_seq.read(1)
self.all_sequence_starts.append(int(
next(span_gen).data[0]))
self.i += 1
return array
self.monitor_block = NumpyBlock(monitor_block_sequences)
blocks = [(NumpySourceBlock(generate_two_different_arrays), {
'out_1': 0
}), (self.monitor_block, {
'in_1': 0,
'out_1': 1
})]
Pipeline(blocks).main()
unique_starts = len(set(self.all_sequence_starts))
self.assertEqual(unique_starts, 2)
def test_simple_single_generation(self):
"""For single yields, should act like a TestingBlock"""
def generate_one_array():
"""Put out a single numpy array"""
yield np.array([1, 2, 3, 4]).astype(np.float32)
def assert_expectation(array):
"""Assert the array is as expected"""
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.occurences += 1
blocks = []
blocks.append((NumpySourceBlock(generate_one_array), {'out_1': 0}))
blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 1)
def test_multiple_yields(self):
"""Should be able to repeat generation of an array"""
def generate_10_arrays():
"""Put out 10 numpy arrays"""
for _ in range(10):
yield np.array([1, 2, 3, 4]).astype(np.float32)
def assert_expectation(array):
"""Assert the array is as expected"""
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.occurences += 1
blocks = []
blocks.append((NumpySourceBlock(generate_10_arrays), {'out_1': 0}))
blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 10)
def test_header_output(self):
"""Output a header for a ring explicitly"""
def generate_array_and_header():
"""Output the desired header of an array"""
header = {'dtype': 'complex128', 'nbit': 128}
yield np.array([1, 2, 3, 4]), header
def assert_expectation(array):
"Assert that the array has a complex datatype"
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.assertEqual(array.dtype, np.dtype('complex128'))
self.occurences += 1
blocks = []
blocks.append((
NumpySourceBlock(generate_array_and_header, grab_headers=True),
{'out_1': 0}))
blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 1)
def test_multiple_output_rings(self):
"""Multiple output ring test."""
def generate_many_arrays():
"""Put out 10x10 numpy arrays"""
for _ in range(10):
yield (np.array([1, 2, 3, 4]).astype(np.float32),) * 10
def assert_expectation(*args):
"""Assert the arrays are as expected"""
assert len(args) == 10
for array in args:
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.occurences += 1
blocks = []
blocks.append((
NumpySourceBlock(generate_many_arrays, outputs=10),
{'out_%d' % (i + 1): i for i in range(10)}))
blocks.append((
NumpyBlock(assert_expectation, inputs=10, outputs=0),
{'in_%d' % (i + 1): i for i in range(10)}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 10)