def test_10_input_10_output(self):
"""Test that 10 input and 10 output rings work"""
def dstack_handler(*args):
"""Stack all input arrays"""
return np.dstack(tuple(args))
def identity(*args):
"""Return all arrays passed"""
return args
number_rings = 10
connections = {}
for index in range(number_rings):
#Simple 1 to 1 copy block
self.blocks.append([
NumpyBlock(function=np.copy),
{'in_1': 0, 'out_1': index + 2}])
connections['in_' + str(index + 1)] = index + 2
connections['out_' + str(index + 1)] = index + 2 + number_rings
#Copy all inputs to all outputs
self.blocks.append([
NumpyBlock(function=identity, inputs=number_rings, outputs=number_rings),
dict(connections)])
second_connections = {}
for key in connections:
if key[:3] == 'out':
second_connections['in' + key[3:]] = int(connections[key])
second_connections['out_1'] = 1
#Stack N input rings into 1 output ring
self.blocks.append([
NumpyBlock(function=dstack_handler, inputs=number_rings, outputs=1),
second_connections])
self.expected_result = np.dstack((self.test_array,) * (len(second_connections) - 1)).ravel()
def test_multiple_sequences(self):
"""Try to send multiple sequences through a branching pipeline"""
def generate_different_arrays():
"""Yield four different groups of two arrays"""
dtypes = ['float32', 'float64', 'complex64', 'int8']
shapes = [(4,), (4, 5), (4, 5, 6), (2,) * 8]
for array_index in range(4):
yield np.ones(
shape=shapes[array_index],
dtype=dtypes[array_index])
yield 2 * np.ones(
shape=shapes[array_index],
dtype=dtypes[array_index])
def switch_types(array):
"""Return two copies of the array, one with a different type"""
return np.copy(array), np.copy(array).astype(np.complex128)
self.occurences = 0
def compare_arrays(array1, array2):
"""Make sure that all arrays coming in are equal"""
self.occurences += 1
np.testing.assert_almost_equal(array1, array2)
blocks = [
(NumpySourceBlock(generate_different_arrays), {'out_1': 0}),
(NumpyBlock(switch_types, outputs=2), {'in_1': 0, 'out_1': 1, 'out_2': 2}),
(NumpyBlock(np.fft.fft), {'in_1': 2, 'out_1': 3}),
(NumpyBlock(np.fft.ifft), {'in_1': 3, 'out_1': 4}),
(NumpyBlock(compare_arrays, inputs=2, outputs=0), {'in_1': 1, 'in_2': 4})]
Pipeline(blocks).main()
self.assertEqual(self.occurences, 8)
def test_output_change(self):
"""Change the output of the source, and expect new sequence"""
self.occurences = 0
def generate_different_arrays():
"""Yield two different arrays"""
yield np.array([1, 2])
yield np.array([1, 2, 3])
def assert_change(array):
"""Assert the input arrays change"""
if self.occurences == 0:
np.testing.assert_almost_equal(array, [1, 2])
else:
np.testing.assert_almost_equal(array, [1, 2, 3])
self.occurences += 1
blocks = [(NumpySourceBlock(generate_different_arrays), {
'out_1': 0
}), (NumpyBlock(np.copy), {
'in_1': 0,
'out_1': 1
}), (NumpyBlock(assert_change, outputs=0), {
'in_1': 1
})]
Pipeline(blocks).main()
self.assertEqual(self.occurences, 2)
def test_two_inputs(self):
"""Test that two input rings work"""
def dstack_handler(array_1, array_2):
"""Stack two matrices along a third dimension"""
return np.dstack((array_1, array_2))
self.blocks.append([
NumpyBlock(function=np.copy),
{'in_1': 0, 'out_1': 2}])
self.blocks.append([
NumpyBlock(function=dstack_handler, inputs=2),
{'in_1': 0, 'in_2': 2, 'out_1': 1}])
self.expected_result = np.dstack((self.test_array, self.test_array)).ravel()
def test_zero_outputs(self):
"""Test zero outputs on NumpyBlock. Nothing should be sent through self.function at init"""
def assert_something(array):
"""Assert the array is only 4 numbers, and return nothing"""
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.blocks.append([
NumpyBlock(function=assert_something, outputs=0),
{'in_1': 0}])
self.blocks.append([
NumpyBlock(function=np.copy, outputs=1),
{'in_1': 0, 'out_1': 1}])
self.expected_result = [1, 2, 3, 4]
def test_100_inputs(self):
"""Test that 100 input rings work"""
def dstack_handler(*args):
"""Stack all input arrays"""
return np.dstack(tuple(args))
number_inputs = 100
connections = {'in_1': 0, 'out_1': 1}
for index in range(number_inputs):
self.blocks.append([
NumpyBlock(function=np.copy),
{'in_1': 0, 'out_1': index + 2}])
connections['in_' + str(index + 2)] = index + 2
self.blocks.append([
NumpyBlock(function=dstack_handler, inputs=len(connections) - 1),
connections])
self.expected_result = np.dstack((self.test_array,) * (len(connections) - 1)).ravel()
def test_global_variable_capture(self):
"""Test that we can pull out a number from a ring using NumpyBlock"""
self.global_variable = np.array([])
def create_global_variable(array):
"""Try to append the array to a global variable"""
self.global_variable = np.copy(array)
self.blocks.append([
NumpyBlock(function=create_global_variable, outputs=0),
{'in_1': 0}])
self.blocks.append([
NumpyBlock(function=np.copy),
{'in_1': 0, 'out_1': 1}])
Pipeline(self.blocks).main()
open('.log.txt', 'w').close()
np.testing.assert_almost_equal(self.global_variable, [1, 2, 3, 4])
self.expected_result = [1, 2, 3, 4]
def test_different_types(self):
"""Try to output different type arrays"""
def generate_different_type_arrays():
"""Put out arrays of different types"""
arrays = []
for array_type in ['float32', 'float64', 'int8', 'uint8']:
numpy_type = np.dtype(array_type).type
arrays.append(np.array([1, 2, 3, 4]).astype(numpy_type))
arrays.append(np.array([1 + 10j]))
yield arrays
def assert_expectation(*args):
"""Assert the arrays are as expected"""
self.occurences += 1
self.assertEqual(len(args), 5)
for index, array_type in enumerate(['float32', 'float64', 'int8', 'uint8']):
self.assertTrue(str(args[index].dtype) == array_type)
np.testing.assert_almost_equal(args[index], [1, 2, 3, 4])
np.testing.assert_almost_equal(args[-1], np.array([1 + 10j]))
blocks = []
blocks.append((
NumpySourceBlock(generate_different_type_arrays, outputs=5),
{'out_%d' % (i + 1): i for i in range(5)}))
blocks.append((
NumpyBlock(assert_expectation, inputs=5, outputs=0),
{'in_%d' % (i + 1): i for i in range(5)}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 1)
def test_add_block(self):
"""Try some syntax on an addition block."""
my_ring = Ring()
blocks = []
blocks.append([TestingBlock([1, 2]), [], [0]])
blocks.append([TestingBlock([1, 6]), [], [1]])
blocks.append([TestingBlock([9, 2]), [], [2]])
blocks.append([TestingBlock([6, 2]), [], [3]])
blocks.append([TestingBlock([1, 2]), [], [4]])
blocks.append([
MultiAddBlock(),
{'in_1': 0, 'in_2': 1, 'out_sum': 'first_sum'}])
blocks.append([
MultiAddBlock(),
{'in_1': 2, 'in_2': 3, 'out_sum': 'second_sum'}])
blocks.append([
MultiAddBlock(),
{'in_1': 'first_sum', 'in_2': 'second_sum', 'out_sum': 'third_sum'}])
blocks.append([
MultiAddBlock(),
{'in_1': 'third_sum', 'in_2': 4, 'out_sum': my_ring}])
def assert_result_of_addition(array):
"""Make sure that the above arrays add up to what we expect"""
np.testing.assert_almost_equal(array, [18, 14])
blocks.append((NumpyBlock(assert_result_of_addition, outputs=0), {'in_1': my_ring}))
Pipeline(blocks).main()
def test_multi_header_output(self):
"""Output multiple arrays and headers to fill up rings"""
def generate_array_and_header():
"""Output the desired header of an array"""
header_1 = {'dtype': 'complex128', 'nbit': 128}
header_2 = {'dtype': 'complex64', 'nbit': 64}
yield (
np.array([1, 2, 3, 4]), header_1,
np.array([1, 2]), header_2)
def assert_expectation(array1, array2):
"Assert that the arrays have different complex datatypes"
np.testing.assert_almost_equal(array1, [1, 2, 3, 4])
np.testing.assert_almost_equal(array2, [1, 2])
self.assertEqual(array1.dtype, np.dtype('complex128'))
self.assertEqual(array2.dtype, np.dtype('complex64'))
self.occurences += 1
blocks = []
blocks.append((
NumpySourceBlock(generate_array_and_header, outputs=2, grab_headers=True),
{'out_1': 0, 'out_2': 1}))
blocks.append((
NumpyBlock(assert_expectation, inputs=2, outputs=0),
{'in_1': 0, 'in_2': 1}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 1)
def test_simple_copy(self):
"""Perform a np.copy on a ring"""
self.blocks.append(
[NumpyBlock(function=np.copy), {
'in_1': 0,
'out_1': 1
}])
self.expected_result = [1, 2, 3, 4]
def test_complex_output(self):
"""Test that complex data can be generated"""
self.blocks.append(
[NumpyBlock(function=np.fft.fft), {
'in_1': 0,
'out_1': 1
}])
self.expected_result = np.fft.fft(self.test_array)
def test_two_outputs(self):
"""Test that two output rings work by copying input data to both"""
def double(array):
"""Return two of the inputted matrix"""
return (array, array)
self.blocks.append([
NumpyBlock(function=double, outputs=2),
{'in_1': 0, 'out_1': 2, 'out_2': 1}])
self.expected_result = [1, 2, 3, 4]
def test_boolean_output(self):
"""Convert a ring into boolean output"""
def greater_than_two(array):
"""Return a matrix representing whether each element
is greater than 2"""
return array > 2
self.blocks.append([
NumpyBlock(function=greater_than_two),
{'in_1': 0, 'out_1': 1}])
self.expected_result = [0, 0, 1, 1]
def test_different_size_output(self):
"""Test that the output size can be different"""
def first_half(array):
"""Only return the first half of the input vector"""
array = np.array(array)
return array[:int(array.size / 2)]
self.blocks.append([
NumpyBlock(function=first_half),
{'in_1': 0, 'out_1': 1}])
self.expected_result = first_half(self.test_array)
def test_two_sequences(self):
"""Make sure multiple sequences only triggered for different headers"""
np.random.seed(44)
def generate_two_different_arrays():
"""Generate 10 of an array shape, then 10 of a different array shape"""
for _ in range(10):
yield np.random.rand(4)
for _ in range(10):
yield np.random.rand(5)
self.triggered = False
self.monitor_block = None
self.sequence_id = ""
self.i = 0
#This array holds all of the
#starting numbers. If there
#are more than two different
#numbers, then there is a problem
self.all_sequence_starts = []
def monitor_block_sequences(array):
"""Read the newest sequence, and append the first
byte to the all_sequence_starts"""
#Avoid reading an empty sequence
if self.i > 1 and self.i < 11:
with self.monitor_block.rings['out_1'].open_latest_sequence(
guarantee=False) as curr_seq:
span_gen = curr_seq.read(1)
self.all_sequence_starts.append(int(
next(span_gen).data[0]))
if self.i > 12:
with self.monitor_block.rings['out_1'].open_latest_sequence(
guarantee=False) as curr_seq:
span_gen = curr_seq.read(1)
self.all_sequence_starts.append(int(
next(span_gen).data[0]))
self.i += 1
return array
self.monitor_block = NumpyBlock(monitor_block_sequences)
blocks = [(NumpySourceBlock(generate_two_different_arrays), {
'out_1': 0
}), (self.monitor_block, {
'in_1': 0,
'out_1': 1
})]
Pipeline(blocks).main()
unique_starts = len(set(self.all_sequence_starts))
self.assertEqual(unique_starts, 2)
def test_simple_single_generation(self):
"""For single yields, should act like a TestingBlock"""
def generate_one_array():
"""Put out a single numpy array"""
yield np.array([1, 2, 3, 4]).astype(np.float32)
def assert_expectation(array):
"""Assert the array is as expected"""
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.occurences += 1
blocks = []
blocks.append((NumpySourceBlock(generate_one_array), {'out_1': 0}))
blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 1)
def test_multiple_yields(self):
"""Should be able to repeat generation of an array"""
def generate_10_arrays():
"""Put out 10 numpy arrays"""
for _ in range(10):
yield np.array([1, 2, 3, 4]).astype(np.float32)
def assert_expectation(array):
"""Assert the array is as expected"""
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.occurences += 1
blocks = []
blocks.append((NumpySourceBlock(generate_10_arrays), {'out_1': 0}))
blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 10)
def test_header_output(self):
"""Output a header for a ring explicitly"""
def generate_array_and_header():
"""Output the desired header of an array"""
header = {'dtype': 'complex128', 'nbit': 128}
yield np.array([1, 2, 3, 4]), header
def assert_expectation(array):
"Assert that the array has a complex datatype"
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.assertEqual(array.dtype, np.dtype('complex128'))
self.occurences += 1
blocks = []
blocks.append((
NumpySourceBlock(generate_array_and_header, grab_headers=True),
{'out_1': 0}))
blocks.append((NumpyBlock(assert_expectation, outputs=0), {'in_1': 0}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 1)
def test_multiple_output_rings(self):
"""Multiple output ring test."""
def generate_many_arrays():
"""Put out 10x10 numpy arrays"""
for _ in range(10):
yield (np.array([1, 2, 3, 4]).astype(np.float32),) * 10
def assert_expectation(*args):
"""Assert the arrays are as expected"""
assert len(args) == 10
for array in args:
np.testing.assert_almost_equal(array, [1, 2, 3, 4])
self.occurences += 1
blocks = []
blocks.append((
NumpySourceBlock(generate_many_arrays, outputs=10),
{'out_%d' % (i + 1): i for i in range(10)}))
blocks.append((
NumpyBlock(assert_expectation, inputs=10, outputs=0),
{'in_%d' % (i + 1): i for i in range(10)}))
Pipeline(blocks).main()
self.assertEqual(self.occurences, 10)