def test_actuals_mismatch_fail(self):
elements = [("a", np.array([1, 2, 4]))]
actual_values = [("b", np.array([7, 3, 9]))]
with self.assertRaisesRegex(ValueError, "Names.* do not match.*"):
shape, primaries, elems_and_dims = optimal_array_structure(
elements, actual_values
)
def test_1d(self):
elements = [('a', np.array([1, 2, 4]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3,))
self.assertEqual(primaries, set('a'))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([1, 2, 4]), (0,))})
def test_1d(self):
elements = [('a', np.array([1, 2, 4]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3,))
self.assertEqual(primaries, set('a'))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([1, 2, 4]), (0,))})
def test_one(self):
# A single value does not make a dimension (no length-1 dims).
elements = [('a', np.array([1]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, ())
self.assertEqual(primaries, set())
self.assertEqual(elems_and_dims, {})
def test_one(self):
# A single value does not make a dimension (no length-1 dims).
elements = [('a', np.array([1]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, ())
self.assertEqual(primaries, set())
self.assertEqual(elems_and_dims, {})
def test_degenerate(self):
# A all-same vector does not appear in the output.
elements = [("a", np.array([1, 2, 3])), ("b", np.array([4, 4, 4]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3, ))
self.assertEqual(primaries, set(["a"]))
self._check_arrays_and_dims(elems_and_dims,
{"a": (np.array([1, 2, 3]), (0, ))})
def test_2d(self):
elements = [('a', np.array([2, 2, 2, 3, 3, 3])),
('b', np.array([7, 8, 9, 7, 8, 9]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (2, 3,))
self.assertEqual(primaries, set(['a', 'b']))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([2, 3]), (0,)),
'b': (np.array([7, 8, 9]), (1,))})
def test_degenerate(self):
# A all-same vector does not appear in the output.
elements = [('a', np.array([1, 2, 3])),
('b', np.array([4, 4, 4]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3,))
self.assertEqual(primaries, set(['a']))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([1, 2, 3]), (0,))})
def test_2d(self):
elements = [('a', np.array([2, 2, 2, 3, 3, 3])),
('b', np.array([7, 8, 9, 7, 8, 9]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (2, 3,))
self.assertEqual(primaries, set(['a', 'b']))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([2, 3]), (0,)),
'b': (np.array([7, 8, 9]), (1,))})
def test_non_2d(self):
# An incomplete 2d expansion just becomes 1d
elements = [('a', np.array([2, 2, 2, 3, 3])),
('b', np.array([7, 8, 9, 7, 8]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (5,))
self.assertEqual(primaries, set())
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([2, 2, 2, 3, 3]), (0,)),
'b': (np.array([7, 8, 9, 7, 8]), (0,))})
def test_1d_duplicates(self):
# When two have the same structure, the first is 'the dimension'.
elements = [('a', np.array([1, 3, 4])),
('b', np.array([6, 7, 9]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3,))
self.assertEqual(primaries, set('a'))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([1, 3, 4]), (0,)),
'b': (np.array([6, 7, 9]), (0,))})
def test_1d_duplicates(self):
# When two have the same structure, the first is 'the dimension'.
elements = [('a', np.array([1, 3, 4])),
('b', np.array([6, 7, 9]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3,))
self.assertEqual(primaries, set('a'))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([1, 3, 4]), (0,)),
'b': (np.array([6, 7, 9]), (0,))})
def test_non_2d(self):
# An incomplete 2d expansion just becomes 1d
elements = [('a', np.array([2, 2, 2, 3, 3])),
('b', np.array([7, 8, 9, 7, 8]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (5,))
self.assertEqual(primaries, set())
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([2, 2, 2, 3, 3]), (0,)),
'b': (np.array([7, 8, 9, 7, 8]), (0,))})
def test_1d_actuals(self):
# Test use of alternate element values for array construction.
elements = [('a', np.array([1, 2, 4]))]
actual_values = [('a', np.array([7, 3, 9]))]
shape, primaries, elems_and_dims = optimal_array_structure(
elements, actual_values)
self.assertEqual(shape, (3,))
self.assertEqual(primaries, set('a'))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([7, 3, 9]), (0,))})
def test_1d_actuals(self):
# Test use of alternate element values for array construction.
elements = [('a', np.array([1, 2, 4]))]
actual_values = [('a', np.array([7, 3, 9]))]
shape, primaries, elems_and_dims = optimal_array_structure(
elements, actual_values)
self.assertEqual(shape, (3,))
self.assertEqual(primaries, set('a'))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([7, 3, 9]), (0,))})
def _calculate_structure(self):
# Make value arrays for the vectorisable field elements.
element_definitions = self._field_vector_element_arrays()
# Identify the vertical elements and payload.
blev_array = dict(element_definitions).get("blev")
vertical_elements = (
"lblev",
"bhlev",
"bhrlev",
"brsvd1",
"brsvd2",
"brlev",
)
# Make an ordering copy.
ordering_definitions = element_definitions[:]
# Replace time value tuples with integers and bind the vertical
# elements to the (expected) primary vertical element "blev".
for index, (name, array) in enumerate(ordering_definitions):
if name in ("t1", "t2"):
array = np.array(
[self._time_comparable_int(*tuple(val)) for val in array])
ordering_definitions[index] = (name, array)
if name in vertical_elements and blev_array is not None:
ordering_definitions[index] = (name, blev_array)
# Perform the main analysis: get vector dimensions, elements, arrays.
(
dims_shape,
primary_elements,
vector_element_arrays_and_dims,
) = optimal_array_structure(ordering_definitions, element_definitions)
# Replace time tuples in the result with real datetime-like values.
# N.B. so we *don't* do this on the whole (expanded) input arrays.
for name in ("t1", "t2"):
if name in vector_element_arrays_and_dims:
arr, dims = vector_element_arrays_and_dims[name]
arr_shape = arr.shape[:-1]
extra_length = arr.shape[-1]
# Flatten out the array apart from the last dimension,
# convert to cftime objects, then reshape back.
arr = np.array([
cftime.datetime(*args)
for args in arr.reshape(-1, extra_length)
]).reshape(arr_shape)
vector_element_arrays_and_dims[name] = (arr, dims)
# Write the private cache values, exposed as public properties.
self._vector_dims_shape = dims_shape
self._primary_dimension_elements = primary_elements
self._element_arrays_and_dims = vector_element_arrays_and_dims
# Do all this only once.
self._structure_calculated = True
def test_1d_duplicates_order(self):
# Same as previous but reverse passed order of elements 'a' and 'b'.
elements = [('b', np.array([6, 7, 9])),
('a', np.array([1, 3, 4]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3,))
# The only difference is the one chosen as 'principal'
self.assertEqual(primaries, set('b'))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([1, 3, 4]), (0,)),
'b': (np.array([6, 7, 9]), (0,))})
def test_3_way(self):
elements = [('t1', np.array([2, 3, 4])),
('t2', np.array([4, 5, 6])),
('period', np.array([9, 8, 7]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3,))
self.assertEqual(primaries, set(['t1']))
self._check_arrays_and_dims(elems_and_dims,
{'t1': (np.array([2, 3, 4]), (0,)),
't2': (np.array([4, 5, 6]), (0,)),
'period': (np.array([9, 8, 7]), (0,))})
def test_3_way(self):
elements = [('t1', np.array([2, 3, 4])),
('t2', np.array([4, 5, 6])),
('period', np.array([9, 8, 7]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3,))
self.assertEqual(primaries, set(['t1']))
self._check_arrays_and_dims(elems_and_dims,
{'t1': (np.array([2, 3, 4]), (0,)),
't2': (np.array([4, 5, 6]), (0,)),
'period': (np.array([9, 8, 7]), (0,))})
def test_1d_duplicates_order(self):
# Same as previous but reverse passed order of elements 'a' and 'b'.
elements = [('b', np.array([6, 7, 9])),
('a', np.array([1, 3, 4]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3,))
# The only difference is the one chosen as 'principal'
self.assertEqual(primaries, set('b'))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([1, 3, 4]), (0,)),
'b': (np.array([6, 7, 9]), (0,))})
def test_2d(self):
elements = [
("a", np.array([2, 2, 2, 3, 3, 3])),
("b", np.array([7, 8, 9, 7, 8, 9])),
]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (2, 3,))
self.assertEqual(primaries, set(["a", "b"]))
self._check_arrays_and_dims(
elems_and_dims,
{"a": (np.array([2, 3]), (0,)), "b": (np.array([7, 8, 9]), (1,))},
)
def test_mixed_dims(self):
elements = [('t1', np.array([1, 1, 11, 11])),
('t2', np.array([15, 16, 25, 26])),
('ft', np.array([15, 16, 15, 16]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (2, 2))
self.assertEqual(primaries, set(['t1', 'ft']))
self._check_arrays_and_dims(
elems_and_dims,
{'t1': (np.array([1, 11]), (0,)),
't2': (np.array([[15, 16], [25, 26]]), (0, 1)),
'ft': (np.array([15, 16]), (1,))})
def test_mixed_dims(self):
elements = [('t1', np.array([1, 1, 11, 11])),
('t2', np.array([15, 16, 25, 26])),
('ft', np.array([15, 16, 15, 16]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (2, 2))
self.assertEqual(primaries, set(['t1', 'ft']))
self._check_arrays_and_dims(
elems_and_dims,
{'t1': (np.array([1, 11]), (0,)),
't2': (np.array([[15, 16], [25, 26]]), (0, 1)),
'ft': (np.array([15, 16]), (1,))})
def test_2d_with_element_values(self):
# Confirm that elements values are used in the output when supplied.
elements = [('a', np.array([2, 2, 2, 3, 3, 3])),
('b', np.array([7, 8, 9, 7, 8, 9]))]
elements_values = [('a', np.array([6, 6, 6, 8, 8, 8])),
('b', np.array([3, 4, 5, 3, 4, 5]))]
shape, primaries, elems_and_dims = \
optimal_array_structure(elements, elements_values)
self.assertEqual(shape, (2, 3,))
self.assertEqual(primaries, set(['a', 'b']))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([6, 8]), (0,)),
'b': (np.array([3, 4, 5]), (1,))})
def test_missing_dim(self):
# Case with no dimension element for dimension 1.
elements = [('t1', np.array([1, 1, 11, 11])),
('t2', np.array([15, 16, 25, 26]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (4,))
# The potential 2d nature can not be recognised.
# 't1' is auxiliary, as it has duplicate values over the dimension.
self.assertEqual(primaries, set(['t2']))
self._check_arrays_and_dims(
elems_and_dims,
{'t1': (np.array([1, 1, 11, 11]), (0,)),
't2': (np.array([15, 16, 25, 26]), (0,))})
def test_2d_with_element_values(self):
# Confirm that elements values are used in the output when supplied.
elements = [('a', np.array([2, 2, 2, 3, 3, 3])),
('b', np.array([7, 8, 9, 7, 8, 9]))]
elements_values = [('a', np.array([6, 6, 6, 8, 8, 8])),
('b', np.array([3, 4, 5, 3, 4, 5]))]
shape, primaries, elems_and_dims = \
optimal_array_structure(elements, elements_values)
self.assertEqual(shape, (2, 3,))
self.assertEqual(primaries, set(['a', 'b']))
self._check_arrays_and_dims(elems_and_dims,
{'a': (np.array([6, 8]), (0,)),
'b': (np.array([3, 4, 5]), (1,))})
def test_missing_dim(self):
# Case with no dimension element for dimension 1.
elements = [('t1', np.array([1, 1, 11, 11])),
('t2', np.array([15, 16, 25, 26]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (4,))
# The potential 2d nature can not be recognised.
# 't1' is auxiliary, as it has duplicate values over the dimension.
self.assertEqual(primaries, set(['t2']))
self._check_arrays_and_dims(
elems_and_dims,
{'t1': (np.array([1, 1, 11, 11]), (0,)),
't2': (np.array([15, 16, 25, 26]), (0,))})
def test_optimal_structure_decision(self):
# Checks the optimal structure decision logic is working correctly:
# given the arrays we have here we would expect 'a' to be the primary
# dimension, as it has higher priority for being supplied first.
elements = [('a', np.array([1, 1, 1, 2, 2, 2])),
('b', np.array([0, 1, 2, 0, 1, 2])),
('c', np.array([11, 11, 11, 14, 14, 14])),
('d', np.array([10, 10, 10, 10, 10, 10]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (2, 3))
self.assertEqual(primaries, set(['a', 'b']))
self._check_arrays_and_dims(
elems_and_dims,
{'a': (np.array([1, 2]), (0,)),
'c': (np.array([11, 14]), (0,)),
'b': (np.array([0, 1, 2]), (1,))})
def test_optimal_structure_decision(self):
# Checks the optimal structure decision logic is working correctly:
# given the arrays we have here we would expect 'a' to be the primary
# dimension, as it has higher priority for being supplied first.
elements = [('a', np.array([1, 1, 1, 2, 2, 2])),
('b', np.array([0, 1, 2, 0, 1, 2])),
('c', np.array([11, 11, 11, 14, 14, 14])),
('d', np.array([10, 10, 10, 10, 10, 10]))]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (2, 3))
self.assertEqual(primaries, set(['a', 'b']))
self._check_arrays_and_dims(
elems_and_dims,
{'a': (np.array([1, 2]), (0,)),
'c': (np.array([11, 14]), (0,)),
'b': (np.array([0, 1, 2]), (1,))})
def test_mixed_dims(self):
elements = [
("t1", np.array([1, 1, 11, 11])),
("t2", np.array([15, 16, 25, 26])),
("ft", np.array([15, 16, 15, 16])),
]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (2, 2))
self.assertEqual(primaries, set(["t1", "ft"]))
self._check_arrays_and_dims(
elems_and_dims,
{
"t1": (np.array([1, 11]), (0, )),
"t2": (np.array([[15, 16], [25, 26]]), (0, 1)),
"ft": (np.array([15, 16]), (1, )),
},
)
def test_3_way(self):
elements = [
("t1", np.array([2, 3, 4])),
("t2", np.array([4, 5, 6])),
("period", np.array([9, 8, 7])),
]
shape, primaries, elems_and_dims = optimal_array_structure(elements)
self.assertEqual(shape, (3, ))
self.assertEqual(primaries, set(["t1"]))
self._check_arrays_and_dims(
elems_and_dims,
{
"t1": (np.array([2, 3, 4]), (0, )),
"t2": (np.array([4, 5, 6]), (0, )),
"period": (np.array([9, 8, 7]), (0, )),
},
)
def _calculate_structure(self):
# Make value arrays for the vectorisable field elements.
element_definitions = self._field_vector_element_arrays()
# Identify the vertical elements and payload.
blev_array = dict(element_definitions).get('blev')
vertical_elements = ('lblev', 'bhlev', 'bhrlev',
'brsvd1', 'brsvd2', 'brlev')
# Make an ordering copy.
ordering_definitions = element_definitions[:]
# Replace time value tuples with integers and bind the vertical
# elements to the (expected) primary vertical element "blev".
for index, (name, array) in enumerate(ordering_definitions):
if name in ('t1', 't2'):
array = np.array(
[self._time_comparable_int(*tuple(val)) for val in array])
ordering_definitions[index] = (name, array)
if name in vertical_elements and blev_array is not None:
ordering_definitions[index] = (name, blev_array)
# Perform the main analysis: get vector dimensions, elements, arrays.
dims_shape, primary_elements, vector_element_arrays_and_dims = \
optimal_array_structure(ordering_definitions,
element_definitions)
# Replace time tuples in the result with real datetime-like values.
# N.B. so we *don't* do this on the whole (expanded) input arrays.
for name in ('t1', 't2'):
if name in vector_element_arrays_and_dims:
arr, dims = vector_element_arrays_and_dims[name]
arr_shape = arr.shape[:-1]
extra_length = arr.shape[-1]
# Flatten out the array apart from the last dimension,
# convert to cftime objects, then reshape back.
arr = np.array([cftime.datetime(*args)
for args in arr.reshape(-1, extra_length)]
).reshape(arr_shape)
vector_element_arrays_and_dims[name] = (arr, dims)
# Write the private cache values, exposed as public properties.
self._vector_dims_shape = dims_shape
self._primary_dimension_elements = primary_elements
self._element_arrays_and_dims = vector_element_arrays_and_dims
# Do all this only once.
self._structure_calculated = True
def test_none(self):
with self.assertRaises(IndexError):
result = optimal_array_structure([], [])
def test_none(self):
with self.assertRaises(IndexError):
result = optimal_array_structure([], [])
def test_actuals_mismatch_fail(self):
elements = [('a', np.array([1, 2, 4]))]
actual_values = [('b', np.array([7, 3, 9]))]
with self.assertRaisesRegexp(ValueError, 'Names.* do not match.*'):
shape, primaries, elems_and_dims = optimal_array_structure(
elements, actual_values)
