def test_build_arrays_unstructured_ndim_c_order(self):
# Passing an unstructured array to build_arrays, should result in the
# appropriately shaped array, plus any trailing dimensions.
grp = GroupStructure(6, {'a': None}, array_order='c')
orig = np.array([1, 2, 3, 4, 5, 6]).reshape(2, 3)
orig = np.dstack([orig, orig + 10])
r = grp.build_arrays((2, 3), {'a': orig.reshape((-1, 2), order='c')})
self.assert_built_array('a', r, (orig, (0, 1)))
def test_build_arrays_unstructured_ndim_c_order(self):
# Passing an unstructured array to build_arrays, should result in the
# appropriately shaped array, plus any trailing dimensions.
grp = GroupStructure(6, {'a': None}, array_order='c')
orig = np.array([1, 2, 3, 4, 5, 6]).reshape(2, 3)
orig = np.dstack([orig, orig + 10])
r = grp.build_arrays((2, 3), {'a': orig.reshape((-1, 2), order='c')})
self.assert_built_array('a', r, (orig, (0, 1)))
def test_simple_3d_structure(self):
# Construct a structure representing a (3, 2, 4) group and assert
# that the result is of the expected form.
array_structures = {'a': ArrayStructure(1, [1, -1, 2]),
'b': ArrayStructure(3, [1, -1]),
'c': ArrayStructure(6, [1, -1, 2, 3])}
structure = GroupStructure(24, array_structures, array_order='f')
expected = ([('a', array_structures['a']),
('b', array_structures['b']),
('c', array_structures['c'])],)
self.assertEqual(structure.possible_structures(), expected)
def test_simple_3d_structure(self):
# Construct a structure representing a (3, 2, 4) group and assert
# that the result is of the expected form.
array_structures = {
'a': ArrayStructure(1, [1, -1, 2]),
'b': ArrayStructure(3, [1, -1]),
'c': ArrayStructure(6, [1, -1, 2, 3])
}
structure = GroupStructure(24, array_structures, array_order='f')
expected = ([('a', array_structures['a']),
('b', array_structures['b']),
('c', array_structures['c'])], )
self.assertEqual(structure.possible_structures(), expected)
def test_simple_3d_structure(self):
# Construct a structure representing a (3, 2, 4) group and assert
# that the result is of the expected form.
array_structures = {
"a": ArrayStructure(1, [1, -1, 2]),
"b": ArrayStructure(3, [1, -1]),
"c": ArrayStructure(6, [1, -1, 2, 3]),
}
structure = GroupStructure(24, array_structures, array_order="f")
expected = ([
("a", array_structures["a"]),
("b", array_structures["b"]),
("c", array_structures["c"]),
], )
self.assertEqual(structure.possible_structures(), expected)
def test_structured_array_not_applicable(self):
# Just because an array has a possible structure, does not mean it
# gets used. Check that 'd' which would make a good 1D array, doesn't
# get used in a specific shape.
elements = regular_array_structures((2, 2, 3))
elements['d'] = ArrayStructure(3, range(4))
grp = GroupStructure(12, elements, array_order='f')
d = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]).reshape((3, 4),
order='f')
expected = np.array([[[0, 1, 2], [0, 2, 3]], [[0, 1, 3], [1, 2, 3]]])
r = grp.build_arrays((2, 2, 3), {'a': np.arange(12),
'b': np.arange(12),
'c': np.arange(12),
'd': d.flatten(order='f')})
self.assert_built_array('d', r, (expected, (0, 1, 2)))
def test_build_arrays_regular_f_order(self):
# Construct simple orthogonal 1d array structures, adding a trailing
# dimension to the second, and assert the result of build_arrays
# produces the required result.
elements = regular_array_structures((2, 3))
a = elements['a'].construct_array(6)
b = elements['b'].construct_array(6)
# Make b 2 dimensional.
b = np.vstack([b, b + 100]).T
grp = GroupStructure(6, elements, array_order='f')
result = grp.build_arrays((2, 3), {'a': a, 'b': b})
self.assert_built_array('a', result, ([0, 1], (0,)))
self.assert_built_array('b', result, ([[0, 100], [1, 101], [2, 102]],
(1,)))
def test_build_arrays_regular_f_order(self):
# Construct simple orthogonal 1d array structures, adding a trailing
# dimension to the second, and assert the result of build_arrays
# produces the required result.
elements = regular_array_structures((2, 3))
a = elements['a'].construct_array(6)
b = elements['b'].construct_array(6)
# Make b 2 dimensional.
b = np.vstack([b, b + 100]).T
grp = GroupStructure(6, elements, array_order='f')
result = grp.build_arrays((2, 3), {'a': a, 'b': b})
self.assert_built_array('a', result, ([0, 1], (0, )))
self.assert_built_array('b', result,
([[0, 100], [1, 101], [2, 102]], (1, )))
def test_structure_creation(self):
# Test that the appropriate dictionary containing ArrayStructures is
# computed when constructing a GroupStructure from_component_arrays.
array = np.arange(6)
expected_structure = {'a': ArrayStructure.from_array(array)}
grp = GroupStructure.from_component_arrays({'a': array})
self.assertEqual(grp.length, 6)
self.assertEqual(grp._cmpt_structure, expected_structure)
def test_structured_array_not_applicable(self):
# Just because an array has a possible structure, does not mean it
# gets used. Check that 'd' which would make a good 1D array, doesn't
# get used in a specific shape.
elements = regular_array_structures((2, 2, 3))
elements['d'] = ArrayStructure(3, np.arange(4))
grp = GroupStructure(12, elements, array_order='f')
d = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]).reshape((3, 4),
order='f')
expected = np.array([[[0, 1, 2], [0, 2, 3]], [[0, 1, 3], [1, 2, 3]]])
r = grp.build_arrays(
(2, 2, 3), {
'a': np.arange(12),
'b': np.arange(12),
'c': np.arange(12),
'd': d.flatten(order='f')
})
self.assert_built_array('d', r, (expected, (0, 1, 2)))
def test_structure_creation(self):
# Test that the appropriate dictionary containing ArrayStructures is
# computed when constructing a GroupStructure from_component_arrays.
array = np.arange(6)
expected_structure = {'a': ArrayStructure.from_array(array)}
grp = GroupStructure.from_component_arrays({'a': array})
self.assertEqual(grp.length, 6)
self.assertEqual(grp._cmpt_structure, expected_structure)
def optimal_array_structure(ordering_elements, actual_values_elements=None):
"""
Calculate an optimal array replication structure for a set of vectors.
Args:
* ordering_elements (iterable of (name, 1-d array)):
Input element names and value-vectors. Must all be the same length
(but not necessarily type). Must have at least one.
Kwargs:
* actual_values_elements (iterable of (name, 1-d array)):
The 'real' values used to construct the result arrays, if different
from 'ordering_elements'. Must contain all the same names (but not
necessarily in the same order).
The 'ordering_elements' arg contains the pattern used to deduce a
structure. The order of this is significant, in that earlier elements get
priority when associating dimensions with specific elements.
Returns:
dims_shape, primary_elements, element_arrays_and_dims, where:
* 'dims_shape' is the shape of the vector dimensions chosen.
* 'primary_elements' is a set of dimension names; the names of input
elements that are identified as dimensions. At most one for each
dimension.
* 'element_arrays_and_dims' is a dictionary [name: (array, dims)],
for all elements that are not dimensionless. Each array is reduced
to the shape of its mapped dimension.
For example::
>>> import iris.fileformats.um._optimal_array_structuring as optdims
>>> elements_structure = [('a', np.array([1, 1, 1, 2, 2, 2])),
... ('b', np.array([0, 1, 2, 0, 1, 2])),
... ('c', np.array([11, 12, 13, 14, 15, 16]))]
>>> elements_values = [('a', np.array([10, 10, 10, 12, 12, 12])),
... ('b', np.array([15, 16, 17, 15, 16, 17])),
... ('c', np.array([9, 3, 5, 2, 7, 1]))]
>>> dims_shape, dim_names, arrays_and_dims = \
... optdims.optimal_array_structure(elements_structure,
... elements_values)
>>> print dims_shape
(2, 3)
>>> print dim_names
set(['a', 'b'])
>>> print arrays_and_dims
{'a': (array([10, 12]), (0,)), 'c': (array([[9, 3, 5],
[2, 7, 1]]), (0, 1)), 'b': (array([15, 16, 17]), (1,))}
"""
# Convert the inputs to dicts.
element_ordering_arrays = dict(ordering_elements)
if actual_values_elements is None:
actual_values_elements = element_ordering_arrays
actual_value_arrays = dict(actual_values_elements)
if set(actual_value_arrays.keys()) != set(element_ordering_arrays.keys()):
msg = 'Names in values arrays do not match those in ordering arrays.'
raise ValueError(msg)
# Define element priorities from ordering, to choose between equally good
# structures, as structure code does not recognise any element ordering.
n_elements = len(ordering_elements)
element_priorities = {
name: n_elements - index
for index, (name, array) in enumerate(ordering_elements)
}
# Calculate the basic fields-group array structure.
base_structure = GroupStructure.from_component_arrays(
element_ordering_arrays)
# Work out the target cube structure.
target_structure = _optimal_dimensioning_structure(base_structure,
element_priorities)
# Work out result cube dimensions.
if not target_structure:
# Get the length of an input array (they are all the same).
# Note that no elements map to multiple dimensions.
elements_length = len(ordering_elements[0][1])
vector_dims_shape = (elements_length, )
else:
vector_dims_shape = tuple(struct.size
for (_, struct) in target_structure)
# Build arrays of element values mapped onto the vectorised dimensions.
elements_and_dimensions = base_structure.build_arrays(
vector_dims_shape, actual_value_arrays)
# Filter out the trivial (scalar) ones.
elements_and_dimensions = {
name: (array, dims)
for name, (array, dims) in elements_and_dimensions.items() if len(dims)
}
# Make a list of 'primary' elements; i.e. those in the target structure.
primary_dimension_elements = set(name for (name, _) in target_structure)
if vector_dims_shape == (1, ):
shape = ()
else:
shape = vector_dims_shape
# Return all the information.
return (shape, primary_dimension_elements, elements_and_dimensions)
def optimal_array_structure(ordering_elements, actual_values_elements=None):
"""
Calculate an optimal array replication structure for a set of vectors.
Args:
* ordering_elements (iterable of (name, 1-d array)):
Input element names and value-vectors. Must all be the same length
(but not necessarily type). Must have at least one.
Kwargs:
* actual_values_elements (iterable of (name, 1-d array)):
The 'real' values used to construct the result arrays, if different
from 'ordering_elements'. Must contain all the same names (but not
necessarily in the same order).
The 'ordering_elements' arg contains the pattern used to deduce a
structure. The order of this is significant, in that earlier elements get
priority when associating dimensions with specific elements.
Returns:
dims_shape, primary_elements, element_arrays_and_dims, where:
* 'dims_shape' is the shape of the vector dimensions chosen.
* 'primary_elements' is a set of dimension names; the names of input
elements that are identified as dimensions. At most one for each
dimension.
* 'element_arrays_and_dims' is a dictionary [name: (array, dims)],
for all elements that are not dimensionless. Each array is reduced
to the shape of its mapped dimension.
For example::
>>> import iris.fileformats.um._optimal_array_structuring as optdims
>>> elements_structure = [('a', np.array([1, 1, 1, 2, 2, 2])),
... ('b', np.array([0, 1, 2, 0, 1, 2])),
... ('c', np.array([11, 12, 13, 14, 15, 16]))]
>>> elements_values = [('a', np.array([10, 10, 10, 12, 12, 12])),
... ('b', np.array([15, 16, 17, 15, 16, 17])),
... ('c', np.array([9, 3, 5, 2, 7, 1]))]
>>> dims_shape, dim_names, arrays_and_dims = \
... optdims.optimal_array_structure(elements_structure,
... elements_values)
>>> print dims_shape
(2, 3)
>>> print dim_names
set(['a', 'b'])
>>> print arrays_and_dims
{'a': (array([10, 12]), (0,)), 'c': (array([[9, 3, 5],
[2, 7, 1]]), (0, 1)), 'b': (array([15, 16, 17]), (1,))}
"""
# Convert the inputs to dicts.
element_ordering_arrays = dict(ordering_elements)
if actual_values_elements is None:
actual_values_elements = element_ordering_arrays
actual_value_arrays = dict(actual_values_elements)
if set(actual_value_arrays.keys()) != set(element_ordering_arrays.keys()):
msg = 'Names in values arrays do not match those in ordering arrays.'
raise ValueError(msg)
# Define element priorities from ordering, to choose between equally good
# structures, as structure code does not recognise any element ordering.
n_elements = len(ordering_elements)
element_priorities = {
name: n_elements - index
for index, (name, array) in enumerate(ordering_elements)}
# Calculate the basic fields-group array structure.
base_structure = GroupStructure.from_component_arrays(
element_ordering_arrays)
# Work out the target cube structure.
target_structure = _optimal_dimensioning_structure(base_structure,
element_priorities)
# Work out result cube dimensions.
if not target_structure:
# Get the length of an input array (they are all the same).
# Note that no elements map to multiple dimensions.
elements_length = len(ordering_elements[0][1])
vector_dims_shape = (elements_length,)
else:
vector_dims_shape = tuple(
struct.size for (_, struct) in target_structure)
# Build arrays of element values mapped onto the vectorised dimensions.
elements_and_dimensions = base_structure.build_arrays(
vector_dims_shape, actual_value_arrays)
# Filter out the trivial (scalar) ones.
elements_and_dimensions = {
name: (array, dims)
for name, (array, dims) in six.iteritems(elements_and_dimensions)
if len(dims)}
# Make a list of 'primary' elements; i.e. those in the target structure.
primary_dimension_elements = set(
name for (name, _) in target_structure)
if vector_dims_shape == (1,):
shape = ()
else:
shape = vector_dims_shape
# Return all the information.
return (shape, primary_dimension_elements, elements_and_dimensions)
def assert_potentials(self, length, array_structures, expected):
structure = GroupStructure(length, array_structures, array_order='f')
allowed = structure.possible_structures()
names = [[name for (name, _) in allowed_structure]
for allowed_structure in allowed]
self.assertEqual(names, expected)
def test_build_arrays_unstructured(self):
# Check that an unstructured array gets reshaped appropriately.
grp = GroupStructure(6, {'a': None}, array_order='c')
orig = np.array([1, 2, 3, 4, 5, 6]).reshape(2, 3)
r = grp.build_arrays((2, 3), {'a': orig.flatten(order='c')})
self.assert_built_array('a', r, (orig, (0, 1)))
def test_different_sizes(self):
arrays = {'a': np.arange(6), 'b': np.arange(5)}
msg = 'All array elements must have the same size.'
with self.assertRaisesRegexp(ValueError, msg):
GroupStructure.from_component_arrays(arrays)
def test_different_sizes(self):
arrays = {'a': np.arange(6), 'b': np.arange(5)}
msg = 'All array elements must have the same size.'
with self.assertRaisesRegexp(ValueError, msg):
GroupStructure.from_component_arrays(arrays)
def test_build_arrays_unstructured(self):
# Check that an unstructured array gets reshaped appropriately.
grp = GroupStructure(6, {'a': None}, array_order='c')
orig = np.array([1, 2, 3, 4, 5, 6]).reshape(2, 3)
r = grp.build_arrays((2, 3), {'a': orig.flatten(order='c')})
self.assert_built_array('a', r, (orig, (0, 1)))
def assert_potentials(self, length, array_structures, expected):
structure = GroupStructure(length, array_structures, array_order='f')
allowed = structure.possible_structures()
names = [[name for (name, _) in allowed_structure]
for allowed_structure in allowed]
self.assertEqual(names, expected)
