Esempio n. 1
0
def _assert_compatible(cube, other):
    """
    Checks to see if cube.data and another array can be broadcast to
    the same shape using ``numpy.broadcast_arrays``.

    """
    # This code previously returned broadcasted versions of the cube
    # data and the other array. As numpy.broadcast_arrays does not work
    # with masked arrays (it returns them as ndarrays) operations
    # involving masked arrays would be broken.

    try:
        if not isinstance(other, biggus.Array):
            data_view, other_view = BA.broadcast_arrays(cube._my_data,
                                                        np.asarray(other))

        else:
            data_view, other_view = BA.broadcast_arrays(cube._my_data, other)
    except ValueError as err:
        # re-raise
        raise ValueError("The array was not broadcastable to the cube's data "
                         "shape. The error message from numpy when "
                         "broadcasting:\n{}\nThe cube's shape was {} and the "
                         "array's shape was {}".format(err, cube.shape,
                                                       other.shape))

    if cube.shape != data_view.shape:
        raise ValueError("The array operation would increase the "
                         "dimensionality of the cube. The new cube's data "
                         "would have had to become: {}".format(
                             data_view.shape))
Esempio n. 2
0
def _assert_compatible(cube, other):
    """
    Checks to see if cube.data and another array can be broadcast to
    the same shape using ``numpy.broadcast_arrays``.

    """
    # This code previously returned broadcasted versions of the cube
    # data and the other array. As numpy.broadcast_arrays does not work
    # with masked arrays (it returns them as ndarrays) operations
    # involving masked arrays would be broken.

    try:
        if not isinstance(other, biggus.Array):
            data_view, other_view = BA.broadcast_arrays(
                cube._my_data, np.asarray(other))

        else:
            data_view, other_view = BA.broadcast_arrays(cube._my_data, other)
    except ValueError as err:
        # re-raise
        raise ValueError("The array was not broadcastable to the cube's data "
                         "shape. The error message from numpy when "
                         "broadcasting:\n{}\nThe cube's shape was {} and the "
                         "array's shape was {}".format(err, cube.shape,
                                                       other.shape))

    if cube.shape != data_view.shape:
        raise ValueError("The array operation would increase the "
                         "dimensionality of the cube. The new cube's data "
                         "would have had to become: {}".format(
                             data_view.shape))
Esempio n. 3
0
 def test_nested_broadcast_avoidance(self):
     orig = np.empty([1, 3, 1, 5, 1])
     a = BroadcastArray(orig, {0: 10, 4: 100}, (3, 2, 1))
     b = BroadcastArray(a, {0: 5, 2: 15}, (4,))
     self.assertIs(b.array, orig)
     self.assertEqual(a._broadcast_dict, {0: 10, 4: 100})
     self.assertEqual(b._broadcast_dict, {0: 5, 2: 15, 4: 100})
     self.assertEqual(b._leading_shape, (4, 3, 2, 1))
     self.assertEqual(b.shape, (4, 3, 2, 1, 5, 3, 15, 5, 100))
Esempio n. 4
0
def _binary_op_common(operation_function,
                      operation_name,
                      cube,
                      other,
                      new_unit,
                      dim=None,
                      in_place=False):
    """
    Function which shares common code between binary operations.

    operation_function   - function which does the operation
                           (e.g. numpy.divide)
    operation_name       - the public name of the operation (e.g. 'divide')
    cube                 - the cube whose data is used as the first argument
                           to `operation_function`
    other                - the cube, coord, ndarray or number whose data is
                           used as the second argument
    new_unit             - unit for the resulting quantity
    dim                  - dimension along which to apply `other` if it's a
                           coordinate that is not found in `cube`
    in_place             - whether or not to apply the operation in place to
                           `cube` and `cube.data`
    """
    _assert_is_cube(cube)

    if isinstance(other, iris.coords.Coord):
        other = _broadcast_cube_coord_data(cube, other, operation_name, dim)
    elif isinstance(other, iris.cube.Cube):
        try:
            BA.broadcast_arrays(cube._my_data, other._my_data)
        except ValueError:
            other = iris.util.as_compatible_shape(other, cube)._my_data
        else:
            other = other._my_data

    # don't worry about checking for other data types (such as scalars or
    # np.ndarrays) because _assert_compatible validates that they are broadcast
    # compatible with cube.data
    _assert_compatible(cube, other)

    def unary_func(x):
        ret = operation_function(x, other)
        if ret is NotImplemented:
            # explicitly raise the TypeError, so it gets raised even if, for
            # example, `iris.analysis.maths.multiply(cube, other)` is called
            # directly instead of `cube * other`
            raise TypeError('cannot %s %r and %r objects' %
                            (operation_function.__name__, type(x).__name__,
                             type(other).__name__))
        return ret

    return _math_op_common(cube, unary_func, new_unit, in_place)
Esempio n. 5
0
    def assertBroadcast(self, s1, s2, expected_shape, broadcast_kwargs):
        # Assert that the operations are symmetric.
        r1 = BroadcastArray._compute_broadcast_kwargs(s1, s2)
        r2 = BroadcastArray._compute_broadcast_kwargs(s2, s1)

        self.assertEqual(r1[0], expected_shape)
        self.assertEqual(r1[0], r2[0])
        self.assertEqual(r1[1], r2[2])
        self.assertEqual(r1[2], r2[1])

        self.assertEqual(dict(list(zip(["broadcast", "leading_shape"], broadcast_kwargs[0]))), r1[1])
        self.assertEqual(dict(list(zip(["broadcast", "leading_shape"], broadcast_kwargs[1]))), r1[2])

        actual = np.broadcast(np.empty(s1), np.empty(s2)).shape
        self.assertEqual(expected_shape, actual)
Esempio n. 6
0
 def test_nothing_done(self):
     orig = np.empty([1, 3, 1, 5, 1])
     a = BroadcastArray(orig, {0: 10, 2: 0, 4: 15})
     result = a[...]
     self.assertIsInstance(result, BroadcastArray)
     self.assertIs(result.array, orig)
     self.assertEqual(result.shape, (10, 3, 0, 5, 15))
Esempio n. 7
0
 def test_indexed(self):
     orig = np.ma.masked_array([[1], [2], [3]], mask=[[1], [0], [1]])
     a = BroadcastArray(orig, {1: 2})
     result = a[0:2, :-1].masked_array()
     expected = np.ma.masked_array([[1], [2]], mask=[[1], [0]])
     assert_array_equal(result.mask, expected.mask)
     assert_array_equal(result.data, expected.data)
Esempio n. 8
0
 def test_indexed(self):
     orig = np.empty([1, 3, 1, 5, 1], dtype='>i4')
     a = BroadcastArray(orig, {0: 10, 2: 0, 4: 15}, (2,))
     result = a[:, :, -1, ::2, ::2]
     expected = as_strided(orig, shape=(2, 10, 0, 3, 15),
                           strides=(0, 0, 0, 4, 0))
     assert_array_equal(result.ndarray(), expected)
Esempio n. 9
0
 def test_lh_broadcast(self):
     a = np.empty([2])
     b = np.empty([1, 2])
     a1, b1 = BroadcastArray.broadcast_arrays(a, b)
     self.assertIs(b1, b)
     self.assertIsInstance(a1, BroadcastArray)
     self.assertEqual(a1.shape, (1, 2))
Esempio n. 10
0
 def test_lh_broadcast(self):
     a = np.empty([2])
     b = np.empty([1, 2])
     a1, b1 = BroadcastArray.broadcast_arrays(a, b)
     self.assertIs(b1, b)
     self.assertIsInstance(a1, BroadcastArray)
     self.assertEqual(a1.shape, (1, 2))
Esempio n. 11
0
 def test_broadcast_with_leading(self):
     a = np.arange(3, dtype='>i4').reshape([3, 1])
     result = BroadcastArray._broadcast_numpy_array(a, {1: 4}, (1,))
     expected = np.array([[[0, 0, 0, 0],
                           [1, 1, 1, 1],
                           [2, 2, 2, 2]]])
     self.assertEqual(result.strides, (0, 4, 0))
     assert_array_equal(result, expected)
Esempio n. 12
0
 def test_simple_broadcast(self):
     a = np.arange(3, dtype='>i4').reshape([3, 1])
     result = BroadcastArray._broadcast_numpy_array(a, {1: 4})
     expected = np.array([[0, 0, 0, 0],
                          [1, 1, 1, 1],
                          [2, 2, 2, 2]])
     assert_array_equal(result, expected)
     self.assertEqual(result.strides, (4, 0))
Esempio n. 13
0
def _binary_op_common(operation_function, operation_name, cube, other,
                      new_unit, dim=None, in_place=False):
    """
    Function which shares common code between binary operations.

    operation_function   - function which does the operation
                           (e.g. numpy.divide)
    operation_name       - the public name of the operation (e.g. 'divide')
    cube                 - the cube whose data is used as the first argument
                           to `operation_function`
    other                - the cube, coord, ndarray or number whose data is
                           used as the second argument
    new_unit             - unit for the resulting quantity
    dim                  - dimension along which to apply `other` if it's a
                           coordinate that is not found in `cube`
    in_place             - whether or not to apply the operation in place to
                           `cube` and `cube.data`
    """
    _assert_is_cube(cube)

    if isinstance(other, iris.coords.Coord):
        other = _broadcast_cube_coord_data(cube, other, operation_name, dim)
    elif isinstance(other, iris.cube.Cube):
        try:
            BA.broadcast_arrays(cube._my_data, other._my_data)
        except ValueError:
            other = iris.util.as_compatible_shape(other, cube)._my_data
        else:
            other = other._my_data

    # don't worry about checking for other data types (such as scalars or
    # np.ndarrays) because _assert_compatible validates that they are broadcast
    # compatible with cube.data
    _assert_compatible(cube, other)

    def unary_func(x):
        ret = operation_function(x, other)
        if ret is NotImplemented:
            # explicitly raise the TypeError, so it gets raised even if, for
            # example, `iris.analysis.maths.multiply(cube, other)` is called
            # directly instead of `cube * other`
            raise TypeError('cannot %s %r and %r objects' %
                            (operation_function.__name__, type(x).__name__,
                             type(other).__name__))
        return ret
    return _math_op_common(cube, unary_func, new_unit, in_place)
Esempio n. 14
0
 def test_simple(self):
     orig = np.ma.masked_array([[1], [2], [3]], mask=[[1], [0], [1]])
     result = BroadcastArray(orig, {1: 2}, (2, )).masked_array()
     expected, _ = np.broadcast_arrays(orig.data, result)
     expected_mask, _ = np.broadcast_arrays(orig.mask, result)
     expected = np.ma.masked_array(expected, mask=expected_mask)
     assert_array_equal(result.mask, expected.mask)
     assert_array_equal(result.data, expected.data)
Esempio n. 15
0
 def test_broadcast_shape(self):
     a = BroadcastArray(np.empty([1, 3, 1, 5, 1]), {
         0: 10,
         2: 0,
         4: 15
     },
                        leading_shape=(3, 4))
     self.assertEqual(a.shape, (3, 4, 10, 3, 0, 5, 15))
Esempio n. 16
0
 def test_have_enough_memory(self):
     # Using np.broadcast results in the actual data needing to be
     # realised. The code gets around this by using strides = 0.
     # Pick an array size which isn't realistic to realise in a
     # full array.
     a = ConstantArray([int(10 ** i) for i in range(4, 12)])
     self.assertEqual(
         BroadcastArray._compute_broadcast_kwargs(a.shape, a.shape)[0], tuple(int(10 ** i) for i in range(4, 12))
     )
Esempio n. 17
0
 def test_have_enough_memory(self):
     # Using np.broadcast results in the actual data needing to be
     # realised. The code gets around this by using strides = 0.
     # Pick an array size which isn't realistic to realise in a
     # full array.
     a = ConstantArray([int(10 ** i) for i in range(4, 12)])
     self.assertEqual(BroadcastArray._compute_broadcast_kwargs(a.shape,
                                                               a.shape)[0],
                      tuple(int(10 ** i) for i in range(4, 12)),
                      )
Esempio n. 18
0
    def assertBroadcast(self, s1, s2, expected_shape, broadcast_kwargs):
        # Assert that the operations are symmetric.
        r1 = BroadcastArray._compute_broadcast_kwargs(s1, s2)
        r2 = BroadcastArray._compute_broadcast_kwargs(s2, s1)

        self.assertEqual(r1[0], expected_shape)
        self.assertEqual(r1[0], r2[0])
        self.assertEqual(r1[1], r2[2])
        self.assertEqual(r1[2], r2[1])

        self.assertEqual(dict(list(zip(['broadcast', 'leading_shape'],
                                       broadcast_kwargs[0]))),
                         r1[1])
        self.assertEqual(dict(list(zip(['broadcast', 'leading_shape'],
                                       broadcast_kwargs[1]))),
                         r1[2])

        actual = np.broadcast(np.empty(s1), np.empty(s2)).shape
        self.assertEqual(expected_shape, actual)
Esempio n. 19
0
 def test_scalar_index_of_broadcast_dimension(self):
     a = BroadcastArray(np.empty((1, 36, 1, 1)), {0: 855, 2: 82, 3: 130})
     self.assertEqual(a[:, :, 30].shape, (855, 36, 130))
Esempio n. 20
0
 def test_rule3_value_error(self):
     msg = "operands could not be broadcast together with shapes " "\(2\,3\) \(1\,2\,4\)"
     with six.assertRaisesRegex(self, ValueError, msg):
         BroadcastArray._compute_broadcast_kwargs([2, 3], [1, 2, 4])
Esempio n. 21
0
 def test_ndarray_indexing(self):
     a = BroadcastArray(np.empty((1, 2)), {0: 5})
     with self.assertRaises(NotImplementedError):
         self.assertEqual(a[np.array([1, 3, 4])].shape, (3, 2))
Esempio n. 22
0
 def test_leading_shape_preserve(self):
     orig = np.empty([5, 1])
     a = BroadcastArray(orig, {1: 10}, leading_shape=(3, 4, 2))
     self.assertEqual(a[:, -1].shape, (3, 2, 5, 10))
Esempio n. 23
0
 def test_index_contained_array_dimension(self):
     orig = np.empty([1, 3, 1, 5, 1])
     a = BroadcastArray(orig, {0: 10, 2: 0, 4: 15})
     result = a[:, -1]
     assert_array_equal(result.array, orig[:, -1])
     self.assertEqual(result.shape, (10, 0, 5, 15))
Esempio n. 24
0
 def test_array(self):
     orig = np.empty([1, 3, 5])
     a = BroadcastArray(orig, {})
     self.assertIs(a.array, orig)
Esempio n. 25
0
 def test_invalid_leading_shape(self):
     msg = 'Leading shape must all be >=1'
     with self.assertRaisesRegexp(ValueError, msg):
         BroadcastArray(np.empty([1]), {}, (-1,))
Esempio n. 26
0
 def test_leading_shape(self):
     a = BroadcastArray(np.empty([1, 3]), {0: 2}, (5, 3))
     self.assertEqual(a._shape, (5, 3, 2, 3))
Esempio n. 27
0
 def test_broadcasting_existing_non_len1_dimension(self):
     msg = 'Attempted to broadcast axis 0 which is of length 3.'
     with self.assertRaisesRegexp(ValueError, msg):
         BroadcastArray(np.empty([3]), {0: 5})
Esempio n. 28
0
 def test_invalid_broadcast_length(self):
     msg = 'Axis length must be positive. Got -1.'
     with self.assertRaisesRegexp(ValueError, msg):
         BroadcastArray(np.empty([1, 3, 1, 5, 1]), {0: -1})
Esempio n. 29
0
 def test_remove_leading_and_broadcast(self):
     a = BroadcastArray(np.empty([1, 2, 1]), {0: 3, 2: 3}, (2, 2))
     self.assertEqual(a[:1, 0, 0, 0, :].shape, (1, 3))
Esempio n. 30
0
 def test_index_exising_broadcast(self):
     orig = np.empty([1, 3, 1, 5, 1])
     a = BroadcastArray(orig, {0: 10, 2: 0, 4: 15})
     result = a[:-1]
     self.assertIs(result.array, orig)
     self.assertEqual(result.shape, (9, 3, 0, 5, 15))
Esempio n. 31
0
 def test_invalid_broadcast_axis(self):
     msg = 'Axis -1 out of range \[0, 5\)'
     with self.assertRaisesRegexp(ValueError, msg):
         BroadcastArray(np.empty([1, 3, 1, 5, 1]), {-1: 10})
Esempio n. 32
0
 def test_rule3_value_error(self):
     msg = ('operands could not be broadcast together with shapes '
            '\(2\,3\) \(1\,2\,4\)')
     with self.assertRaisesRegexp(ValueError, msg):
         BroadcastArray._compute_broadcast_kwargs([2, 3], [1, 2, 4])