def test_bitmap__invalid_indicator(self): values = np.arange(12) message = _make_test_message({3: self._section_3, 6: {'bitMapIndicator': 100, 'bitmap': None}, 7: {'codedValues': values}}) with self.assertRaisesRegex(TranslationError, 'unsupported bitmap'): as_concrete_data(message.data)
def test_bitmap__invalid_indicator(self): values = np.arange(12) message = _make_test_message({3: self._section_3, 6: {'bitMapIndicator': 100, 'bitmap': None}, 7: {'codedValues': values}}) with self.assertRaisesRegexp(TranslationError, 'unsupported bitmap'): as_concrete_data(message.data, nans_replacement=ma.masked)
def test_bitmap__shapes_mismatch(self): # Test the behaviour where bitmap and codedValues shapes do not match. # Too many or too few unmasked values in codedValues will cause this. values = np.arange(6) message = _make_test_message({3: self._section_3, 6: {'bitMapIndicator': 0, 'bitmap': self.bitmap}, 7: {'codedValues': values}}) with self.assertRaisesRegexp(TranslationError, 'do not match'): as_concrete_data(message.data, nans_replacement=ma.masked)
def test_bitmap__shapes_mismatch(self): # Test the behaviour where bitmap and codedValues shapes do not match. # Too many or too few unmasked values in codedValues will cause this. values = np.arange(6) message = _make_test_message({3: self._section_3, 6: {'bitMapIndicator': 0, 'bitmap': self.bitmap}, 7: {'codedValues': values}}) with self.assertRaisesRegex(TranslationError, 'do not match'): as_concrete_data(message.data)
def test_lazy_data(self): data = np.arange(24).reshape((2, 12)) lazy_array = as_lazy_data(data) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertArrayEqual(result, data)
def test_mdtol_default(self): # Default operation is "mdtol=1" --> unmasked if *any* valid points. # --> output column masks = [0, 0, 0, 1] agg = MEAN.lazy_aggregate(self.array, axis=self.axis) masked_result = as_concrete_data(agg) self.assertMaskedArrayAlmostEqual(masked_result, self.expected_masked)
def test_mdtol_intermediate(self): # mdtol=0.5 --> masked columns = [0, 0, 1, 1] agg = MEAN.lazy_aggregate(self.array, axis=self.axis, mdtol=0.5) masked_result = as_concrete_data(agg) expected_masked = self.expected_masked expected_masked.mask = [False, False, True, True] self.assertMaskedArrayAlmostEqual(masked_result, expected_masked)
def test_mdtol_default(self): # Default operation is "mdtol=1" --> unmasked if *any* valid points. # --> output column masks = [0, 0, 0, 1] agg = MEAN.lazy_aggregate(self.array, axis=self.axis) masked_result = as_concrete_data(agg) self.assertMaskedArrayAlmostEqual(masked_result, self.expected_masked)
def test_mdtol_intermediate(self): # mdtol=0.5 --> masked columns = [0, 0, 1, 1] agg = MEAN.lazy_aggregate(self.array, axis=self.axis, mdtol=0.5) masked_result = as_concrete_data(agg) expected_masked = self.expected_masked expected_masked.mask = [False, False, True, True] self.assertMaskedArrayAlmostEqual(masked_result, expected_masked)
def data(self): """ Returns the real data. Any lazy data being managed will be realised. Returns: :class:`~numpy.ndarray` or :class:`numpy.ma.core.MaskedArray`. """ if self.has_lazy_data(): try: # Realise the lazy data. result = as_concrete_data(self._lazy_array) # Assign the realised result. self._real_array = result # Reset the lazy data and the realised dtype. self._lazy_array = None except MemoryError: emsg = ("Failed to realise the lazy data as there was not " "enough memory available.\n" "The data shape would have been {!r} with {!r}.\n " "Consider freeing up variables or indexing the data " "before trying again.") raise MemoryError(emsg.format(self.shape, self.dtype)) # Check the manager contract, as the managed data has changed. self._assert_axioms() return self._real_array
def test_lazy_data(self): data = np.arange(24).reshape((2, 12)) lazy_array = as_lazy_data(data) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertArrayEqual(result, data)
def data(self): """ Returns the real data. Any lazy data being managed will be realised. Returns: :class:`~numpy.ndarray` or :class:`numpy.ma.core.MaskedArray`. """ if self.has_lazy_data(): try: # Realise the lazy data. result = as_concrete_data(self._lazy_array, nans_replacement=ma.masked, result_dtype=self.dtype) # Assign the realised result. self._real_array = result # Reset the lazy data and the realised dtype. self._lazy_array = None self._realised_dtype = None except MemoryError: emsg = ('Failed to realise the lazy data as there was not ' 'enough memory available.\n' 'The data shape would have been {!r} with {!r}.\n ' 'Consider freeing up variables or indexing the data ' 'before trying again.') raise MemoryError(emsg.format(self.shape, self.dtype)) if ma.isMaskedArray(self._real_array): # Align the numpy fill-value with the data manager fill-value. self._real_array.fill_value = self.fill_value # Check the manager contract, as the managed data has changed. self._assert_axioms() return self._real_array
def test_mdtol_aboveall(self): # mdtol=0.75 --> masked columns = [0, 0, 0, 1] # In this case, effectively the same as mdtol=None. agg = MEAN.lazy_aggregate(self.array, axis=self.axis, mdtol=0.75) masked_result = as_concrete_data(agg) self.assertMaskedArrayAlmostEqual(masked_result, self.expected_masked)
def test_mdtol_aboveall(self): # mdtol=0.75 --> masked columns = [0, 0, 0, 1] # In this case, effectively the same as mdtol=None. agg = MEAN.lazy_aggregate(self.array, axis=self.axis, mdtol=0.75) masked_result = as_concrete_data(agg) self.assertMaskedArrayAlmostEqual(masked_result, self.expected_masked)
def data(self): """ Returns the real data. Any lazy data being managed will be realised. Returns: :class:`~numpy.ndarray` or :class:`numpy.ma.core.MaskedArray`. """ if self.has_lazy_data(): try: # Realise the lazy data. result = as_concrete_data(self._lazy_array) # Assign the realised result. self._real_array = result # Reset the lazy data and the realised dtype. self._lazy_array = None except MemoryError: emsg = ('Failed to realise the lazy data as there was not ' 'enough memory available.\n' 'The data shape would have been {!r} with {!r}.\n ' 'Consider freeing up variables or indexing the data ' 'before trying again.') raise MemoryError(emsg.format(self.shape, self.dtype)) # Check the manager contract, as the managed data has changed. self._assert_axioms() return self._real_array
def test_mdtol_below(self): agg = MEAN.lazy_aggregate(self.array, axis=self.axis, mdtol=0.3) masked_result = as_concrete_data(agg, nans_replacement=ma.masked) expected_masked = self.expected_masked expected_masked.mask = [False, True, True, True] self.assertMaskedArrayAlmostEqual(masked_result, expected_masked)
def test_multi_axis(self): data = np.arange(24.0).reshape((2, 3, 4)) collapse_axes = (0, 2) lazy_data = as_lazy_data(data) agg = MEAN.lazy_aggregate(lazy_data, axis=collapse_axes) result = as_concrete_data(agg, nans_replacement=ma.masked) expected = np.mean(data, axis=collapse_axes) self.assertArrayAllClose(result, expected)
def test_lazy_scalar_proxy_masked(self): a = np.ma.masked_array(5, True) proxy = MyProxy(a) lazy_array = as_lazy_data(proxy) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertMaskedArrayEqual(result, a)
def test_lazy_scalar_proxy(self): a = np.array(5) proxy = MyProxy(a) lazy_array = as_lazy_data(proxy) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertEqual(result, a)
def test_lazy_scalar_proxy_masked(self): a = np.ma.masked_array(5, True) proxy = MyProxy(a) lazy_array = as_lazy_data(proxy) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertMaskedArrayEqual(result, a)
def test_lazy_scalar_proxy(self): a = np.array(5) proxy = MyProxy(a) lazy_array = as_lazy_data(proxy) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertEqual(result, a)
def test_multi_axis(self): data = np.arange(24.0).reshape((2, 3, 4)) collapse_axes = (0, 2) lazy_data = as_lazy_data(data) agg = MEAN.lazy_aggregate(lazy_data, axis=collapse_axes) result = as_concrete_data(agg) expected = np.mean(data, axis=collapse_axes) self.assertArrayAllClose(result, expected)
def test_no_bitmap(self): values = np.arange(12) message = _make_test_message({3: self._section_3, 6: SECTION_6_NO_BITMAP, 7: {'codedValues': values}}) result = as_concrete_data(message.data) expected = values.reshape(self.shape) self.assertEqual(result.shape, self.shape) self.assertArrayEqual(result, expected)
def test_no_bitmap(self): values = np.arange(12) message = _make_test_message({3: self._section_3, 6: SECTION_6_NO_BITMAP, 7: {'codedValues': values}}) result = as_concrete_data(message.data) expected = values.reshape(self.shape) self.assertEqual(result.shape, self.shape) self.assertArrayEqual(result, expected)
def test_lazy_mask_data(self): data = np.arange(24).reshape((2, 12)) fill_value = 1234 mask_data = ma.masked_array(data, fill_value=fill_value) lazy_array = as_lazy_data(mask_data) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertMaskedArrayEqual(result, mask_data) self.assertEqual(result.fill_value, fill_value)
def check(self, real_data, keys, expect_result, expect_map): real_data = np.array(real_data) lazy_data = as_lazy_data(real_data, real_data.shape) real_dim_map, real_result = _slice_data_with_keys(real_data, keys) lazy_dim_map, lazy_result = _slice_data_with_keys(lazy_data, keys) lazy_result = as_concrete_data(lazy_result) self.assertArrayEqual(real_result, expect_result) self.assertArrayEqual(lazy_result, expect_result) self.assertEqual(real_dim_map, expect_map) self.assertEqual(lazy_dim_map, expect_map)
def check(self, real_data, keys, expect_result, expect_map): real_data = np.array(real_data) lazy_data = as_lazy_data(real_data, real_data.shape) real_dim_map, real_result = _slice_data_with_keys(real_data, keys) lazy_dim_map, lazy_result = _slice_data_with_keys(lazy_data, keys) lazy_result = as_concrete_data(lazy_result) self.assertArrayEqual(real_result, expect_result) self.assertArrayEqual(lazy_result, expect_result) self.assertEqual(real_dim_map, expect_map) self.assertEqual(lazy_dim_map, expect_map)
def test_mdtol(self): na = np.nan array = np.array([[1., 2., 1., 2.], [1., 2., 3., na], [1., 2., na, na]]) array = as_lazy_data(array) var = STD_DEV.lazy_aggregate(array, axis=1, mdtol=0.3) masked_result = as_concrete_data(var, nans_replacement=np.ma.masked) masked_expected = np.ma.masked_array([0.57735, 1., 0.707107], mask=[0, 0, 1]) self.assertMaskedArrayAlmostEqual(masked_result, masked_expected)
def test_lazy_mask_data(self): data = np.arange(24).reshape((2, 12)) fill_value = 1234 mask_data = ma.masked_array(data, fill_value=fill_value) lazy_array = as_lazy_data(mask_data) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertMaskedArrayEqual(result, mask_data) self.assertEqual(result.fill_value, fill_value)
def _test(self, scanning_mode): message = _make_test_message( {3: self.section_3(scanning_mode), 6: SECTION_6_NO_BITMAP, 7: {'codedValues': np.arange(12)}}) data = message.data self.assertTrue(is_lazy_data(data)) self.assertEqual(data.shape, (3, 4)) self.assertEqual(data.dtype, np.floating) self.assertArrayEqual(as_concrete_data(data), np.arange(12).reshape(3, 4))
def _test(self, scanning_mode): message = _make_test_message( {3: self.section_3(scanning_mode), 6: SECTION_6_NO_BITMAP, 7: {'codedValues': np.arange(12)}}) data = message.data self.assertTrue(is_lazy_data(data)) self.assertEqual(data.shape, (3, 4)) self.assertEqual(data.dtype, np.floating) self.assertArrayEqual(as_concrete_data(data), np.arange(12).reshape(3, 4))
def test_mdtol(self): na = np.nan array = np.array([[1., 2., 1., 2.], [1., 2., 3., na], [1., 2., na, na]]) array = as_lazy_data(array) var = STD_DEV.lazy_aggregate(array, axis=1, mdtol=0.3) masked_result = as_concrete_data(var, nans_replacement=np.ma.masked) masked_expected = np.ma.masked_array([0.57735, 1., 0.707107], mask=[0, 0, 1]) self.assertMaskedArrayAlmostEqual(masked_result, masked_expected)
def test_mdtol(self): na = -999.888 array = np.ma.masked_equal( [[1.0, 2.0, 1.0, 2.0], [1.0, 2.0, 3.0, na], [1.0, 2.0, na, na]], na ) array = as_lazy_data(array) var = STD_DEV.lazy_aggregate(array, axis=1, mdtol=0.3) masked_result = as_concrete_data(var) masked_expected = np.ma.masked_array( [0.57735, 1.0, 0.707107], mask=[0, 0, 1] ) self.assertMaskedArrayAlmostEqual(masked_result, masked_expected)
def test_last_axis(self): # From setUp: # self.data.mask = [[0, 0, 0, 1], # [0, 0, 1, 1], # [0, 1, 1, 1]] # --> fractions of masked-points in ROWS = [1/4, 1/2, 3/4] axis = -1 agg = MEAN.lazy_aggregate(self.array, axis=axis, mdtol=0.51) expected_masked = ma.mean(self.data, axis=-1) expected_masked = np.ma.masked_array(expected_masked, [0, 0, 1]) masked_result = as_concrete_data(agg) self.assertMaskedArrayAlmostEqual(masked_result, expected_masked)
def test_last_axis(self): # From setUp: # self.data.mask = [[0, 0, 0, 1], # [0, 0, 1, 1], # [0, 1, 1, 1]] # --> fractions of masked-points in ROWS = [1/4, 1/2, 3/4] axis = -1 agg = MEAN.lazy_aggregate(self.array, axis=axis, mdtol=0.51) expected_masked = ma.mean(self.data, axis=-1) expected_masked = np.ma.masked_array(expected_masked, [0, 0, 1]) masked_result = as_concrete_data(agg) self.assertMaskedArrayAlmostEqual(masked_result, expected_masked)
def test_bitmap_present(self): # Test the behaviour where bitmap and codedValues shapes # are not equal. input_values = np.arange(5) output_values = np.array([-1, -1, 0, 1, -1, -1, -1, 2, -1, 3, -1, 4]) message = _make_test_message({3: self._section_3, 6: {'bitMapIndicator': 0, 'bitmap': self.bitmap}, 7: {'codedValues': input_values}}) result = as_concrete_data(message.data) expected = ma.masked_array(output_values, np.logical_not(self.bitmap)) expected = expected.reshape(self.shape) self.assertMaskedArrayEqual(result, expected)
def test_bitmap_present(self): # Test the behaviour where bitmap and codedValues shapes # are not equal. input_values = np.arange(5) output_values = np.array([-1, -1, 0, 1, -1, -1, -1, 2, -1, 3, -1, 4]) message = _make_test_message({3: self._section_3, 6: {'bitMapIndicator': 0, 'bitmap': self.bitmap}, 7: {'codedValues': input_values}}) result = as_concrete_data(message.data) expected = ma.masked_array(output_values, np.logical_not(self.bitmap)) expected = expected.reshape(self.shape) self.assertMaskedArrayEqual(result, expected)
def _check(self, stack_shape): vals = np.arange(np.prod(stack_shape)).reshape(stack_shape) stack = np.empty(stack_shape, 'object') # Define the shape of each element in the stack. stack_element_shape = (4, 5) expected = np.empty(stack_shape + stack_element_shape, dtype=int) for index, val in np.ndenumerate(vals): stack[index] = as_lazy_data(val * np.ones(stack_element_shape)) expected[index] = val result = multidim_lazy_stack(stack) self.assertEqual(result.shape, stack_shape + stack_element_shape) self.assertIsInstance(result, da.core.Array) result = as_concrete_data(result) self.assertArrayAllClose(result, expected)
def _check(self, stack_shape): vals = np.arange(np.prod(stack_shape)).reshape(stack_shape) stack = np.empty(stack_shape, 'object') # Define the shape of each element in the stack. stack_element_shape = (4, 5) expected = np.empty(stack_shape + stack_element_shape, dtype=int) for index, val in np.ndenumerate(vals): stack[index] = as_lazy_data(val * np.ones(stack_element_shape)) expected[index] = val result = multidim_lazy_stack(stack) self.assertEqual(result.shape, stack_shape + stack_element_shape) self.assertIsInstance(result, da.core.Array) result = as_concrete_data(result) self.assertArrayAllClose(result, expected)
def test_bitmap_present_float_data(self): # Test the behaviour where bitmap and codedValues shapes # are not equal, and codedValues is float data. data_type = np.float32 input_values = np.arange(5, dtype=np.float32) + 5 output_values = np.array([-1, -1, 5, 6, -1, -1, -1, 7, -1, 8, -1, 9], dtype=data_type) message = _make_test_message({3: self._section_3, 6: {'bitMapIndicator': 0, 'bitmap': self.bitmap}, 7: {'codedValues': input_values}}) result = as_concrete_data(message.data, nans_replacement=ma.masked, result_dtype=data_type) expected = ma.masked_array(output_values, np.logical_not(self.bitmap)) expected = expected.reshape(self.shape) self.assertMaskedArrayEqual(result, expected)
def test_lazy_data(self): # Minimal testing as as_concrete_data is a wrapper to # convert_nans_array data = np.arange(24).reshape((2, 12)) lazy_array = as_lazy_data(data) sentinel = mock.sentinel.data with mock.patch('iris._lazy_data.convert_nans_array') as conv_nans: conv_nans.return_value = sentinel result = as_concrete_data(lazy_array) self.assertEqual(sentinel, result) # Check call to convert_nans_array self.assertEqual(conv_nans.call_count, 1) args, kwargs = conv_nans.call_args arg, = args self.assertFalse(is_lazy_data(arg)) self.assertArrayEqual(arg, data) self.assertEqual(kwargs, {})
def test_bitmap_present_float_data(self): # Test the behaviour where bitmap and codedValues shapes # are not equal, and codedValues is float data. data_type = np.float32 input_values = np.arange(5, dtype=np.float32) + 5 output_values = np.array([-1, -1, 5, 6, -1, -1, -1, 7, -1, 8, -1, 9], dtype=data_type) message = _make_test_message({ 3: self._section_3, 6: { 'bitMapIndicator': 0, 'bitmap': self.bitmap }, 7: { 'codedValues': input_values } }) result = as_concrete_data(message.data, nans_replacement=ma.masked, result_dtype=data_type) expected = ma.masked_array(output_values, np.logical_not(self.bitmap)) expected = expected.reshape(self.shape) self.assertMaskedArrayEqual(result, expected)
def test_lazy_data_pass_thru_kwargs(self): # Minimal testing as as_concrete_data is a wrapper to # convert_nans_array data = np.arange(24).reshape((2, 12)) lazy_array = as_lazy_data(data) nans_replacement = 7 result_dtype = np.int16 sentinel = mock.sentinel.data with mock.patch('iris._lazy_data.convert_nans_array') as conv_nans: conv_nans.return_value = sentinel result = as_concrete_data(lazy_array, nans_replacement=nans_replacement, result_dtype=result_dtype) self.assertEqual(sentinel, result) # Check call to convert_nans_array self.assertEqual(conv_nans.call_count, 1) args, kwargs = conv_nans.call_args arg, = args self.assertFalse(is_lazy_data(arg)) self.assertArrayEqual(arg, data) self.assertEqual(kwargs, {'nans_replacement': nans_replacement, 'result_dtype': result_dtype, })
def patched(cf_var, filename): return as_concrete_data(pre_patched(cf_var, filename))
def test_mdtol_above(self): agg = MEAN.lazy_aggregate(self.array, axis=self.axis, mdtol=0.4) masked_result = as_concrete_data(agg, nans_replacement=ma.masked) self.assertMaskedArrayAlmostEqual(masked_result, self.expected_masked)
def test_concrete_masked_input_data(self): data = ma.masked_array([10, 12, 8, 2], mask=[True, True, False, True]) result = as_concrete_data(data) self.assertIs(data, result) self.assertFalse(is_lazy_data(result))
def _real_data(cube): # Get the concrete data of the cube for performing data values # comparison checks. return as_concrete_data(cube.core_data(), nans_replacement=cube.fill_value, result_dtype=cube.dtype)
def test_concrete_masked_input_data(self): data = ma.masked_array([10, 12, 8, 2], mask=[True, True, False, True]) result = as_concrete_data(data) self.assertIs(data, result) self.assertFalse(is_lazy_data(result))
def test_all_axes_abovetol(self): agg = MEAN.lazy_aggregate(self.array, axis=None, mdtol=0.45) expected_masked = ma.masked_less([0.0], 1) masked_result = as_concrete_data(agg) self.assertMaskedArrayAlmostEqual(masked_result, expected_masked)
def test_ddof_zero(self): array = as_lazy_data(np.arange(8)) var = STD_DEV.lazy_aggregate(array, axis=0, ddof=0) result = as_concrete_data(var) self.assertArrayAlmostEqual(result, np.array(2.291287))
def _real_data(cube): # Get the concrete data of the cube for performing data values # comparison checks. return as_concrete_data(cube.core_data())
def test_ddof_zero(self): array = as_lazy_data(np.arange(8)) var = STD_DEV.lazy_aggregate(array, axis=0, ddof=0) result = as_concrete_data(var) self.assertArrayAlmostEqual(result, np.array(2.291287))
def _real_data(cube): # Get the concrete data of the cube for performing data values # comparison checks. return as_concrete_data(cube.core_data())
def test_mdtol_below(self): agg = MEAN.lazy_aggregate(self.array, axis=self.axis, mdtol=0.3) masked_result = as_concrete_data(agg, nans_replacement=ma.masked) expected_masked = self.expected_masked expected_masked.mask = [False, True, True, True] self.assertMaskedArrayAlmostEqual(masked_result, expected_masked)
def test_concrete_input_data(self): data = np.arange(24).reshape((4, 6)) result = as_concrete_data(data) self.assertIs(data, result) self.assertFalse(is_lazy_data(result))
def test_concrete_input_data(self): data = np.arange(24).reshape((4, 6)) result = as_concrete_data(data) self.assertIs(data, result) self.assertFalse(is_lazy_data(result))
def test_all_axes_belowtol(self): agg = MEAN.lazy_aggregate(self.array, axis=None, mdtol=0.75) expected_masked = ma.mean(self.data) masked_result = as_concrete_data(agg) self.assertMaskedArrayAlmostEqual(masked_result, expected_masked)
def test_mdtol_above(self): agg = MEAN.lazy_aggregate(self.array, axis=self.axis, mdtol=0.4) masked_result = as_concrete_data(agg, nans_replacement=ma.masked) self.assertMaskedArrayAlmostEqual(masked_result, self.expected_masked)
def test_all_axes_belowtol(self): agg = MEAN.lazy_aggregate(self.array, axis=None, mdtol=0.75) expected_masked = ma.mean(self.data) masked_result = as_concrete_data(agg) self.assertMaskedArrayAlmostEqual(masked_result, expected_masked)
def test_all_axes_abovetol(self): agg = MEAN.lazy_aggregate(self.array, axis=None, mdtol=0.45) expected_masked = ma.masked_less([0.0], 1) masked_result = as_concrete_data(agg) self.assertMaskedArrayAlmostEqual(masked_result, expected_masked)