def test_meshcoord_leaves_originals_lazy(self): self._make_test_meshcoord(lazy_sources=True) mesh = self.mesh meshcoord = self.meshcoord # Fetch the relevant source objects from the mesh. def fetch_sources_from_mesh(): return ( mesh.coord(include_nodes=True, axis="x"), mesh.coord(include_faces=True, axis="x"), mesh.face_node_connectivity, ) # Check all the source coords are lazy. for coord in fetch_sources_from_mesh(): # Note: not all are actual Coords, so can't use 'has_lazy_points'. self.assertTrue(is_lazy_data(coord._core_values())) # Calculate both points + bounds of the meshcoord self.assertTrue(meshcoord.has_lazy_points()) self.assertTrue(meshcoord.has_lazy_bounds()) meshcoord.points meshcoord.bounds self.assertFalse(meshcoord.has_lazy_points()) self.assertFalse(meshcoord.has_lazy_bounds()) # Check all the source coords are still lazy. for coord in fetch_sources_from_mesh(): # Note: not all are actual Coords, so can't use 'has_lazy_points'. self.assertTrue(is_lazy_data(coord._core_values()))
def test_lazy_data(self): data = np.arange(24).reshape((2, 12)) lazy_array = as_lazy_data(data) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertArrayEqual(result, data)
def test_lazy_scalar_proxy_masked(self): a = np.ma.masked_array(5, True) proxy = MyProxy(a) lazy_array = as_lazy_data(proxy) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertMaskedArrayEqual(result, a)
def test_lazy_scalar_proxy(self): a = np.array(5) proxy = MyProxy(a) lazy_array = as_lazy_data(proxy) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertEqual(result, a)
def test_lazy_mask_data(self): data = np.arange(24).reshape((2, 12)) fill_value = 1234 mask_data = ma.masked_array(data, fill_value=fill_value) lazy_array = as_lazy_data(mask_data) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertMaskedArrayEqual(result, mask_data) self.assertEqual(result.fill_value, fill_value)
def test_lazy_complex(self): raw_points = np.arange(12).reshape(4, 3) points = as_lazy_data(raw_points, raw_points.shape) coord = AuxCoord(points) self.assertTrue(is_lazy_data(coord.core_points())) result = AuxCoordFactory._nd_points(coord, (3, 2), 5) # Check we haven't triggered the loading of the coordinate values. self.assertTrue(is_lazy_data(coord.core_points())) self.assertTrue(is_lazy_data(result)) expected = raw_points.T[np.newaxis, np.newaxis, ..., np.newaxis] self.assertArrayEqual(result, expected)
def test_lazy_complex(self): raw_points = np.arange(12).reshape(4, 3) points = as_lazy_data(raw_points, raw_points.shape) raw_bounds = np.arange(24).reshape(4, 3, 2) bounds = as_lazy_data(raw_bounds, raw_bounds.shape) coord = AuxCoord(points, bounds=bounds) self.assertTrue(is_lazy_data(coord.core_bounds())) result = AuxCoordFactory._nd_bounds(coord, (3, 2), 5) # Check we haven't triggered the loading of the coordinate values. self.assertTrue(is_lazy_data(coord.core_bounds())) self.assertTrue(is_lazy_data(result)) expected = raw_bounds.transpose((1, 0, 2)).reshape(1, 1, 3, 4, 1, 2) self.assertArrayEqual(result, expected)
def test_returned(self): lazy_data, weights = SUM.lazy_aggregate( self.cube_2d.lazy_data(), axis=0, returned=True ) self.assertTrue(is_lazy_data(lazy_data)) self.assertArrayEqual(lazy_data.compute(), [7, 9, 11, 13, 15]) self.assertArrayEqual(weights, [2, 2, 2, 2, 2])
def test_dtype_change(self): concrete_array = np.array([True, False]) lazy_array = as_lazy_data(concrete_array) wrapped = lazy_elementwise(lazy_array, _test_elementwise_op) self.assertTrue(is_lazy_data(wrapped)) self.assertEqual(wrapped.dtype, np.int) self.assertEqual(wrapped.compute().dtype, wrapped.dtype)
def test_load_rotated_xy_land(self): # Test loading single xy rotated pole CF-netCDF file. cube = iris.load_cube(tests.get_data_path( ('NetCDF', 'rotated', 'xy', 'rotPole_landAreaFraction.nc'))) # Make sure the AuxCoords have lazy data. self.assertTrue(is_lazy_data(cube.coord('latitude').core_points())) self.assertCML(cube, ('netcdf', 'netcdf_rotated_xy_land.cml'))
def test_dtype_same(self): concrete_array = np.array([3.0], dtype=np.float16) lazy_array = as_lazy_data(concrete_array) wrapped = lazy_elementwise(lazy_array, _test_elementwise_op) self.assertTrue(is_lazy_data(wrapped)) self.assertEqual(wrapped.dtype, np.float16) self.assertEqual(wrapped.compute().dtype, np.float16)
def test_basic(self): concrete_array = np.arange(30).reshape((2, 5, 3)) lazy_array = as_lazy_data(concrete_array) wrapped = lazy_elementwise(lazy_array, _test_elementwise_op) self.assertTrue(is_lazy_data(wrapped)) self.assertArrayAllClose(wrapped.compute(), _test_elementwise_op(concrete_array))
def test_dtype_same(self): concrete_array = np.array([3.], dtype=np.float16) lazy_array = as_lazy_data(concrete_array) wrapped = lazy_elementwise(lazy_array, _test_elementwise_op) self.assertTrue(is_lazy_data(wrapped)) self.assertEqual(wrapped.dtype, np.float16) self.assertEqual(wrapped.compute().dtype, np.float16)
def test_load_rotated_xy_land(self): # Test loading single xy rotated pole CF-netCDF file. cube = iris.load_cube( tests.get_data_path( ("NetCDF", "rotated", "xy", "rotPole_landAreaFraction.nc"))) # Make sure the AuxCoords have lazy data. self.assertTrue(is_lazy_data(cube.coord("latitude").core_points())) self.assertCML(cube, ("netcdf", "netcdf_rotated_xy_land.cml"))
def create_mock_cube(array): cube = unittest.mock.Mock() cube_data = unittest.mock.PropertyMock(return_value=array) type(cube).data = cube_data cube.dtype = array.dtype cube.has_lazy_data = unittest.mock.Mock(return_value=is_lazy_data(array)) cube.lazy_data = unittest.mock.Mock(return_value=array) cube.shape = array.shape return cube, cube_data
def test_rechunk(self): lazy_array = da.asarray(self.array, chunks=((1, 1), (2, 2))) cube, _ = create_mock_cube(lazy_array) result = map_complete_blocks(cube, self.func, dims=(1, ), out_sizes=(4, )) self.assertTrue(is_lazy_data(result)) self.assertArrayEqual(result.compute(), self.func_result)
def test_multidimensional_input(self): array = np.arange(2 * 3 * 4).reshape(2, 3, 4) lazy_array = da.asarray(array, chunks=((1, 1), (1, 2), (4, ))) cube, _ = create_mock_cube(lazy_array) result = map_complete_blocks(cube, self.func, dims=(1, 2), out_sizes=(3, 4)) self.assertTrue(is_lazy_data(result)) self.assertArrayEqual(result.compute(), array + 1)
def _test(self, scanning_mode): message = _make_test_message( {3: self.section_3(scanning_mode), 6: SECTION_6_NO_BITMAP, 7: {'codedValues': np.arange(12)}}) data = message.data self.assertTrue(is_lazy_data(data)) self.assertEqual(data.shape, (3, 4)) self.assertEqual(data.dtype, np.floating) self.assertArrayEqual(as_concrete_data(data), np.arange(12).reshape(3, 4))
def test_different_out_shape(self): lazy_array = da.asarray(self.array, chunks=((1, 1), (4, ))) cube, _ = create_mock_cube(lazy_array) def func(_): return np.arange(2).reshape(1, 2) func_result = [[0, 1], [0, 1]] result = map_complete_blocks(cube, func, dims=(1, ), out_sizes=(2, )) self.assertTrue(is_lazy_data(result)) self.assertArrayEqual(result.compute(), func_result)
def test_non_lazy_input(self): # Check that a non-lazy input doesn't trip up the functionality. cube, cube_data = create_mock_cube(self.array) result = map_complete_blocks(cube, self.func, dims=(1, ), out_sizes=(4, )) self.assertFalse(is_lazy_data(result)) self.assertArrayEqual(result, self.func_result) # check correct data was accessed cube.lazy_data.assert_not_called() cube_data.assert_called_once()
def test_lazy_input(self): lazy_array = da.asarray(self.array, chunks=((1, 1), (4, ))) cube, cube_data = create_mock_cube(lazy_array) result = map_complete_blocks(cube, self.func, dims=(1, ), out_sizes=(4, )) self.assertTrue(is_lazy_data(result)) self.assertArrayEqual(result.compute(), self.func_result) # check correct data was accessed cube.lazy_data.assert_called_once() cube_data.assert_not_called()
def data(self, data): """ Replaces the currently managed data with the specified data, which must be of an equivalent shape. Note that, the only shape promotion permitted is for 0-dimensional scalar data to be replaced with a single item 1-dimensional data. Args: * data: The :class:`~numpy.ndarray` or :class:`~numpy.ma.core.MaskedArray` real data, or :class:`~dask.array.core.Array` lazy data to be managed. """ # Ensure we have numpy-like data. if not (hasattr(data, 'shape') and hasattr(data, 'dtype')): data = np.asanyarray(data) # Determine whether the class instance has been created, # as this method is called from within the __init__. init_done = (self._lazy_array is not None or self._real_array is not None) if init_done and self.shape != data.shape: # The _ONLY_ data reshape permitted is converting a 0-dimensional # array i.e. self.shape == () into a 1-dimensional array of length # one i.e. data.shape == (1,) if self.shape or data.shape != (1,): emsg = 'Require data with shape {!r}, got {!r}.' raise ValueError(emsg.format(self.shape, data.shape)) # Set lazy or real data, and reset the other. if is_lazy_data(data): self._lazy_array = data self._real_array = None else: if not ma.isMaskedArray(data): # Coerce input data to ndarray (including ndarray subclasses). data = np.asarray(data) if isinstance(data, ma.core.MaskedConstant): # Promote to a masked array so that the fill-value is # writeable to the data owner. data = ma.array(data.data, mask=data.mask, dtype=data.dtype) self._lazy_array = None self._real_array = data # Check the manager contract, as the managed data has changed. self._assert_axioms()
def data(self, data): """ Replaces the currently managed data with the specified data, which must be of an equivalent shape. Note that, the only shape promotion permitted is for 0-dimensional scalar data to be replaced with a single item 1-dimensional data. Args: * data: The :class:`~numpy.ndarray` or :class:`~numpy.ma.core.MaskedArray` real data, or :class:`~dask.array.core.Array` lazy data to be managed. """ # Ensure we have numpy-like data. if not (hasattr(data, 'shape') and hasattr(data, 'dtype')): data = np.asanyarray(data) # Determine whether the class instance has been created, # as this method is called from within the __init__. init_done = (self._lazy_array is not None or self._real_array is not None) if init_done and self.shape != data.shape: # The _ONLY_ data reshape permitted is converting a 0-dimensional # array i.e. self.shape == () into a 1-dimensional array of length # one i.e. data.shape == (1,) if self.shape or data.shape != (1, ): emsg = 'Require data with shape {!r}, got {!r}.' raise ValueError(emsg.format(self.shape, data.shape)) # Set lazy or real data, and reset the other. if is_lazy_data(data): self._lazy_array = data self._real_array = None else: if not ma.isMaskedArray(data): # Coerce input data to ndarray (including ndarray subclasses). data = np.asarray(data) if isinstance(data, ma.core.MaskedConstant): # Promote to a masked array so that the fill-value is # writeable to the data owner. data = ma.array(data.data, mask=data.mask, dtype=data.dtype) self._lazy_array = None self._real_array = data # Check the manager contract, as the managed data has changed. self._assert_axioms()
def test_lazy_data(self): # Minimal testing as as_concrete_data is a wrapper to # convert_nans_array data = np.arange(24).reshape((2, 12)) lazy_array = as_lazy_data(data) sentinel = mock.sentinel.data with mock.patch('iris._lazy_data.convert_nans_array') as conv_nans: conv_nans.return_value = sentinel result = as_concrete_data(lazy_array) self.assertEqual(sentinel, result) # Check call to convert_nans_array self.assertEqual(conv_nans.call_count, 1) args, kwargs = conv_nans.call_args arg, = args self.assertFalse(is_lazy_data(arg)) self.assertArrayEqual(arg, data) self.assertEqual(kwargs, {})
def replace(self, data, fill_value=None, realised_dtype=None): """ Perform an in-place replacement of the managed data. Args: * data: Replace the managed data with either the :class:`~numpy.ndarray` or :class:`~numpy.ma.core.MaskedArray` real data, or lazy :class:`dask.array.core.Array` Kwargs: * fill_value: Replacement for the :class:`~iris._data_manager.DataManager` fill-value. * realised_dtype: The intended dtype of the specified lazy data. .. note:: Data replacement alone will clear the intended dtype of the realised lazy data, and the fill-value. """ # Snapshot the currently managed data. original_data = self.core_data() # Perform in-place data assignment. self.data = data try: self._realised_dtype_setter(realised_dtype) self.fill_value = fill_value except ValueError as error: # Backout the data replacement, and reinstate the cached # original managed data. self._lazy_array = self._real_array = None if is_lazy_data(original_data): self._lazy_array = original_data else: self._real_array = original_data raise error
def test_agg_by_aux_coord(self): problem_test_file = tests.get_data_path(('NetCDF', 'testing', 'small_theta_colpex.nc')) cube = iris.load_cube(problem_test_file, 'air_potential_temperature') # Test aggregating by aux coord, notably the `forecast_period` aux # coord on `cube`, whose `_points` attribute is a lazy array. # This test then ensures that aggregating using `points` instead is # successful. # First confirm we've got a lazy array. # NB. This checks the merge process in `load_cube()` hasn't # triggered the load of the coordinate's data. forecast_period_coord = cube.coord('forecast_period') self.assertTrue(is_lazy_data(forecast_period_coord.core_points())) # Now confirm we can aggregate along this coord. res_cube = cube.aggregated_by('forecast_period', MEAN) res_cell_methods = res_cube.cell_methods[0] self.assertEqual(res_cell_methods.coord_names, ('forecast_period',)) self.assertEqual(res_cell_methods.method, 'mean')
def test_agg_by_aux_coord(self): problem_test_file = tests.get_data_path( ('NetCDF', 'testing', 'small_theta_colpex.nc')) cube = iris.load_cube(problem_test_file, 'air_potential_temperature') # Test aggregating by aux coord, notably the `forecast_period` aux # coord on `cube`, whose `_points` attribute is a lazy array. # This test then ensures that aggregating using `points` instead is # successful. # First confirm we've got a lazy array. # NB. This checks the merge process in `load_cube()` hasn't # triggered the load of the coordinate's data. forecast_period_coord = cube.coord('forecast_period') self.assertTrue(is_lazy_data(forecast_period_coord.core_points())) # Now confirm we can aggregate along this coord. res_cube = cube.aggregated_by('forecast_period', MEAN) res_cell_methods = res_cube.cell_methods[0] self.assertEqual(res_cell_methods.coord_names, ('forecast_period', )) self.assertEqual(res_cell_methods.method, 'mean')
def test_lazy_data_pass_thru_kwargs(self): # Minimal testing as as_concrete_data is a wrapper to # convert_nans_array data = np.arange(24).reshape((2, 12)) lazy_array = as_lazy_data(data) nans_replacement = 7 result_dtype = np.int16 sentinel = mock.sentinel.data with mock.patch('iris._lazy_data.convert_nans_array') as conv_nans: conv_nans.return_value = sentinel result = as_concrete_data(lazy_array, nans_replacement=nans_replacement, result_dtype=result_dtype) self.assertEqual(sentinel, result) # Check call to convert_nans_array self.assertEqual(conv_nans.call_count, 1) args, kwargs = conv_nans.call_args arg, = args self.assertFalse(is_lazy_data(arg)) self.assertArrayEqual(arg, data) self.assertEqual(kwargs, {'nans_replacement': nans_replacement, 'result_dtype': result_dtype, })
def test_concrete_masked_input_data(self): data = ma.masked_array([10, 12, 8, 2], mask=[True, True, False, True]) result = as_concrete_data(data) self.assertIs(data, result) self.assertFalse(is_lazy_data(result))
def test_real(self): real_array = np.arange(24).reshape((2, 3, 4)) self.assertFalse(is_lazy_data(real_array))
def test_lazy(self): values = np.arange(30).reshape((2, 5, 3)) lazy_array = da.from_array(values, chunks=_MAX_CHUNK_SIZE) self.assertTrue(is_lazy_data(lazy_array))
def test_lazy(self): lazy_data = COUNT.lazy_aggregate(self.lazy_cube.lazy_data(), axis=0, function=self.func) self.assertTrue(is_lazy_data(lazy_data))
def test_lazy(self): lazy_data = STD_DEV.lazy_aggregate(self.lazy_cube.lazy_data(), axis=0) self.assertTrue(is_lazy_data(lazy_data))
def test_concrete_input_data(self): data = np.arange(24).reshape((4, 6)) result = as_concrete_data(data) self.assertIs(data, result) self.assertFalse(is_lazy_data(result))
def test_lazy_ma(self): lazy_data = MIN.lazy_aggregate(self.cube.lazy_data(), axis=0) self.assertTrue(is_lazy_data(lazy_data)) self.assertArrayEqual(lazy_data.compute(), [3])
def test_lazy_input(self): result = self.func(self.lazy_array) self.assertFalse(is_lazy_data(result)) self.assertArrayEqual(result, self.func_result)
def lazyness_string(data): # Represent the lazyness of an array as a string. return 'lazy' if is_lazy_data(data) else 'real'
def assertIsLazyArray(self, array, *args, **kwargs): # Check that the arg is a lazy array. self.assertTrue(is_lazy_data(array), *args, **kwargs)
def test_regular_data(self): filename = tests.get_data_path(('GRIB', 'gaussian', 'regular_gg.grib1')) messages = list(_load_generate(filename)) self.assertTrue(is_lazy_data(messages[0]._data))
def test_reduced_data(self): filename = tests.get_data_path(('GRIB', 'reduced', 'reduced_ll.grib1')) messages = list(_load_generate(filename)) self.assertTrue(is_lazy_data(messages[0]._data))
def test_non_lazy_input(self): # Check that a non-lazy input doesn't trip up the functionality. result = self.func(self.array) self.assertFalse(is_lazy_data(result)) self.assertArrayEqual(result, self.func_result)
def test_lazy(self): lazy_data = MIN.lazy_aggregate(self.lazy_cube.lazy_data(), axis=0) self.assertTrue(is_lazy_data(lazy_data))
def test_ma(self): lazy_data = COUNT.lazy_aggregate(self.lazy_cube.lazy_data(), axis=0, function=self.func) self.assertTrue(is_lazy_data(lazy_data)) self.assertArrayEqual(lazy_data.compute(), [2])
def test_lazy_ma(self): lazy_data = MAX.lazy_aggregate(self.cube.lazy_data(), axis=0) self.assertTrue(is_lazy_data(lazy_data)) self.assertArrayEqual(lazy_data.compute(), [3])