def test_lazy_with_lazy__fill_value_failure(self): fill_value = 1234 dm1 = DataManager(as_lazy_data(self.real_array), fill_value=fill_value) dm2 = DataManager(as_lazy_data(self.real_array), fill_value=fill_value) self.assertFalse(dm1 != dm2)
def test_lazy_with_lazy__realised_dtype(self): dtype = np.dtype('int16') dm1 = DataManager(as_lazy_data(self.real_array), realised_dtype=dtype) dm2 = DataManager(as_lazy_data(self.real_array), realised_dtype=dtype) self.assertEqual(dm1, dm2)
def test_lazy_with_lazy(self): dm = DataManager(as_lazy_data(self.real_array)) data = as_lazy_data(self.real_array) * 10 result = dm._deepcopy(self.memo, data=data) expected = DataManager(data) self.assertEqual(result, expected) self.assertIs(result._lazy_array, data)
def test_lazy_with_lazy__realised_dtype_failure(self): dtype = np.dtype('int16') dm1 = DataManager(as_lazy_data(self.real_array), realised_dtype=dtype) dm2 = DataManager(as_lazy_data(self.real_array), realised_dtype=dtype) self.assertFalse(dm1 != dm2)
def test_zero_ndim_lazy_with_zero_ndim_lazy(self): lazy_array = as_lazy_data(np.array(456)) dm = DataManager(as_lazy_data(np.array(123))) self.assertTrue(dm.has_lazy_data()) dm.data = lazy_array self.assertTrue(dm.has_lazy_data()) self.assertArrayEqual(dm.data, lazy_array.compute())
def test_zero_ndim_lazy_to_scalar_1d_lazy_promote(self): lazy_array = as_lazy_data(np.array([456])) dm = DataManager(as_lazy_data(np.array(123))) self.assertTrue(dm.has_lazy_data()) dm.data = lazy_array self.assertTrue(dm.has_lazy_data()) self.assertArrayEqual(dm.data, lazy_array.compute())
def test_lazy_array__fill_value_clearance(self): fill_value = 1234 dm = DataManager(as_lazy_data(np.array(0)), fill_value=fill_value) self.assertEqual(dm.fill_value, fill_value) dm.data = as_lazy_data(np.array(1)) self.assertIsNone(dm.fill_value)
def test_lazy_with_lazy__fill_value(self): fill_value = 1234 dm1 = DataManager(as_lazy_data(self.real_array), fill_value=fill_value) dm2 = DataManager(as_lazy_data(self.real_array), fill_value=fill_value) self.assertEqual(dm1, dm2)
def test_default_chunks_limiting(self): # Check that chunking is limited when no specific 'chunks' given. limitcall_patch = self.patch('iris._lazy_data._limited_shape') test_shape = (3, 2, 4) data = self._dummydata(test_shape) as_lazy_data(data) self.assertEqual(limitcall_patch.call_args_list, [mock.call(test_shape)])
def test_lazy_with_realised_dtype(self): dm = DataManager(as_lazy_data(self.real_array), realised_dtype=np.dtype('int16')) data = as_lazy_data(self.real_array) * 10 dtype = np.dtype('int8') result = dm._deepcopy(self.memo, data=data, realised_dtype=dtype) expected = DataManager(data, realised_dtype=dtype) self.assertEqual(result, expected) self.assertIs(result._lazy_array, data)
def test_1d_lazy_weighted(self): # 1-dimensional input with lazy weights. data = as_lazy_data(np.array([4, 7, 10, 8], dtype=np.float64), chunks=-1) weights = as_lazy_data(np.array([1, 4, 3, 2], dtype=np.float64), chunks=-1) expected_rms = 8.0 # https://github.com/dask/dask/issues/3846. with self.assertRaisesRegexp(TypeError, 'unexpected keyword argument'): rms = RMS.lazy_aggregate(data, 0, weights=weights) self.assertAlmostEqual(rms, expected_rms)
def test_default_chunks_limiting(self): # Check that chunking is still controlled when no specific 'chunks' # is passed. limitcall_patch = self.patch('iris._lazy_data._optimum_chunksize') test_shape = (3, 2, 4) data = self._dummydata(test_shape) as_lazy_data(data) self.assertEqual(limitcall_patch.call_args_list, [mock.call(list(test_shape), shape=test_shape, dtype=np.dtype('f4'))])
def test_1d_lazy_weighted(self): # 1-dimensional input with lazy weights. data = as_lazy_data(np.array([4, 7, 10, 8], dtype=np.float64), chunks=-1) weights = as_lazy_data(np.array([1, 4, 3, 2], dtype=np.float64), chunks=-1) expected_rms = 8.0 # https://github.com/dask/dask/issues/3846. with self.assertRaisesRegexp(TypeError, 'unexpected keyword argument'): rms = RMS.lazy_aggregate(data, 0, weights=weights) self.assertAlmostEqual(rms, expected_rms)
def setUp(self): shape = (2, 3, 4) size = np.prod(shape) self.real_array = np.arange(size).reshape(shape) self.lazy_array = as_lazy_data(self.real_array) self.mask_array = ma.masked_array(self.real_array) self.mask_array_masked = self.mask_array.copy() self.mask_array_masked[0, 0, 0] = ma.masked self.realised_dtype = self.mask_array.dtype self.lazy_mask_array = as_lazy_data(self.mask_array) self.lazy_mask_array_masked = as_lazy_data(self.mask_array_masked)
def setUp(self): shape = (2, 3, 4) size = np.prod(shape) self.real_array = np.arange(size).reshape(shape) self.lazy_array = as_lazy_data(self.real_array) self.mask_array = ma.masked_array(self.real_array) self.mask_array_masked = self.mask_array.copy() self.mask_array_masked[0, 0, 0] = ma.masked self.realised_dtype = self.mask_array.dtype self.lazy_mask_array = as_lazy_data(self.mask_array) self.lazy_mask_array_masked = as_lazy_data(self.mask_array_masked)
def sample_coord( self, datatype=float, dates=False, units="m", long_name="x", shape=(5, ), masked=False, bounded=False, dimcoord=False, lazy_points=False, lazy_bounds=False, *coord_args, **coord_kwargs, ): if masked: dimcoord = False if dates: # Use a pre-programmed date unit. units = Unit("days since 1970-03-5") if not isinstance(units, Unit): # This operation is *not* a no-op, it will wipe calendars ! units = Unit(units) values = self.sample_data(datatype=datatype, units=units, shape=shape, masked=masked) cls = DimCoord if dimcoord else AuxCoord coord = cls( points=values, units=units, long_name=long_name, *coord_args, **coord_kwargs, ) if bounded or lazy_bounds: if shape == (1, ): # Guess-bounds doesn't work ! val = coord.points[0] bounds = [val - 10, val + 10] # NB preserve masked/unmasked : avoid converting masks to NaNs if np.ma.isMaskedArray(coord.points): array = np.ma.array else: array = np.array coord.bounds = array(bounds) else: coord.guess_bounds() if lazy_points: coord.points = lazy.as_lazy_data(coord.points) if lazy_bounds: coord.bounds = lazy.as_lazy_data(coord.bounds) return coord
def test_lazy_complex(self): raw_points = np.arange(12).reshape(4, 3) points = as_lazy_data(raw_points, raw_points.shape) raw_bounds = np.arange(24).reshape(4, 3, 2) bounds = as_lazy_data(raw_bounds, raw_bounds.shape) coord = AuxCoord(points, bounds=bounds) self.assertTrue(is_lazy_data(coord.core_bounds())) result = AuxCoordFactory._nd_bounds(coord, (3, 2), 5) # Check we haven't triggered the loading of the coordinate values. self.assertTrue(is_lazy_data(coord.core_bounds())) self.assertTrue(is_lazy_data(result)) expected = raw_bounds.transpose((1, 0, 2)).reshape(1, 1, 3, 4, 1, 2) self.assertArrayEqual(result, expected)
def test_lazy_complex(self): raw_points = np.arange(12).reshape(4, 3) points = as_lazy_data(raw_points, raw_points.shape) raw_bounds = np.arange(24).reshape(4, 3, 2) bounds = as_lazy_data(raw_bounds, raw_bounds.shape) coord = AuxCoord(points, bounds=bounds) self.assertTrue(is_lazy_data(coord.core_bounds())) result = AuxCoordFactory._nd_bounds(coord, (3, 2), 5) # Check we haven't triggered the loading of the coordinate values. self.assertTrue(is_lazy_data(coord.core_bounds())) self.assertTrue(is_lazy_data(result)) expected = raw_bounds.transpose((1, 0, 2)).reshape(1, 1, 3, 4, 1, 2) self.assertArrayEqual(result, expected)
def test__str__lazy(self): # Displays lazy content (and does not realise!). self.meshcoord.points = as_lazy_data(self.meshcoord.points) self.meshcoord.bounds = as_lazy_data(self.meshcoord.bounds) result = str(self.meshcoord) self.assertTrue(self.meshcoord.has_lazy_points()) self.assertTrue(self.meshcoord.has_lazy_bounds()) self.assertIn("points: <lazy>", result) self.assertIn("bounds: <lazy>", result) re_expected = self._expected_elements_regexp() self.assertRegex(result, re_expected)
def test_preserves_lazy(self): test_bounds = np.array([[[11.0, 12.0], [12.0, 13.0], [13.0, 14.0]], [[21.0, 22.0], [22.0, 23.0], [23.0, 24.0]]]) test_points = np.array([[11.1, 12.2, 13.3], [21.4, 22.5, 23.6]]) lazy_points = as_lazy_data(test_points) lazy_bounds = as_lazy_data(test_bounds) coord = AuxCoord(points=lazy_points, bounds=lazy_bounds, units='m') coord.convert_units('ft') self.assertTrue(coord.has_lazy_points()) self.assertTrue(coord.has_lazy_bounds()) test_points_ft = Unit('m').convert(test_points, 'ft') test_bounds_ft = Unit('m').convert(test_bounds, 'ft') self.assertArrayAllClose(coord.points, test_points_ft) self.assertArrayAllClose(coord.bounds, test_bounds_ft)
def test_dtype_change(self): concrete_array = np.array([True, False]) lazy_array = as_lazy_data(concrete_array) wrapped = lazy_elementwise(lazy_array, _test_elementwise_op) self.assertTrue(is_lazy_data(wrapped)) self.assertEqual(wrapped.dtype, np.int) self.assertEqual(wrapped.compute().dtype, wrapped.dtype)
def test_cube_with_deferred_unit_conversion(self): real_data = np.arange(12.0).reshape((3, 4)) lazy_data = as_lazy_data(real_data) cube = iris.cube.Cube(lazy_data, units="m") cube.convert_units("ft") _, recon_cube = next(self.pickle_then_unpickle(cube)) self.assertEqual(recon_cube, cube)
def test_zero_ndim_lazy_to_scalar_1d_real_promote(self): real_array = np.array([456]) dm = DataManager(as_lazy_data(np.array(123))) self.assertTrue(dm.has_lazy_data()) dm.data = real_array self.assertFalse(dm.has_lazy_data()) self.assertArrayEqual(dm.data, real_array)
def test_basic(self): concrete_array = np.arange(30).reshape((2, 5, 3)) lazy_array = as_lazy_data(concrete_array) wrapped = lazy_elementwise(lazy_array, _test_elementwise_op) self.assertTrue(is_lazy_data(wrapped)) self.assertArrayAllClose(wrapped.compute(), _test_elementwise_op(concrete_array))
def test_ndim_0(self): real_array = np.array(0) dm = DataManager(real_array) self.assertEqual(dm.ndim, 0) lazy_array = as_lazy_data(real_array) dm = DataManager(lazy_array) self.assertEqual(dm.ndim, 0)
def test_lazy_data(self): data = np.arange(24).reshape((2, 12)) lazy_array = as_lazy_data(data) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertArrayEqual(result, data)
def test_shape_scalar(self): real_array = np.array(0) dm = DataManager(real_array) self.assertEqual(dm.shape, ()) lazy_array = as_lazy_data(real_array) dm = DataManager(lazy_array) self.assertEqual(dm.shape, ())
def test_dtype_same(self): concrete_array = np.array([3.0], dtype=np.float16) lazy_array = as_lazy_data(concrete_array) wrapped = lazy_elementwise(lazy_array, _test_elementwise_op) self.assertTrue(is_lazy_data(wrapped)) self.assertEqual(wrapped.dtype, np.float16) self.assertEqual(wrapped.compute().dtype, np.float16)
def test_1d(self): # 1-dimensional input. data = as_lazy_data(np.array([5, 2, 6, 4], dtype=np.float64), chunks=-1) rms = RMS.lazy_aggregate(data, 0) expected_rms = 4.5 self.assertAlmostEqual(rms, expected_rms)
def test_dtype_change(self): concrete_array = np.array([True, False]) lazy_array = as_lazy_data(concrete_array) wrapped = lazy_elementwise(lazy_array, _test_elementwise_op) self.assertTrue(is_lazy_data(wrapped)) self.assertEqual(wrapped.dtype, np.int) self.assertEqual(wrapped.compute().dtype, wrapped.dtype)
def test_basic(self): concrete_array = np.arange(30).reshape((2, 5, 3)) lazy_array = as_lazy_data(concrete_array) wrapped = lazy_elementwise(lazy_array, _test_elementwise_op) self.assertTrue(is_lazy_data(wrapped)) self.assertArrayAllClose(wrapped.compute(), _test_elementwise_op(concrete_array))
def test_repr_lazy(self): # Displays lazy content (and does not realise!). self.meshcoord.points = as_lazy_data(self.meshcoord.points) self.meshcoord.bounds = as_lazy_data(self.meshcoord.bounds) self.assertTrue(self.meshcoord.has_lazy_points()) self.assertTrue(self.meshcoord.has_lazy_bounds()) result = repr(self.meshcoord) self.assertTrue(self.meshcoord.has_lazy_points()) self.assertTrue(self.meshcoord.has_lazy_bounds()) expected = ( "<MeshCoord: longitude / (degrees_east) " "mesh(test_mesh) location(face) <lazy>+bounds shape(3,)>" ) self.assertEqual(expected, result)
def setUp(self): lazy_data = as_lazy_data(ma.masked_equal([1, 2, 3, 4, 5], 3)) self.lazy_cube = Cube(lazy_data) self.lazy_cube.add_dim_coord(DimCoord([6, 7, 8, 9, 10], long_name='foo'), 0) self.func = lambda x: x >= 3
def test_lazy_data(self): cube = iris.cube.Cube(as_lazy_data(self.data)) cube.add_aux_coord(iris.coords.DimCoord([1], standard_name="time")) res = new_axis(cube, "time") self.assertTrue(cube.has_lazy_data()) self.assertTrue(res.has_lazy_data()) self.assertEqual(res.shape, (1, ) + cube.shape)
def test_ndim_0(self): real_array = np.array(0) dm = DataManager(real_array) self.assertEqual(dm.ndim, 0) lazy_array = as_lazy_data(real_array) dm = DataManager(lazy_array) self.assertEqual(dm.ndim, 0)
def test_2d(self): # 2-dimensional input. data = as_lazy_data( np.array([[5, 2, 6, 4], [12, 4, 10, 8]], dtype=np.float64)) expected_rms = np.array([4.5, 9.0], dtype=np.float64) rms = RMS.lazy_aggregate(data, 1) self.assertArrayAlmostEqual(rms, expected_rms)
def create_syntheticube(template_cube, object_uri, replacement_coords, fill_value=1e20, storage_options=None): cnp = CheckingNetCDFDataProxy(shape=template_cube.shape, dtype=template_cube.dtype, path=object_uri, variable_name=template_cube.var_name, fill_value=fill_value, storage_options=storage_options) new_mdata = as_lazy_data(cnp) syntheticube = template_cube.copy(data=new_mdata) for coord_name, coord_value in replacement_coords.items(): if coord_name == 'forecast_reference_time': coord_value = pd.to_datetime(coord_value) try: syntheticube.coord(coord_name).points = coord_value except ValueError: if isinstance(coord_value, str): syntheticube.coord(coord_name).points = [ float(x) for x in coord_value.split() ] else: syntheticube.coord(coord_name).points = syntheticube.coord( coord_name).units.date2num(coord_value) syntheticube.coord(coord_name).bounds = None return syntheticube
def test_zero_ndim_lazy_with_zero_ndim_real(self): real_array = np.array(456) dm = DataManager(as_lazy_data(np.array(123))) self.assertTrue(dm.has_lazy_data()) dm.data = real_array self.assertFalse(dm.has_lazy_data()) self.assertArrayEqual(dm.data, real_array)
def test_lazy_data(self): data = np.arange(24).reshape((2, 12)) lazy_array = as_lazy_data(data) self.assertTrue(is_lazy_data(lazy_array)) result = as_concrete_data(lazy_array) self.assertFalse(is_lazy_data(result)) self.assertArrayEqual(result, data)
def setUp(self): self.data = ma.arange(12).reshape(3, 4) self.data.mask = [[0, 0, 0, 1], [0, 0, 1, 1], [0, 1, 1, 1]] # --> fractions of masked-points in columns = [0, 1/3, 2/3, 1] self.array = as_lazy_data(self.data) self.axis = 0 self.expected_masked = ma.mean(self.data, axis=self.axis)
def test_1d(self): # 1-dimensional input. data = as_lazy_data(np.array([5, 2, 6, 4], dtype=np.float64), chunks=-1) rms = RMS.lazy_aggregate(data, 0) expected_rms = 4.5 self.assertAlmostEqual(rms, expected_rms)
def test_dtype_same(self): concrete_array = np.array([3.], dtype=np.float16) lazy_array = as_lazy_data(concrete_array) wrapped = lazy_elementwise(lazy_array, _test_elementwise_op) self.assertTrue(is_lazy_data(wrapped)) self.assertEqual(wrapped.dtype, np.float16) self.assertEqual(wrapped.compute().dtype, np.float16)
def test_lazy_data(self): cube = iris.cube.Cube(as_lazy_data(self.data)) cube.add_aux_coord(iris.coords.DimCoord([1], standard_name='time')) res = new_axis(cube, 'time') self.assertTrue(cube.has_lazy_data()) self.assertTrue(res.has_lazy_data()) self.assertEqual(res.shape, (1,) + cube.shape)
def test_zero_ndim_real_to_scalar_1d_lazy_promote(self): lazy_array = as_lazy_data(np.array([456])) dm = DataManager(np.array(123)) self.assertFalse(dm.has_lazy_data()) dm.data = lazy_array self.assertTrue(dm.has_lazy_data()) self.assertArrayEqual(dm.data, lazy_array.compute())
def test_zero_ndim_lazy_to_scalar_1d_real_promote(self): real_array = np.array([456]) dm = DataManager(as_lazy_data(np.array(123))) self.assertTrue(dm.has_lazy_data()) dm.data = real_array self.assertFalse(dm.has_lazy_data()) self.assertArrayEqual(dm.data, real_array)
def test_zero_ndim_real_with_zero_ndim_lazy(self): lazy_array = as_lazy_data(np.array(456)) dm = DataManager(np.array(123)) self.assertFalse(dm.has_lazy_data()) dm.data = lazy_array self.assertTrue(dm.has_lazy_data()) self.assertArrayEqual(dm.data, lazy_array.compute())
def test_zero_ndim_lazy_with_zero_ndim_real(self): real_array = np.array(456) dm = DataManager(as_lazy_data(np.array(123))) self.assertTrue(dm.has_lazy_data()) dm.data = real_array self.assertFalse(dm.has_lazy_data()) self.assertArrayEqual(dm.data, real_array)
def test_lazy_with_real(self): dm = DataManager(as_lazy_data(self.real_array)) data = self.real_array.copy() * 10 result = dm._deepcopy(self.memo, data=data) expected = DataManager(data) self.assertEqual(result, expected) self.assertIs(result._real_array, data)
def _make_data( self, data, dtype=np.dtype("int32"), fill_value=None, mask=None, lazy=False, N=3, ): if isinstance(data, Iterable): shape = (len(data), N, N) data = np.array(data).reshape(-1, 1, 1) else: shape = (N, N) if mask is not None: payload = ma.empty(shape, dtype=dtype, fill_value=fill_value) payload.data[:] = data if isinstance(mask, bool): payload.mask = mask else: payload[mask] = ma.masked else: payload = np.empty(shape, dtype=dtype) payload[:] = data if lazy: payload = as_lazy_data(payload) return payload
def test_shape_scalar(self): real_array = np.array(0) dm = DataManager(real_array) self.assertEqual(dm.shape, ()) lazy_array = as_lazy_data(real_array) dm = DataManager(lazy_array) self.assertEqual(dm.shape, ())
def setUp(self): data = np.array([1, 2, 3, 4, 5]) coord = DimCoord([6, 7, 8, 9, 10], long_name='foo') self.cube = Cube(data) self.cube.add_dim_coord(coord, 0) self.lazy_cube = Cube(as_lazy_data(data)) self.lazy_cube.add_dim_coord(coord, 0)
def setUp(self): self.data = np.arange(12.0).reshape(3, 4) self.data[2, 1:] = np.nan self.array = as_lazy_data(self.data) masked_data = ma.masked_array(self.data, mask=np.isnan(self.data)) self.axis = 0 self.expected_masked = ma.mean(masked_data, axis=self.axis)
def setUp(self): data = np.array([1, 2, 3, 4, 5]) coord = DimCoord([6, 7, 8, 9, 10], long_name='foo') self.cube = Cube(data) self.cube.add_dim_coord(coord, 0) self.lazy_cube = Cube(as_lazy_data(data)) self.lazy_cube.add_dim_coord(coord, 0)
def test_lazy_with_real(self): lazy_array = as_lazy_data(self.real_array) dm = DataManager(lazy_array * 10) self.assertTrue(dm.has_lazy_data()) dm.replace(self.real_array) self.assertFalse(dm.has_lazy_data()) self.assertIs(dm._real_array, self.real_array) self.assertIs(dm.data, self.real_array)
def test_lazy_with_lazy(self): lazy_array = as_lazy_data(self.real_array) dm = DataManager(lazy_array * 10) self.assertTrue(dm.has_lazy_data()) dm.replace(lazy_array) self.assertTrue(dm.has_lazy_data()) self.assertIs(dm._lazy_array, lazy_array) self.assertArrayEqual(dm.data, self.real_array)
def test_ndim_nd(self): shape = (2, 3, 4) real_array = np.arange(24).reshape(shape) dm = DataManager(real_array) self.assertEqual(dm.ndim, len(shape)) lazy_array = as_lazy_data(real_array) dm = DataManager(lazy_array) self.assertEqual(dm.ndim, len(shape))
def test_2d(self): # 2-dimensional input. data = as_lazy_data(np.array([[5, 2, 6, 4], [12, 4, 10, 8]], dtype=np.float64), chunks=-1) expected_rms = np.array([4.5, 9.0], dtype=np.float64) rms = RMS.lazy_aggregate(data, 1) self.assertArrayAlmostEqual(rms, expected_rms)
def test_ndim_nd(self): shape = (2, 3, 4) real_array = np.arange(24).reshape(shape) dm = DataManager(real_array) self.assertEqual(dm.ndim, len(shape)) lazy_array = as_lazy_data(real_array) dm = DataManager(lazy_array) self.assertEqual(dm.ndim, len(shape))
def test_large_specific_chunk_passthrough(self): # Check that even a too-large specific 'chunks' arg is honoured. limitcall_patch = self.patch('iris._lazy_data._limited_shape') huge_test_shape = (1001, 1002, 1003, 1004) data = self._dummydata(huge_test_shape) result = as_lazy_data(data, chunks=huge_test_shape) self.assertEqual(limitcall_patch.call_args_list, []) self.assertEqual(result.shape, huge_test_shape)