def test_lazy_with_lazy__fill_value_failure(self):
fill_value = 1234
dm1 = DataManager(as_lazy_data(self.real_array),
fill_value=fill_value)
dm2 = DataManager(as_lazy_data(self.real_array),
fill_value=fill_value)
self.assertFalse(dm1 != dm2)
def test_lazy_with_lazy__realised_dtype(self):
dtype = np.dtype('int16')
dm1 = DataManager(as_lazy_data(self.real_array),
realised_dtype=dtype)
dm2 = DataManager(as_lazy_data(self.real_array),
realised_dtype=dtype)
self.assertEqual(dm1, dm2)
def test_lazy_with_lazy(self):
dm = DataManager(as_lazy_data(self.real_array))
data = as_lazy_data(self.real_array) * 10
result = dm._deepcopy(self.memo, data=data)
expected = DataManager(data)
self.assertEqual(result, expected)
self.assertIs(result._lazy_array, data)
def test_lazy_with_lazy__realised_dtype_failure(self):
dtype = np.dtype('int16')
dm1 = DataManager(as_lazy_data(self.real_array),
realised_dtype=dtype)
dm2 = DataManager(as_lazy_data(self.real_array),
realised_dtype=dtype)
self.assertFalse(dm1 != dm2)
def test_zero_ndim_lazy_with_zero_ndim_lazy(self):
lazy_array = as_lazy_data(np.array(456))
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = lazy_array
self.assertTrue(dm.has_lazy_data())
self.assertArrayEqual(dm.data, lazy_array.compute())
def test_zero_ndim_lazy_to_scalar_1d_lazy_promote(self):
lazy_array = as_lazy_data(np.array([456]))
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = lazy_array
self.assertTrue(dm.has_lazy_data())
self.assertArrayEqual(dm.data, lazy_array.compute())
def test_lazy_array__fill_value_clearance(self):
fill_value = 1234
dm = DataManager(as_lazy_data(np.array(0)),
fill_value=fill_value)
self.assertEqual(dm.fill_value, fill_value)
dm.data = as_lazy_data(np.array(1))
self.assertIsNone(dm.fill_value)
def test_lazy_with_lazy__fill_value(self):
fill_value = 1234
dm1 = DataManager(as_lazy_data(self.real_array),
fill_value=fill_value)
dm2 = DataManager(as_lazy_data(self.real_array),
fill_value=fill_value)
self.assertEqual(dm1, dm2)
def test_default_chunks_limiting(self):
# Check that chunking is limited when no specific 'chunks' given.
limitcall_patch = self.patch('iris._lazy_data._limited_shape')
test_shape = (3, 2, 4)
data = self._dummydata(test_shape)
as_lazy_data(data)
self.assertEqual(limitcall_patch.call_args_list,
[mock.call(test_shape)])
def test_lazy_with_realised_dtype(self):
dm = DataManager(as_lazy_data(self.real_array),
realised_dtype=np.dtype('int16'))
data = as_lazy_data(self.real_array) * 10
dtype = np.dtype('int8')
result = dm._deepcopy(self.memo, data=data, realised_dtype=dtype)
expected = DataManager(data, realised_dtype=dtype)
self.assertEqual(result, expected)
self.assertIs(result._lazy_array, data)
def test_1d_lazy_weighted(self):
# 1-dimensional input with lazy weights.
data = as_lazy_data(np.array([4, 7, 10, 8], dtype=np.float64),
chunks=-1)
weights = as_lazy_data(np.array([1, 4, 3, 2], dtype=np.float64),
chunks=-1)
expected_rms = 8.0
# https://github.com/dask/dask/issues/3846.
with self.assertRaisesRegexp(TypeError, 'unexpected keyword argument'):
rms = RMS.lazy_aggregate(data, 0, weights=weights)
self.assertAlmostEqual(rms, expected_rms)
def test_default_chunks_limiting(self):
# Check that chunking is still controlled when no specific 'chunks'
# is passed.
limitcall_patch = self.patch('iris._lazy_data._optimum_chunksize')
test_shape = (3, 2, 4)
data = self._dummydata(test_shape)
as_lazy_data(data)
self.assertEqual(limitcall_patch.call_args_list,
[mock.call(list(test_shape),
shape=test_shape,
dtype=np.dtype('f4'))])
def test_1d_lazy_weighted(self):
# 1-dimensional input with lazy weights.
data = as_lazy_data(np.array([4, 7, 10, 8], dtype=np.float64),
chunks=-1)
weights = as_lazy_data(np.array([1, 4, 3, 2], dtype=np.float64),
chunks=-1)
expected_rms = 8.0
# https://github.com/dask/dask/issues/3846.
with self.assertRaisesRegexp(TypeError, 'unexpected keyword argument'):
rms = RMS.lazy_aggregate(data, 0, weights=weights)
self.assertAlmostEqual(rms, expected_rms)
def setUp(self):
shape = (2, 3, 4)
size = np.prod(shape)
self.real_array = np.arange(size).reshape(shape)
self.lazy_array = as_lazy_data(self.real_array)
self.mask_array = ma.masked_array(self.real_array)
self.mask_array_masked = self.mask_array.copy()
self.mask_array_masked[0, 0, 0] = ma.masked
self.realised_dtype = self.mask_array.dtype
self.lazy_mask_array = as_lazy_data(self.mask_array)
self.lazy_mask_array_masked = as_lazy_data(self.mask_array_masked)
def setUp(self):
shape = (2, 3, 4)
size = np.prod(shape)
self.real_array = np.arange(size).reshape(shape)
self.lazy_array = as_lazy_data(self.real_array)
self.mask_array = ma.masked_array(self.real_array)
self.mask_array_masked = self.mask_array.copy()
self.mask_array_masked[0, 0, 0] = ma.masked
self.realised_dtype = self.mask_array.dtype
self.lazy_mask_array = as_lazy_data(self.mask_array)
self.lazy_mask_array_masked = as_lazy_data(self.mask_array_masked)
def sample_coord(
self,
datatype=float,
dates=False,
units="m",
long_name="x",
shape=(5, ),
masked=False,
bounded=False,
dimcoord=False,
lazy_points=False,
lazy_bounds=False,
*coord_args,
**coord_kwargs,
):
if masked:
dimcoord = False
if dates:
# Use a pre-programmed date unit.
units = Unit("days since 1970-03-5")
if not isinstance(units, Unit):
# This operation is *not* a no-op, it will wipe calendars !
units = Unit(units)
values = self.sample_data(datatype=datatype,
units=units,
shape=shape,
masked=masked)
cls = DimCoord if dimcoord else AuxCoord
coord = cls(
points=values,
units=units,
long_name=long_name,
*coord_args,
**coord_kwargs,
)
if bounded or lazy_bounds:
if shape == (1, ):
# Guess-bounds doesn't work !
val = coord.points[0]
bounds = [val - 10, val + 10]
# NB preserve masked/unmasked : avoid converting masks to NaNs
if np.ma.isMaskedArray(coord.points):
array = np.ma.array
else:
array = np.array
coord.bounds = array(bounds)
else:
coord.guess_bounds()
if lazy_points:
coord.points = lazy.as_lazy_data(coord.points)
if lazy_bounds:
coord.bounds = lazy.as_lazy_data(coord.bounds)
return coord
def test_lazy_complex(self):
raw_points = np.arange(12).reshape(4, 3)
points = as_lazy_data(raw_points, raw_points.shape)
raw_bounds = np.arange(24).reshape(4, 3, 2)
bounds = as_lazy_data(raw_bounds, raw_bounds.shape)
coord = AuxCoord(points, bounds=bounds)
self.assertTrue(is_lazy_data(coord.core_bounds()))
result = AuxCoordFactory._nd_bounds(coord, (3, 2), 5)
# Check we haven't triggered the loading of the coordinate values.
self.assertTrue(is_lazy_data(coord.core_bounds()))
self.assertTrue(is_lazy_data(result))
expected = raw_bounds.transpose((1, 0, 2)).reshape(1, 1, 3, 4, 1, 2)
self.assertArrayEqual(result, expected)
def test_lazy_complex(self):
raw_points = np.arange(12).reshape(4, 3)
points = as_lazy_data(raw_points, raw_points.shape)
raw_bounds = np.arange(24).reshape(4, 3, 2)
bounds = as_lazy_data(raw_bounds, raw_bounds.shape)
coord = AuxCoord(points, bounds=bounds)
self.assertTrue(is_lazy_data(coord.core_bounds()))
result = AuxCoordFactory._nd_bounds(coord, (3, 2), 5)
# Check we haven't triggered the loading of the coordinate values.
self.assertTrue(is_lazy_data(coord.core_bounds()))
self.assertTrue(is_lazy_data(result))
expected = raw_bounds.transpose((1, 0, 2)).reshape(1, 1, 3, 4, 1, 2)
self.assertArrayEqual(result, expected)
def test__str__lazy(self):
# Displays lazy content (and does not realise!).
self.meshcoord.points = as_lazy_data(self.meshcoord.points)
self.meshcoord.bounds = as_lazy_data(self.meshcoord.bounds)
result = str(self.meshcoord)
self.assertTrue(self.meshcoord.has_lazy_points())
self.assertTrue(self.meshcoord.has_lazy_bounds())
self.assertIn("points: <lazy>", result)
self.assertIn("bounds: <lazy>", result)
re_expected = self._expected_elements_regexp()
self.assertRegex(result, re_expected)
def test_preserves_lazy(self):
test_bounds = np.array([[[11.0, 12.0], [12.0, 13.0], [13.0, 14.0]],
[[21.0, 22.0], [22.0, 23.0], [23.0, 24.0]]])
test_points = np.array([[11.1, 12.2, 13.3], [21.4, 22.5, 23.6]])
lazy_points = as_lazy_data(test_points)
lazy_bounds = as_lazy_data(test_bounds)
coord = AuxCoord(points=lazy_points, bounds=lazy_bounds, units='m')
coord.convert_units('ft')
self.assertTrue(coord.has_lazy_points())
self.assertTrue(coord.has_lazy_bounds())
test_points_ft = Unit('m').convert(test_points, 'ft')
test_bounds_ft = Unit('m').convert(test_bounds, 'ft')
self.assertArrayAllClose(coord.points, test_points_ft)
self.assertArrayAllClose(coord.bounds, test_bounds_ft)
def test_dtype_change(self):
concrete_array = np.array([True, False])
lazy_array = as_lazy_data(concrete_array)
wrapped = lazy_elementwise(lazy_array, _test_elementwise_op)
self.assertTrue(is_lazy_data(wrapped))
self.assertEqual(wrapped.dtype, np.int)
self.assertEqual(wrapped.compute().dtype, wrapped.dtype)
def test_cube_with_deferred_unit_conversion(self):
real_data = np.arange(12.0).reshape((3, 4))
lazy_data = as_lazy_data(real_data)
cube = iris.cube.Cube(lazy_data, units="m")
cube.convert_units("ft")
_, recon_cube = next(self.pickle_then_unpickle(cube))
self.assertEqual(recon_cube, cube)
def test_zero_ndim_lazy_to_scalar_1d_real_promote(self):
real_array = np.array([456])
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = real_array
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, real_array)
def test_basic(self):
concrete_array = np.arange(30).reshape((2, 5, 3))
lazy_array = as_lazy_data(concrete_array)
wrapped = lazy_elementwise(lazy_array, _test_elementwise_op)
self.assertTrue(is_lazy_data(wrapped))
self.assertArrayAllClose(wrapped.compute(),
_test_elementwise_op(concrete_array))
def test_ndim_0(self):
real_array = np.array(0)
dm = DataManager(real_array)
self.assertEqual(dm.ndim, 0)
lazy_array = as_lazy_data(real_array)
dm = DataManager(lazy_array)
self.assertEqual(dm.ndim, 0)
def test_lazy_data(self):
data = np.arange(24).reshape((2, 12))
lazy_array = as_lazy_data(data)
self.assertTrue(is_lazy_data(lazy_array))
result = as_concrete_data(lazy_array)
self.assertFalse(is_lazy_data(result))
self.assertArrayEqual(result, data)
def test_shape_scalar(self):
real_array = np.array(0)
dm = DataManager(real_array)
self.assertEqual(dm.shape, ())
lazy_array = as_lazy_data(real_array)
dm = DataManager(lazy_array)
self.assertEqual(dm.shape, ())
def test_dtype_same(self):
concrete_array = np.array([3.0], dtype=np.float16)
lazy_array = as_lazy_data(concrete_array)
wrapped = lazy_elementwise(lazy_array, _test_elementwise_op)
self.assertTrue(is_lazy_data(wrapped))
self.assertEqual(wrapped.dtype, np.float16)
self.assertEqual(wrapped.compute().dtype, np.float16)
def test_1d(self):
# 1-dimensional input.
data = as_lazy_data(np.array([5, 2, 6, 4], dtype=np.float64),
chunks=-1)
rms = RMS.lazy_aggregate(data, 0)
expected_rms = 4.5
self.assertAlmostEqual(rms, expected_rms)
def test_dtype_change(self):
concrete_array = np.array([True, False])
lazy_array = as_lazy_data(concrete_array)
wrapped = lazy_elementwise(lazy_array, _test_elementwise_op)
self.assertTrue(is_lazy_data(wrapped))
self.assertEqual(wrapped.dtype, np.int)
self.assertEqual(wrapped.compute().dtype, wrapped.dtype)
def test_basic(self):
concrete_array = np.arange(30).reshape((2, 5, 3))
lazy_array = as_lazy_data(concrete_array)
wrapped = lazy_elementwise(lazy_array, _test_elementwise_op)
self.assertTrue(is_lazy_data(wrapped))
self.assertArrayAllClose(wrapped.compute(),
_test_elementwise_op(concrete_array))
def test_repr_lazy(self):
# Displays lazy content (and does not realise!).
self.meshcoord.points = as_lazy_data(self.meshcoord.points)
self.meshcoord.bounds = as_lazy_data(self.meshcoord.bounds)
self.assertTrue(self.meshcoord.has_lazy_points())
self.assertTrue(self.meshcoord.has_lazy_bounds())
result = repr(self.meshcoord)
self.assertTrue(self.meshcoord.has_lazy_points())
self.assertTrue(self.meshcoord.has_lazy_bounds())
expected = (
"<MeshCoord: longitude / (degrees_east) "
"mesh(test_mesh) location(face) <lazy>+bounds shape(3,)>"
)
self.assertEqual(expected, result)
def setUp(self):
lazy_data = as_lazy_data(ma.masked_equal([1, 2, 3, 4, 5], 3))
self.lazy_cube = Cube(lazy_data)
self.lazy_cube.add_dim_coord(DimCoord([6, 7, 8, 9, 10],
long_name='foo'),
0)
self.func = lambda x: x >= 3
def test_lazy_data(self):
cube = iris.cube.Cube(as_lazy_data(self.data))
cube.add_aux_coord(iris.coords.DimCoord([1], standard_name="time"))
res = new_axis(cube, "time")
self.assertTrue(cube.has_lazy_data())
self.assertTrue(res.has_lazy_data())
self.assertEqual(res.shape, (1, ) + cube.shape)
def test_ndim_0(self):
real_array = np.array(0)
dm = DataManager(real_array)
self.assertEqual(dm.ndim, 0)
lazy_array = as_lazy_data(real_array)
dm = DataManager(lazy_array)
self.assertEqual(dm.ndim, 0)
def test_2d(self):
# 2-dimensional input.
data = as_lazy_data(
np.array([[5, 2, 6, 4], [12, 4, 10, 8]], dtype=np.float64))
expected_rms = np.array([4.5, 9.0], dtype=np.float64)
rms = RMS.lazy_aggregate(data, 1)
self.assertArrayAlmostEqual(rms, expected_rms)
def create_syntheticube(template_cube,
object_uri,
replacement_coords,
fill_value=1e20,
storage_options=None):
cnp = CheckingNetCDFDataProxy(shape=template_cube.shape,
dtype=template_cube.dtype,
path=object_uri,
variable_name=template_cube.var_name,
fill_value=fill_value,
storage_options=storage_options)
new_mdata = as_lazy_data(cnp)
syntheticube = template_cube.copy(data=new_mdata)
for coord_name, coord_value in replacement_coords.items():
if coord_name == 'forecast_reference_time':
coord_value = pd.to_datetime(coord_value)
try:
syntheticube.coord(coord_name).points = coord_value
except ValueError:
if isinstance(coord_value, str):
syntheticube.coord(coord_name).points = [
float(x) for x in coord_value.split()
]
else:
syntheticube.coord(coord_name).points = syntheticube.coord(
coord_name).units.date2num(coord_value)
syntheticube.coord(coord_name).bounds = None
return syntheticube
def test_zero_ndim_lazy_with_zero_ndim_real(self):
real_array = np.array(456)
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = real_array
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, real_array)
def test_lazy_data(self):
data = np.arange(24).reshape((2, 12))
lazy_array = as_lazy_data(data)
self.assertTrue(is_lazy_data(lazy_array))
result = as_concrete_data(lazy_array)
self.assertFalse(is_lazy_data(result))
self.assertArrayEqual(result, data)
def setUp(self):
self.data = ma.arange(12).reshape(3, 4)
self.data.mask = [[0, 0, 0, 1], [0, 0, 1, 1], [0, 1, 1, 1]]
# --> fractions of masked-points in columns = [0, 1/3, 2/3, 1]
self.array = as_lazy_data(self.data)
self.axis = 0
self.expected_masked = ma.mean(self.data, axis=self.axis)
def test_1d(self):
# 1-dimensional input.
data = as_lazy_data(np.array([5, 2, 6, 4], dtype=np.float64),
chunks=-1)
rms = RMS.lazy_aggregate(data, 0)
expected_rms = 4.5
self.assertAlmostEqual(rms, expected_rms)
def test_dtype_same(self):
concrete_array = np.array([3.], dtype=np.float16)
lazy_array = as_lazy_data(concrete_array)
wrapped = lazy_elementwise(lazy_array, _test_elementwise_op)
self.assertTrue(is_lazy_data(wrapped))
self.assertEqual(wrapped.dtype, np.float16)
self.assertEqual(wrapped.compute().dtype, np.float16)
def test_lazy_data(self):
cube = iris.cube.Cube(as_lazy_data(self.data))
cube.add_aux_coord(iris.coords.DimCoord([1], standard_name='time'))
res = new_axis(cube, 'time')
self.assertTrue(cube.has_lazy_data())
self.assertTrue(res.has_lazy_data())
self.assertEqual(res.shape, (1,) + cube.shape)
def test_zero_ndim_real_to_scalar_1d_lazy_promote(self):
lazy_array = as_lazy_data(np.array([456]))
dm = DataManager(np.array(123))
self.assertFalse(dm.has_lazy_data())
dm.data = lazy_array
self.assertTrue(dm.has_lazy_data())
self.assertArrayEqual(dm.data, lazy_array.compute())
def test_zero_ndim_lazy_to_scalar_1d_real_promote(self):
real_array = np.array([456])
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = real_array
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, real_array)
def test_zero_ndim_real_with_zero_ndim_lazy(self):
lazy_array = as_lazy_data(np.array(456))
dm = DataManager(np.array(123))
self.assertFalse(dm.has_lazy_data())
dm.data = lazy_array
self.assertTrue(dm.has_lazy_data())
self.assertArrayEqual(dm.data, lazy_array.compute())
def test_zero_ndim_lazy_with_zero_ndim_real(self):
real_array = np.array(456)
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = real_array
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, real_array)
def test_lazy_with_real(self):
dm = DataManager(as_lazy_data(self.real_array))
data = self.real_array.copy() * 10
result = dm._deepcopy(self.memo, data=data)
expected = DataManager(data)
self.assertEqual(result, expected)
self.assertIs(result._real_array, data)
def _make_data(
self,
data,
dtype=np.dtype("int32"),
fill_value=None,
mask=None,
lazy=False,
N=3,
):
if isinstance(data, Iterable):
shape = (len(data), N, N)
data = np.array(data).reshape(-1, 1, 1)
else:
shape = (N, N)
if mask is not None:
payload = ma.empty(shape, dtype=dtype, fill_value=fill_value)
payload.data[:] = data
if isinstance(mask, bool):
payload.mask = mask
else:
payload[mask] = ma.masked
else:
payload = np.empty(shape, dtype=dtype)
payload[:] = data
if lazy:
payload = as_lazy_data(payload)
return payload
def test_shape_scalar(self):
real_array = np.array(0)
dm = DataManager(real_array)
self.assertEqual(dm.shape, ())
lazy_array = as_lazy_data(real_array)
dm = DataManager(lazy_array)
self.assertEqual(dm.shape, ())
def setUp(self):
data = np.array([1, 2, 3, 4, 5])
coord = DimCoord([6, 7, 8, 9, 10], long_name='foo')
self.cube = Cube(data)
self.cube.add_dim_coord(coord, 0)
self.lazy_cube = Cube(as_lazy_data(data))
self.lazy_cube.add_dim_coord(coord, 0)
def setUp(self):
self.data = np.arange(12.0).reshape(3, 4)
self.data[2, 1:] = np.nan
self.array = as_lazy_data(self.data)
masked_data = ma.masked_array(self.data, mask=np.isnan(self.data))
self.axis = 0
self.expected_masked = ma.mean(masked_data, axis=self.axis)
def setUp(self):
data = np.array([1, 2, 3, 4, 5])
coord = DimCoord([6, 7, 8, 9, 10], long_name='foo')
self.cube = Cube(data)
self.cube.add_dim_coord(coord, 0)
self.lazy_cube = Cube(as_lazy_data(data))
self.lazy_cube.add_dim_coord(coord, 0)
def test_lazy_with_real(self):
lazy_array = as_lazy_data(self.real_array)
dm = DataManager(lazy_array * 10)
self.assertTrue(dm.has_lazy_data())
dm.replace(self.real_array)
self.assertFalse(dm.has_lazy_data())
self.assertIs(dm._real_array, self.real_array)
self.assertIs(dm.data, self.real_array)
def test_lazy_with_lazy(self):
lazy_array = as_lazy_data(self.real_array)
dm = DataManager(lazy_array * 10)
self.assertTrue(dm.has_lazy_data())
dm.replace(lazy_array)
self.assertTrue(dm.has_lazy_data())
self.assertIs(dm._lazy_array, lazy_array)
self.assertArrayEqual(dm.data, self.real_array)
def test_ndim_nd(self):
shape = (2, 3, 4)
real_array = np.arange(24).reshape(shape)
dm = DataManager(real_array)
self.assertEqual(dm.ndim, len(shape))
lazy_array = as_lazy_data(real_array)
dm = DataManager(lazy_array)
self.assertEqual(dm.ndim, len(shape))
def test_2d(self):
# 2-dimensional input.
data = as_lazy_data(np.array([[5, 2, 6, 4], [12, 4, 10, 8]],
dtype=np.float64),
chunks=-1)
expected_rms = np.array([4.5, 9.0], dtype=np.float64)
rms = RMS.lazy_aggregate(data, 1)
self.assertArrayAlmostEqual(rms, expected_rms)
def test_ndim_nd(self):
shape = (2, 3, 4)
real_array = np.arange(24).reshape(shape)
dm = DataManager(real_array)
self.assertEqual(dm.ndim, len(shape))
lazy_array = as_lazy_data(real_array)
dm = DataManager(lazy_array)
self.assertEqual(dm.ndim, len(shape))
def test_large_specific_chunk_passthrough(self):
# Check that even a too-large specific 'chunks' arg is honoured.
limitcall_patch = self.patch('iris._lazy_data._limited_shape')
huge_test_shape = (1001, 1002, 1003, 1004)
data = self._dummydata(huge_test_shape)
result = as_lazy_data(data, chunks=huge_test_shape)
self.assertEqual(limitcall_patch.call_args_list, [])
self.assertEqual(result.shape, huge_test_shape)
