def test_zero_ndim_lazy_to_scalar_1d_real_promote(self):
real_array = np.array([456])
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = real_array
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, real_array)
def test_real_with_real(self):
dm = DataManager(self.real_array)
data = self.real_array.copy() * 10
result = dm._deepcopy(self.memo, data=data)
expected = DataManager(data)
self.assertEqual(result, expected)
self.assertIs(result._real_array, data)
def test_real_with_real(self):
dm = DataManager(self.real_array * 10)
self.assertFalse(dm.has_lazy_data())
dm.replace(self.real_array)
self.assertFalse(dm.has_lazy_data())
self.assertIs(dm._real_array, self.real_array)
self.assertIs(dm.data, self.real_array)
def test_zero_ndim_lazy_to_scalar_1d_lazy_promote(self):
lazy_array = as_lazy_data(np.array([456]))
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = lazy_array
self.assertTrue(dm.has_lazy_data())
self.assertArrayEqual(dm.data, lazy_array.compute())
def test_fill_value_overflow(self):
fill_value = np.float(1e+20)
data = np.array([0], dtype=np.int32)
dm = DataManager(data)
emsg = 'Fill value of .* invalid for dtype'
with self.assertRaisesRegexp(ValueError, emsg):
dm.fill_value = fill_value
def test_zero_ndim_lazy_to_scalar_1d_lazy_promote(self):
lazy_array = as_lazy_data(np.array([456]))
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = lazy_array
self.assertTrue(dm.has_lazy_data())
self.assertArrayEqual(dm.data, lazy_array.compute())
def test_lazy_with_lazy__fill_value_failure(self):
fill_value = 1234
dm1 = DataManager(as_lazy_data(self.real_array),
fill_value=fill_value)
dm2 = DataManager(as_lazy_data(self.real_array),
fill_value=fill_value)
self.assertFalse(dm1 != dm2)
def test_lazy_with_lazy__realised_dtype(self):
dtype = np.dtype('int16')
dm1 = DataManager(as_lazy_data(self.real_array),
realised_dtype=dtype)
dm2 = DataManager(as_lazy_data(self.real_array),
realised_dtype=dtype)
self.assertEqual(dm1, dm2)
def test_shape_scalar(self):
real_array = np.array(0)
dm = DataManager(real_array)
self.assertEqual(dm.shape, ())
lazy_array = as_lazy_data(real_array)
dm = DataManager(lazy_array)
self.assertEqual(dm.shape, ())
def test_with_real_array(self):
dm = DataManager(self.real_array)
self.assertFalse(dm.has_lazy_data())
result = dm.lazy_data()
self.assertFalse(dm.has_lazy_data())
self.assertEqual(result, self.lazy_array)
self.assertFalse(dm.has_lazy_data())
def test_nop__with_real_array(self):
fill_value = 1234
dm = DataManager(self.real_array)
dm.fill_value = 1234
self.assertEqual(dm.fill_value, fill_value)
dm._propagate_masked_data_fill_value()
self.assertEqual(dm.fill_value, fill_value)
def test_real_with_real__realised_dtype_failure(self):
dm = DataManager(self.real_array)
emsg = 'Cannot set realised dtype, no lazy data is available.'
with self.assertRaisesRegexp(ValueError, emsg):
dm.replace(self.real_array * 10, realised_dtype=np.dtype('int16'))
self.assertIs(dm._real_array, self.real_array)
self.assertArrayEqual(dm.data, self.real_array)
def test_zero_ndim_lazy_with_zero_ndim_real(self):
real_array = np.array(456)
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = real_array
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, real_array)
def test_mask_array__with_fill_value(self):
dm = DataManager(np.array(0))
self.assertIsNone(dm.fill_value)
fill_value = 1234
data = ma.array(1, fill_value=fill_value)
dm.data = data
self.assertEqual(dm.fill_value, fill_value)
def test_fill_value_overflow(self):
fill_value = np.float(1e+20)
data = np.array([0], dtype=np.int32)
dm = DataManager(data)
emsg = 'Fill value of .* invalid for dtype'
with self.assertRaisesRegexp(ValueError, emsg):
dm.fill_value = fill_value
def test_mask_array__with_fill_value(self):
dm = DataManager(np.array(0))
self.assertIsNone(dm.fill_value)
fill_value = 1234
data = ma.array(1, fill_value=fill_value)
dm.data = data
self.assertEqual(dm.fill_value, fill_value)
class Test__propagate_masked_data_fill_value(tests.IrisTest):
def setUp(self):
self.real_array = np.array(0)
self.mask_array = ma.array(0)
np_fill_value = ma.array(0, dtype=self.mask_array.dtype).fill_value
assert np_fill_value == self.mask_array.fill_value
self.np_fill_value = np_fill_value
self.dm = DataManager(self.mask_array)
self.dm.fill_value = None
def test_nop__with_real_array(self):
fill_value = 1234
dm = DataManager(self.real_array)
dm.fill_value = 1234
self.assertEqual(dm.fill_value, fill_value)
dm._propagate_masked_data_fill_value()
self.assertEqual(dm.fill_value, fill_value)
def test_clear_fill_value__with_np_default(self):
fill_value = 1234
self.dm.fill_value = fill_value
self.assertEqual(self.dm.fill_value, fill_value)
self.dm._propagate_masked_data_fill_value()
self.assertIsNone(self.dm.fill_value)
def test_set_fill_value(self):
fill_value = 1234
self.assertIsNone(self.dm.fill_value)
self.dm._real_array.fill_value = fill_value
self.dm._propagate_masked_data_fill_value()
self.assertEqual(self.dm.fill_value, fill_value)
def test_nop__with_real_array(self):
fill_value = 1234
dm = DataManager(self.real_array)
dm.fill_value = 1234
self.assertEqual(dm.fill_value, fill_value)
dm._propagate_masked_data_fill_value()
self.assertEqual(dm.fill_value, fill_value)
def test_ndim_0(self):
real_array = np.array(0)
dm = DataManager(real_array)
self.assertEqual(dm.ndim, 0)
lazy_array = as_lazy_data(real_array)
dm = DataManager(lazy_array)
self.assertEqual(dm.ndim, 0)
def test_lazy_with_lazy(self):
dm = DataManager(as_lazy_data(self.real_array))
data = as_lazy_data(self.real_array) * 10
result = dm._deepcopy(self.memo, data=data)
expected = DataManager(data)
self.assertEqual(result, expected)
self.assertIs(result._lazy_array, data)
def test_real_with_real(self):
dm = DataManager(self.real_array)
data = self.real_array.copy() * 10
result = dm._deepcopy(self.memo, data=data)
expected = DataManager(data)
self.assertEqual(result, expected)
self.assertIs(result._real_array, data)
def test__clear_fill_value_masked(self):
fill_value = 1234
dm = DataManager(np.array(0), fill_value=fill_value)
self.assertEqual(dm.fill_value, fill_value)
data = ma.masked_array(1, fill_value=4321)
dm.replace(data)
self.assertIsNone(dm.fill_value)
def test_lazy_array__fill_value_clearance(self):
fill_value = 1234
dm = DataManager(as_lazy_data(np.array(0)),
fill_value=fill_value)
self.assertEqual(dm.fill_value, fill_value)
dm.data = as_lazy_data(np.array(1))
self.assertIsNone(dm.fill_value)
def test_zero_ndim_lazy_with_zero_ndim_lazy(self):
lazy_array = as_lazy_data(np.array(456))
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = lazy_array
self.assertTrue(dm.has_lazy_data())
self.assertArrayEqual(dm.data, lazy_array.compute())
def test_zero_ndim_real_with_scalar_float(self):
value = 456.0
dm = DataManager(np.array(123))
self.assertFalse(dm.has_lazy_data())
dm.data = value
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, np.array(value))
def test_lazy_array__fill_value_clearance(self):
fill_value = 1234
dm = DataManager(as_lazy_data(np.array(0)),
fill_value=fill_value)
self.assertEqual(dm.fill_value, fill_value)
dm.data = as_lazy_data(np.array(1))
self.assertIsNone(dm.fill_value)
def test_zero_ndim_lazy_with_zero_ndim_real(self):
real_array = np.array(456)
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = real_array
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, real_array)
def test_zero_ndim_real_with_scalar_float(self):
value = 456.0
dm = DataManager(np.array(123))
self.assertFalse(dm.has_lazy_data())
dm.data = value
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, np.array(value))
def test_zero_ndim_lazy_with_zero_ndim_lazy(self):
lazy_array = as_lazy_data(np.array(456))
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = lazy_array
self.assertTrue(dm.has_lazy_data())
self.assertArrayEqual(dm.data, lazy_array.compute())
class Test__propagate_masked_data_fill_value(tests.IrisTest):
def setUp(self):
self.real_array = np.array(0)
self.mask_array = ma.array(0)
np_fill_value = ma.array(0, dtype=self.mask_array.dtype).fill_value
assert np_fill_value == self.mask_array.fill_value
self.np_fill_value = np_fill_value
self.dm = DataManager(self.mask_array)
self.dm.fill_value = None
def test_nop__with_real_array(self):
fill_value = 1234
dm = DataManager(self.real_array)
dm.fill_value = 1234
self.assertEqual(dm.fill_value, fill_value)
dm._propagate_masked_data_fill_value()
self.assertEqual(dm.fill_value, fill_value)
def test_clear_fill_value__with_np_default(self):
fill_value = 1234
self.dm.fill_value = fill_value
self.assertEqual(self.dm.fill_value, fill_value)
self.dm._propagate_masked_data_fill_value()
self.assertIsNone(self.dm.fill_value)
def test_set_fill_value(self):
fill_value = 1234
self.assertIsNone(self.dm.fill_value)
self.dm._real_array.fill_value = fill_value
self.dm._propagate_masked_data_fill_value()
self.assertEqual(self.dm.fill_value, fill_value)
def test_lazy_with_lazy(self):
dm = DataManager(as_lazy_data(self.real_array))
data = as_lazy_data(self.real_array) * 10
result = dm._deepcopy(self.memo, data=data)
expected = DataManager(data)
self.assertEqual(result, expected)
self.assertIs(result._lazy_array, data)
def test_real_with_real__promote_shape_with_dtype_failure(self):
data = np.array(666)
dm = DataManager(data)
emsg = 'Cannot set realised dtype, no lazy data is available'
with self.assertRaisesRegexp(ValueError, emsg):
dm.replace(np.array([999]), realised_dtype=np.dtype('float32'))
self.assertArrayEqual(dm.data, data)
def test_lazy_with_lazy__realised_dtype_failure(self):
dtype = np.dtype('int16')
dm1 = DataManager(as_lazy_data(self.real_array),
realised_dtype=dtype)
dm2 = DataManager(as_lazy_data(self.real_array),
realised_dtype=dtype)
self.assertFalse(dm1 != dm2)
def test__clear_fill_value_masked(self):
fill_value = 1234
dm = DataManager(np.array(0), fill_value=fill_value)
self.assertEqual(dm.fill_value, fill_value)
data = ma.masked_array(1, fill_value=4321)
dm.replace(data)
self.assertIsNone(dm.fill_value)
def test_lazy_with_lazy__fill_value(self):
fill_value = 1234
dm1 = DataManager(as_lazy_data(self.real_array),
fill_value=fill_value)
dm2 = DataManager(as_lazy_data(self.real_array),
fill_value=fill_value)
self.assertEqual(dm1, dm2)
def test_zero_ndim_lazy_to_scalar_1d_real_promote(self):
real_array = np.array([456])
dm = DataManager(as_lazy_data(np.array(123)))
self.assertTrue(dm.has_lazy_data())
dm.data = real_array
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, real_array)
def test(self):
dm = DataManager(np.array(0))
kwargs = dict(data=self.data, fill_value='none', realised_dtype='none')
with mock.patch(self.method) as mocker:
mocker.return_value = self.return_value
result = dm.copy(data=self.data)
mocker.assert_called_once_with(self.memo, **kwargs)
self.assertIs(result, self.return_value)
def test_real_with_real__realised_dtype_failure(self):
dm = DataManager(self.real_array)
emsg = 'Cannot set realised dtype, no lazy data is available.'
with self.assertRaisesRegexp(ValueError, emsg):
dm.replace(self.real_array * 10,
realised_dtype=np.dtype('int16'))
self.assertIs(dm._real_array, self.real_array)
self.assertArrayEqual(dm.data, self.real_array)
def test_real_with_real__promote_shape_with_dtype_failure(self):
data = np.array(666)
dm = DataManager(data)
emsg = 'Cannot set realised dtype, no lazy data is available'
with self.assertRaisesRegexp(ValueError, emsg):
dm.replace(np.array([999]),
realised_dtype=np.dtype('float32'))
self.assertArrayEqual(dm.data, data)
def test(self):
dm = DataManager(np.array(0))
kwargs = dict(data=self.data, fill_value='none', realised_dtype='none')
with mock.patch(self.method) as mocker:
mocker.return_value = self.return_value
result = dm.copy(data=self.data)
mocker.assert_called_once_with(self.memo, **kwargs)
self.assertIs(result, self.return_value)
def test_lazy_with_lazy(self):
lazy_array = as_lazy_data(self.real_array)
dm = DataManager(lazy_array * 10)
self.assertTrue(dm.has_lazy_data())
dm.replace(lazy_array)
self.assertTrue(dm.has_lazy_data())
self.assertIs(dm._lazy_array, lazy_array)
self.assertArrayEqual(dm.data, self.real_array)
def test_ndim_nd(self):
shape = (2, 3, 4)
real_array = np.arange(24).reshape(shape)
dm = DataManager(real_array)
self.assertEqual(dm.ndim, len(shape))
lazy_array = as_lazy_data(real_array)
dm = DataManager(lazy_array)
self.assertEqual(dm.ndim, len(shape))
def setUp(self):
self.real_array = np.array(0)
self.mask_array = ma.array(0)
np_fill_value = ma.array(0, dtype=self.mask_array.dtype).fill_value
assert np_fill_value == self.mask_array.fill_value
self.np_fill_value = np_fill_value
self.dm = DataManager(self.mask_array)
self.dm.fill_value = None
def test_fill_value_uint(self):
fill_value = np.uint(1234)
for dtype in self.dtypes:
data = np.array([0], dtype=dtype)
dm = DataManager(data)
dm.fill_value = fill_value
[expected] = np.array([fill_value], dtype=dtype)
self.assertEqual(dm.fill_value, expected)
self.assertEqual(dm.fill_value.dtype, dtype)
def test_with_real_masked_constant(self):
masked_data = ma.masked_array([666], mask=True, dtype=np.dtype('f8'))
masked_constant = masked_data[0]
dm = DataManager(masked_constant)
result = dm.data
self.assertFalse(dm.has_lazy_data())
self.assertIsInstance(result, ma.MaskedArray)
self.assertNotIsInstance(result, ma.core.MaskedConstant)
self.assertMaskedArrayEqual(result, masked_data)
def test_nd_real_to_nd_real(self):
shape = (2, 3, 4)
size = np.prod(shape)
real_array = np.arange(size).reshape(shape)
dm = DataManager(real_array * 10)
self.assertFalse(dm.has_lazy_data())
dm.data = real_array
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, real_array)
def test_with_lazy_mask_array__masked(self):
dm = DataManager(self.lazy_mask_array_masked)
self.assertTrue(dm.has_lazy_data())
result = dm.data
self.assertFalse(dm.has_lazy_data())
self.assertIsInstance(result, ma.MaskedArray)
self.assertEqual(dm.dtype, self.dtype)
self.assertEqual(result.fill_value, self.fill_value)
self.assertArrayEqual(result, self.mask_array_masked)
def test_lazy_with_realised_dtype(self):
dm = DataManager(as_lazy_data(self.real_array),
realised_dtype=np.dtype('int16'))
data = as_lazy_data(self.real_array) * 10
dtype = np.dtype('int8')
result = dm._deepcopy(self.memo, data=data, realised_dtype=dtype)
expected = DataManager(data, realised_dtype=dtype)
self.assertEqual(result, expected)
self.assertIs(result._lazy_array, data)
def test_lazy_with_realised_dtype(self):
dm = DataManager(as_lazy_data(self.real_array),
realised_dtype=np.dtype('int16'))
data = as_lazy_data(self.real_array) * 10
dtype = np.dtype('int8')
result = dm._deepcopy(self.memo, data=data, realised_dtype=dtype)
expected = DataManager(data, realised_dtype=dtype)
self.assertEqual(result, expected)
self.assertIs(result._lazy_array, data)
def test_fill_value_uint(self):
fill_value = np.uint(1234)
for dtype in self.dtypes:
data = np.array([0], dtype=dtype)
dm = DataManager(data)
dm.fill_value = fill_value
[expected] = np.array([fill_value], dtype=dtype)
self.assertEqual(dm.fill_value, expected)
self.assertEqual(dm.fill_value.dtype, dtype)
def test_nd_real_to_nd_real(self):
shape = (2, 3, 4)
size = np.prod(shape)
real_array = np.arange(size).reshape(shape)
dm = DataManager(real_array * 10)
self.assertFalse(dm.has_lazy_data())
dm.data = real_array
self.assertFalse(dm.has_lazy_data())
self.assertArrayEqual(dm.data, real_array)
def test_with_real_masked_constant(self):
masked_data = ma.masked_array([666], mask=True, dtype=np.dtype('f8'))
masked_constant = masked_data[0]
dm = DataManager(masked_constant)
result = dm.data
self.assertFalse(dm.has_lazy_data())
self.assertIsInstance(result, ma.MaskedArray)
self.assertNotIsInstance(result, ma.core.MaskedConstant)
self.assertMaskedArrayEqual(result, masked_data)
def test_with_lazy_mask_array__masked(self):
dm = DataManager(self.lazy_mask_array_masked)
self.assertTrue(dm.has_lazy_data())
result = dm.data
self.assertFalse(dm.has_lazy_data())
self.assertIsInstance(result, ma.MaskedArray)
self.assertEqual(dm.dtype, self.dtype)
self.assertEqual(result.fill_value, self.fill_value)
self.assertArrayEqual(result, self.mask_array_masked)
def test_with_lazy_mask_array__masked(self):
dm = DataManager(self.lazy_mask_array_masked,
realised_dtype=self.realised_dtype)
self.assertTrue(dm.has_lazy_data())
result = dm.data
self.assertFalse(dm.has_lazy_data())
self.assertIsInstance(result, ma.MaskedArray)
self.assertIsNone(dm._realised_dtype)
self.assertEqual(dm.dtype, self.realised_dtype)
self.assertArrayEqual(result, self.lazy_mask_array_masked.compute())
def test_coerce_to_ndarray(self):
shape = (2, 3)
size = np.prod(shape)
real_array = np.arange(size).reshape(shape)
matrix = np.matrix(real_array)
dm = DataManager(real_array)
dm.data = matrix
self.assertIsInstance(dm._real_array, np.core.ndarray)
self.assertIsInstance(dm.data, np.core.ndarray)
self.assertArrayEqual(dm.data, real_array)
def test_nd_lazy_to_nd_lazy(self):
shape = (2, 3, 4)
size = np.prod(shape)
real_array = np.arange(size).reshape(shape)
lazy_array = as_lazy_data(real_array)
dm = DataManager(lazy_array * 10)
self.assertTrue(dm.has_lazy_data())
dm.data = lazy_array
self.assertTrue(dm.has_lazy_data())
self.assertArrayEqual(dm.data, lazy_array.compute())
def test_coerce_to_ndarray(self):
shape = (2, 3)
size = np.prod(shape)
real_array = np.arange(size).reshape(shape)
matrix = np.matrix(real_array)
dm = DataManager(real_array)
dm.data = matrix
self.assertIsInstance(dm._real_array, np.core.ndarray)
self.assertIsInstance(dm.data, np.core.ndarray)
self.assertArrayEqual(dm.data, real_array)
def test_fill_value_float(self):
fill_value = np.float(123.4)
for dtype in self.dtypes:
data = np.array([0], dtype=dtype)
dm = DataManager(data)
dm.fill_value = fill_value
if dtype.kind in 'biu':
fill_value = np.rint(fill_value)
[expected] = np.asarray([fill_value], dtype=dtype)
self.assertEqual(dm.fill_value, expected)
self.assertEqual(dm.fill_value.dtype, dtype)
def test_with_lazy_masked_constant(self):
dtype = np.dtype('int16')
masked_data = ma.masked_array([666], mask=True, dtype=dtype)
masked_constant = masked_data[0]
lazy_masked_constant = as_lazy_data(masked_constant)
dm = DataManager(lazy_masked_constant, realised_dtype=dtype)
self.assertEqual(dm.dtype, dtype)
result = dm.data
self.assertFalse(dm.has_lazy_data())
self.assertIsInstance(result, ma.MaskedArray)
self.assertNotIsInstance(result, ma.core.MaskedConstant)
self.assertMaskedArrayEqual(result, masked_data)
def test_realised_dtype_none(self):
shape = (2, 3, 4)
size = np.prod(shape)
mask_array = ma.arange(size).reshape(shape)
dtype = mask_array.dtype
lazy_array = as_lazy_data(mask_array)
dm = DataManager(lazy_array, realised_dtype=dtype)
self.assertIsNone(dm._realised_dtype)
self.assertEqual(dm.dtype, dtype)
dm.data = mask_array
self.assertIs(dm.data, mask_array)
self.assertIsNone(dm._realised_dtype)
self.assertEqual(dm.dtype, dtype)
