def test_iter_regridded_fields_problem_bounds(self):
"""Test a dataset with crap bounds will work when with_corners is False."""
dst = self.test_data.get_rd("cancm4_tas").get()[:, :, :, 20:25, 30:35]
dst.spatial.crs = Spherical()
src = deepcopy(dst[0, 0, 0, :, :])
egrid_dst = get_esmf_grid_from_sdim(dst.spatial)
egrid_src = get_esmf_grid_from_sdim(src.spatial)
self.assertEqual(egrid_dst.mask[0].sum(), 25)
self.assertEqual(egrid_src.mask[0].sum(), 25)
self.assertNumpyAll(egrid_dst.coords[0][0], egrid_src.coords[0][0])
self.assertNumpyAll(egrid_dst.coords[0][1], egrid_src.coords[0][1])
ret = list(iter_esmf_fields(src))
self.assertEqual(len(ret), 1)
variable_alias, efield, tidx = ret[0]
self.assertNumpyAll(efield.grid.coords[0][0], dst.spatial.grid.value.data[1])
self.assertNumpyAll(efield.grid.coords[0][1], dst.spatial.grid.value.data[0])
ret = list(iter_regridded_fields([src], dst, with_corners=False))[0]
actual = dst[0, 0, 0, :, :].variables.first().value
to_test = ret.variables.first().value
self.assertFalse(to_test.mask.any())
self.assertNumpyAllClose(to_test.data, actual.data)
self.assertNumpyAll(to_test.mask, actual.mask)
def test_iter_esmf_fields(self):
ofield = self.get_ofield()
# add a unique spatial mask and value mask
mask = ofield.spatial.get_mask()
mask[1, 1] = True
ofield.spatial.set_mask(mask)
for variable in ofield.variables.itervalues():
variable.value.mask[:, :, :, :, :] = mask.copy()
dtype = [("tidx", int), ("variable_alias", object), ("efield", object)]
fill = np.array([], dtype=dtype)
for row in iter_esmf_fields(ofield, with_corners=True, value_mask=None):
app = np.zeros(1, dtype=dtype)
app[0] = row
fill = np.append(fill, app)
self.assertEqual(fill.shape[0], 122)
self.assertEqual(np.unique(fill[0:61]["variable_alias"]), "foo")
self.assertEqual(np.unique(fill[61:]["variable_alias"]), "foo2")
means = np.array([element.data.mean() for element in fill["efield"]])
self.assertTrue(np.all(means[0:61] == 2.5))
self.assertTrue(np.all(means[61:] == 30))
for idx in range(fill.shape[0]):
efield = fill["efield"][idx]
egrid = efield.grid
egrid_mask = np.invert(egrid.mask[0].astype(bool))
self.assertNumpyAll(egrid_mask, mask)
def test_iter_esmf_fields(self):
ofield = self.get_ofield()
# Add a unique spatial and value mask.
mask = ofield.grid.get_mask(create=True)
mask[1, 1] = True
ofield.grid.set_mask(mask)
dtype = [('variable_alias', object), ('efield', object),
('tidx', object)]
fill = np.array([], dtype=dtype)
from ocgis.regrid.base import iter_esmf_fields
for row in iter_esmf_fields(ofield, value_mask=None, split=False):
app = np.zeros(1, dtype=dtype)
app[0] = row
fill = np.append(fill, app)
# Without splitting, the time index is a single slice.
self.assertTrue(all([ii == slice(None) for ii in fill['tidx']]))
# There are two variables.
self.assertEqual(fill.shape[0], 2)
self.assertEqual(np.unique(fill[0]['variable_alias']), 'foo')
self.assertEqual(np.unique(fill[1]['variable_alias']), 'foo2')
# Test data is properly attributed to the correct variable.
means = np.array([element.data.mean() for element in fill['efield']])
self.assertTrue(np.all(means[0] == 2.5))
self.assertTrue(np.all(means[1] == 30))
for idx in range(fill.shape[0]):
efield = fill[idx]['efield']
# Test all time slices are accounted for.
self.assertEqual(efield.data.shape[-1], ofield.time.shape[0])
# Test masks are equivalent. ESMF masks have "1" for false requiring the invert operation.
egrid = efield.grid
egrid_mask = np.invert(egrid.mask[0].astype(bool))
self.assertNumpyAll(egrid_mask, mask)
# Test splitting the data.
ofield = self.get_ofield()
res = list(iter_esmf_fields(ofield))
self.assertEqual(len(res),
ofield.time.shape[0] * len(ofield.data_variables))
def test_iter_esmf_fields(self):
ofield = self.get_ofield()
# Add a unique spatial and value mask.
mask = ofield.spatial.get_mask()
mask[1, 1] = True
ofield.spatial.set_mask(mask)
for variable in ofield.variables.itervalues():
variable.value.mask[:, :, :, :, :] = mask.copy()
dtype = [("variable_alias", object), ("efield", object), ("tidx", object)]
fill = np.array([], dtype=dtype)
for row in iter_esmf_fields(ofield, with_corners=True, value_mask=None, split=False):
app = np.zeros(1, dtype=dtype)
app[0] = row
fill = np.append(fill, app)
# Without splitting, the time index is a single slice.
self.assertTrue(all([ii == slice(None) for ii in fill["tidx"]]))
# There are two variables.
self.assertEqual(fill.shape[0], 2)
self.assertEqual(np.unique(fill[0]["variable_alias"]), "foo")
self.assertEqual(np.unique(fill[1]["variable_alias"]), "foo2")
# Test data is properly attributed to the correct variable.
means = np.array([element.data.mean() for element in fill["efield"]])
self.assertTrue(np.all(means[0] == 2.5))
self.assertTrue(np.all(means[1] == 30))
for idx in range(fill.shape[0]):
efield = fill[idx]["efield"]
# Test all time slices are accounted for.
self.assertEqual(efield.data.shape[0], ofield.shape[1])
# Test masks are equivalent. ESMF masks have "1" for false requiring the invert operation.
egrid = efield.grid
egrid_mask = np.invert(egrid.mask[0].astype(bool))
self.assertNumpyAll(egrid_mask, mask)
# Test splitting the data.
ofield = self.get_ofield()
res = list(iter_esmf_fields(ofield))
self.assertEqual(len(res), ofield.shape[1] * len(ofield.variables))
def test_iter_esmf_fields(self):
ofield = self.get_ofield()
# Add a unique spatial and value mask.
mask = ofield.grid.get_mask(create=True)
mask[1, 1] = True
ofield.grid.set_mask(mask)
dtype = [('variable_alias', object), ('efield', object), ('tidx', object)]
fill = np.array([], dtype=dtype)
from ocgis.regrid.base import iter_esmf_fields
for row in iter_esmf_fields(ofield, value_mask=None, split=False):
app = np.zeros(1, dtype=dtype)
app[0] = row
fill = np.append(fill, app)
# Without splitting, the time index is a single slice.
self.assertTrue(all([ii is None for ii in fill['tidx']]))
# There are two variables.
self.assertEqual(fill.shape[0], 2)
self.assertEqual(np.unique(fill[0]['variable_alias']), 'foo')
self.assertEqual(np.unique(fill[1]['variable_alias']), 'foo2')
# Test data is properly attributed to the correct variable.
means = np.array([element.data.mean() for element in fill['efield']])
self.assertTrue(np.all(means[0] == 2.5))
self.assertTrue(np.all(means[1] == 30))
for idx in range(fill.shape[0]):
efield = fill[idx]['efield']
# Test all time slices are accounted for.
self.assertEqual(efield.data.shape[-1], ofield.time.shape[0])
# Test masks are equivalent. ESMF masks have "1" for false requiring the invert operation.
egrid = efield.grid
egrid_mask = np.invert(egrid.mask[0].astype(bool))
self.assertNumpyAll(egrid_mask, mask)
# Test splitting the data.
ofield = self.get_ofield()
res = list(iter_esmf_fields(ofield))
self.assertEqual(len(res), ofield.time.shape[0] * len(ofield.data_variables))
def test_system(self):
from ocgis.regrid.base import create_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects
import ESMF
yc = Variable(name='yc', value=np.arange(-90 + (45 / 2.), 90, 45), dimensions='ydim', dtype=float)
xc = Variable(name='xc', value=np.arange(15, 360, 30), dimensions='xdim', dtype=float)
ogrid = Grid(y=yc, x=xc, crs=Spherical())
ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds')
np.random.seed(1)
mask = np.random.rand(*ogrid.shape)
mask = mask > 0.5
self.assertTrue(mask.sum() > 3)
ogrid.set_mask(mask)
egrid = create_esmf_grid(ogrid)
actual_shape = egrid.size[0].tolist()
desired_shape = np.flipud(ogrid.shape).tolist()
self.assertEqual(actual_shape, desired_shape)
desired = ogrid.get_value_stacked()
desired = np.ma.array(desired, mask=False)
desired.mask[0, :, :] = ogrid.get_mask()
desired.mask[1, :, :] = ogrid.get_mask()
desired = desired.sum()
actual_col = egrid.get_coords(0)
actual_row = egrid.get_coords(1)
actual_mask = np.invert(egrid.mask[0].astype(bool))
actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array(actual_col, mask=actual_mask).sum()
self.assertEqual(actual, desired)
desired = 9900.0
corners = egrid.coords[ESMF.StaggerLoc.CORNER]
actual = corners[0].sum() + corners[1].sum()
self.assertEqual(actual, desired)
ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical())
variable_name, efield, tidx = list(iter_esmf_fields(ofield, split=False))[0]
desired_value = ofield['data'].get_value()
self.assertAlmostEqual(efield.data.sum(), desired_value.sum(), places=3)
destroy_esmf_objects([egrid, efield])
ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True)
desired_value = ofield['data'].get_masked_value()
keywords = dict(split=[False, True])
for k in self.iter_product_keywords(keywords):
opts = {'split': k.split}
dofield = ofield.deepcopy()
dofield['data'].get_value().fill(0)
ro = RegridOperation(ofield, dofield, regrid_options=opts)
actual_field = ro.execute()
actual_value = actual_field['data'].get_masked_value()
self.assertAlmostEqual(0.0, np.abs(desired_value - actual_value).max())
def test_system(self):
from ocgis.regrid.base import get_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects
import ESMF
yc = Variable(name='yc',
value=np.arange(-90 + (45 / 2.), 90, 45),
dimensions='ydim',
dtype=float)
xc = Variable(name='xc',
value=np.arange(15, 360, 30),
dimensions='xdim',
dtype=float)
ogrid = Grid(y=yc, x=xc, crs=Spherical())
ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds')
np.random.seed(1)
mask = np.random.rand(*ogrid.shape)
mask = mask > 0.5
self.assertTrue(mask.sum() > 3)
ogrid.set_mask(mask)
egrid = get_esmf_grid(ogrid)
actual_shape = egrid.size[0].tolist()
desired_shape = np.flipud(ogrid.shape).tolist()
self.assertEqual(actual_shape, desired_shape)
desired = ogrid.get_value_stacked()
desired = np.ma.array(desired, mask=False)
desired.mask[0, :, :] = ogrid.get_mask()
desired.mask[1, :, :] = ogrid.get_mask()
desired = desired.sum()
actual_col = egrid.get_coords(0)
actual_row = egrid.get_coords(1)
actual_mask = np.invert(egrid.mask[0].astype(bool))
actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array(
actual_col, mask=actual_mask).sum()
self.assertEqual(actual, desired)
desired = 9900.0
corners = egrid.coords[ESMF.StaggerLoc.CORNER]
actual = corners[0].sum() + corners[1].sum()
self.assertEqual(actual, desired)
ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical())
variable_name, efield, tidx = list(
iter_esmf_fields(ofield, split=False))[0]
desired_value = ofield['data'].get_value()
self.assertAlmostEqual(efield.data.sum(),
desired_value.sum(),
places=3)
destroy_esmf_objects([egrid, efield])
ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True)
desired_value = ofield['data'].get_masked_value()
keywords = dict(split=[False, True])
for k in self.iter_product_keywords(keywords):
opts = {'split': k.split}
dofield = ofield.deepcopy()
dofield['data'].get_value().fill(0)
ro = RegridOperation(ofield, dofield, regrid_options=opts)
actual_field = ro.execute()
actual_value = actual_field['data'].get_masked_value()
self.assertAlmostEqual(0.0,
np.abs(desired_value - actual_value).max())
