def test_iter_regridded_fields_problem_bounds(self): """Test a dataset with crap bounds will work when with_corners is False.""" dst = self.test_data.get_rd("cancm4_tas").get()[:, :, :, 20:25, 30:35] dst.spatial.crs = Spherical() src = deepcopy(dst[0, 0, 0, :, :]) egrid_dst = get_esmf_grid_from_sdim(dst.spatial) egrid_src = get_esmf_grid_from_sdim(src.spatial) self.assertEqual(egrid_dst.mask[0].sum(), 25) self.assertEqual(egrid_src.mask[0].sum(), 25) self.assertNumpyAll(egrid_dst.coords[0][0], egrid_src.coords[0][0]) self.assertNumpyAll(egrid_dst.coords[0][1], egrid_src.coords[0][1]) ret = list(iter_esmf_fields(src)) self.assertEqual(len(ret), 1) variable_alias, efield, tidx = ret[0] self.assertNumpyAll(efield.grid.coords[0][0], dst.spatial.grid.value.data[1]) self.assertNumpyAll(efield.grid.coords[0][1], dst.spatial.grid.value.data[0]) ret = list(iter_regridded_fields([src], dst, with_corners=False))[0] actual = dst[0, 0, 0, :, :].variables.first().value to_test = ret.variables.first().value self.assertFalse(to_test.mask.any()) self.assertNumpyAllClose(to_test.data, actual.data) self.assertNumpyAll(to_test.mask, actual.mask)
def test_iter_esmf_fields(self): ofield = self.get_ofield() # add a unique spatial mask and value mask mask = ofield.spatial.get_mask() mask[1, 1] = True ofield.spatial.set_mask(mask) for variable in ofield.variables.itervalues(): variable.value.mask[:, :, :, :, :] = mask.copy() dtype = [("tidx", int), ("variable_alias", object), ("efield", object)] fill = np.array([], dtype=dtype) for row in iter_esmf_fields(ofield, with_corners=True, value_mask=None): app = np.zeros(1, dtype=dtype) app[0] = row fill = np.append(fill, app) self.assertEqual(fill.shape[0], 122) self.assertEqual(np.unique(fill[0:61]["variable_alias"]), "foo") self.assertEqual(np.unique(fill[61:]["variable_alias"]), "foo2") means = np.array([element.data.mean() for element in fill["efield"]]) self.assertTrue(np.all(means[0:61] == 2.5)) self.assertTrue(np.all(means[61:] == 30)) for idx in range(fill.shape[0]): efield = fill["efield"][idx] egrid = efield.grid egrid_mask = np.invert(egrid.mask[0].astype(bool)) self.assertNumpyAll(egrid_mask, mask)
def test_iter_esmf_fields(self): ofield = self.get_ofield() # Add a unique spatial and value mask. mask = ofield.grid.get_mask(create=True) mask[1, 1] = True ofield.grid.set_mask(mask) dtype = [('variable_alias', object), ('efield', object), ('tidx', object)] fill = np.array([], dtype=dtype) from ocgis.regrid.base import iter_esmf_fields for row in iter_esmf_fields(ofield, value_mask=None, split=False): app = np.zeros(1, dtype=dtype) app[0] = row fill = np.append(fill, app) # Without splitting, the time index is a single slice. self.assertTrue(all([ii == slice(None) for ii in fill['tidx']])) # There are two variables. self.assertEqual(fill.shape[0], 2) self.assertEqual(np.unique(fill[0]['variable_alias']), 'foo') self.assertEqual(np.unique(fill[1]['variable_alias']), 'foo2') # Test data is properly attributed to the correct variable. means = np.array([element.data.mean() for element in fill['efield']]) self.assertTrue(np.all(means[0] == 2.5)) self.assertTrue(np.all(means[1] == 30)) for idx in range(fill.shape[0]): efield = fill[idx]['efield'] # Test all time slices are accounted for. self.assertEqual(efield.data.shape[-1], ofield.time.shape[0]) # Test masks are equivalent. ESMF masks have "1" for false requiring the invert operation. egrid = efield.grid egrid_mask = np.invert(egrid.mask[0].astype(bool)) self.assertNumpyAll(egrid_mask, mask) # Test splitting the data. ofield = self.get_ofield() res = list(iter_esmf_fields(ofield)) self.assertEqual(len(res), ofield.time.shape[0] * len(ofield.data_variables))
def test_iter_esmf_fields(self): ofield = self.get_ofield() # Add a unique spatial and value mask. mask = ofield.spatial.get_mask() mask[1, 1] = True ofield.spatial.set_mask(mask) for variable in ofield.variables.itervalues(): variable.value.mask[:, :, :, :, :] = mask.copy() dtype = [("variable_alias", object), ("efield", object), ("tidx", object)] fill = np.array([], dtype=dtype) for row in iter_esmf_fields(ofield, with_corners=True, value_mask=None, split=False): app = np.zeros(1, dtype=dtype) app[0] = row fill = np.append(fill, app) # Without splitting, the time index is a single slice. self.assertTrue(all([ii == slice(None) for ii in fill["tidx"]])) # There are two variables. self.assertEqual(fill.shape[0], 2) self.assertEqual(np.unique(fill[0]["variable_alias"]), "foo") self.assertEqual(np.unique(fill[1]["variable_alias"]), "foo2") # Test data is properly attributed to the correct variable. means = np.array([element.data.mean() for element in fill["efield"]]) self.assertTrue(np.all(means[0] == 2.5)) self.assertTrue(np.all(means[1] == 30)) for idx in range(fill.shape[0]): efield = fill[idx]["efield"] # Test all time slices are accounted for. self.assertEqual(efield.data.shape[0], ofield.shape[1]) # Test masks are equivalent. ESMF masks have "1" for false requiring the invert operation. egrid = efield.grid egrid_mask = np.invert(egrid.mask[0].astype(bool)) self.assertNumpyAll(egrid_mask, mask) # Test splitting the data. ofield = self.get_ofield() res = list(iter_esmf_fields(ofield)) self.assertEqual(len(res), ofield.shape[1] * len(ofield.variables))
def test_iter_esmf_fields(self): ofield = self.get_ofield() # Add a unique spatial and value mask. mask = ofield.grid.get_mask(create=True) mask[1, 1] = True ofield.grid.set_mask(mask) dtype = [('variable_alias', object), ('efield', object), ('tidx', object)] fill = np.array([], dtype=dtype) from ocgis.regrid.base import iter_esmf_fields for row in iter_esmf_fields(ofield, value_mask=None, split=False): app = np.zeros(1, dtype=dtype) app[0] = row fill = np.append(fill, app) # Without splitting, the time index is a single slice. self.assertTrue(all([ii is None for ii in fill['tidx']])) # There are two variables. self.assertEqual(fill.shape[0], 2) self.assertEqual(np.unique(fill[0]['variable_alias']), 'foo') self.assertEqual(np.unique(fill[1]['variable_alias']), 'foo2') # Test data is properly attributed to the correct variable. means = np.array([element.data.mean() for element in fill['efield']]) self.assertTrue(np.all(means[0] == 2.5)) self.assertTrue(np.all(means[1] == 30)) for idx in range(fill.shape[0]): efield = fill[idx]['efield'] # Test all time slices are accounted for. self.assertEqual(efield.data.shape[-1], ofield.time.shape[0]) # Test masks are equivalent. ESMF masks have "1" for false requiring the invert operation. egrid = efield.grid egrid_mask = np.invert(egrid.mask[0].astype(bool)) self.assertNumpyAll(egrid_mask, mask) # Test splitting the data. ofield = self.get_ofield() res = list(iter_esmf_fields(ofield)) self.assertEqual(len(res), ofield.time.shape[0] * len(ofield.data_variables))
def test_system(self): from ocgis.regrid.base import create_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects import ESMF yc = Variable(name='yc', value=np.arange(-90 + (45 / 2.), 90, 45), dimensions='ydim', dtype=float) xc = Variable(name='xc', value=np.arange(15, 360, 30), dimensions='xdim', dtype=float) ogrid = Grid(y=yc, x=xc, crs=Spherical()) ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds') np.random.seed(1) mask = np.random.rand(*ogrid.shape) mask = mask > 0.5 self.assertTrue(mask.sum() > 3) ogrid.set_mask(mask) egrid = create_esmf_grid(ogrid) actual_shape = egrid.size[0].tolist() desired_shape = np.flipud(ogrid.shape).tolist() self.assertEqual(actual_shape, desired_shape) desired = ogrid.get_value_stacked() desired = np.ma.array(desired, mask=False) desired.mask[0, :, :] = ogrid.get_mask() desired.mask[1, :, :] = ogrid.get_mask() desired = desired.sum() actual_col = egrid.get_coords(0) actual_row = egrid.get_coords(1) actual_mask = np.invert(egrid.mask[0].astype(bool)) actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array(actual_col, mask=actual_mask).sum() self.assertEqual(actual, desired) desired = 9900.0 corners = egrid.coords[ESMF.StaggerLoc.CORNER] actual = corners[0].sum() + corners[1].sum() self.assertEqual(actual, desired) ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical()) variable_name, efield, tidx = list(iter_esmf_fields(ofield, split=False))[0] desired_value = ofield['data'].get_value() self.assertAlmostEqual(efield.data.sum(), desired_value.sum(), places=3) destroy_esmf_objects([egrid, efield]) ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True) desired_value = ofield['data'].get_masked_value() keywords = dict(split=[False, True]) for k in self.iter_product_keywords(keywords): opts = {'split': k.split} dofield = ofield.deepcopy() dofield['data'].get_value().fill(0) ro = RegridOperation(ofield, dofield, regrid_options=opts) actual_field = ro.execute() actual_value = actual_field['data'].get_masked_value() self.assertAlmostEqual(0.0, np.abs(desired_value - actual_value).max())
def test_system(self): from ocgis.regrid.base import get_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects import ESMF yc = Variable(name='yc', value=np.arange(-90 + (45 / 2.), 90, 45), dimensions='ydim', dtype=float) xc = Variable(name='xc', value=np.arange(15, 360, 30), dimensions='xdim', dtype=float) ogrid = Grid(y=yc, x=xc, crs=Spherical()) ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds') np.random.seed(1) mask = np.random.rand(*ogrid.shape) mask = mask > 0.5 self.assertTrue(mask.sum() > 3) ogrid.set_mask(mask) egrid = get_esmf_grid(ogrid) actual_shape = egrid.size[0].tolist() desired_shape = np.flipud(ogrid.shape).tolist() self.assertEqual(actual_shape, desired_shape) desired = ogrid.get_value_stacked() desired = np.ma.array(desired, mask=False) desired.mask[0, :, :] = ogrid.get_mask() desired.mask[1, :, :] = ogrid.get_mask() desired = desired.sum() actual_col = egrid.get_coords(0) actual_row = egrid.get_coords(1) actual_mask = np.invert(egrid.mask[0].astype(bool)) actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array( actual_col, mask=actual_mask).sum() self.assertEqual(actual, desired) desired = 9900.0 corners = egrid.coords[ESMF.StaggerLoc.CORNER] actual = corners[0].sum() + corners[1].sum() self.assertEqual(actual, desired) ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical()) variable_name, efield, tidx = list( iter_esmf_fields(ofield, split=False))[0] desired_value = ofield['data'].get_value() self.assertAlmostEqual(efield.data.sum(), desired_value.sum(), places=3) destroy_esmf_objects([egrid, efield]) ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True) desired_value = ofield['data'].get_masked_value() keywords = dict(split=[False, True]) for k in self.iter_product_keywords(keywords): opts = {'split': k.split} dofield = ofield.deepcopy() dofield['data'].get_value().fill(0) ro = RegridOperation(ofield, dofield, regrid_options=opts) actual_field = ro.execute() actual_value = actual_field['data'].get_masked_value() self.assertAlmostEqual(0.0, np.abs(desired_value - actual_value).max())