def _get_regridded_field_with_subset_(self, sfield, subset_field_for_regridding=None): """ Regrid ``sfield`` subsetting the regrid destination in the process. :param sfield: The input field to regrid. :type sfield: :class:`ocgis.Field` :param subset_field_for_regridding: The original, unaltered spatial dimension to use for subsetting. :type subset_field_for_regridding: :class:`ocgis.Field` :rtype: :class:`~ocgis.Field` """ from ocgis.regrid.base import RegridOperation ocgis_lh(logger=self._subset_log, msg='Starting regrid operation...', level=logging.INFO) ro = RegridOperation(sfield, self.ops.regrid_destination, subset_field=subset_field_for_regridding, regrid_options=self.ops.regrid_options) sfield = ro.execute() ocgis_lh(logger=self._subset_log, msg='Regrid operation complete.', level=logging.INFO) return sfield
def test_system_masking_with_smm(self): """Test masking with sparse matrix multiplication.""" from ocgis.regrid import RegridOperation grid = create_gridxy_global(with_bounds=False, crs=Spherical(), dist_dimname='x', resolution=5.0) src_field = create_exact_field(grid, 'exact', ntime=3) mask = src_field.grid.get_mask(create=True) mask[0:2, :] = True mask[:, -2:] = True mask[-2:, :] = True mask[:, 0:2] = True src_field.grid.set_mask(mask, cascade=True) src_field['exact'].set_value(src_field['exact'].mv().filled()) dst_field = deepcopy(src_field) dst_field.remove_variable('exact') weights = self.get_temporary_file_path('weights.nc', collective=True) weights = vm.bcast(weights) ro = RegridOperation(src_field, dst_field, regrid_options={'weights_out': weights, 'split': False}) _ = ro.execute() ro2 = RegridOperation(src_field, dst_field, regrid_options={'weights_in': weights, 'split': True}) result = ro2.execute() actual = result['exact'].mv() desired = src_field['exact'].mv() self.assertNumpyAllClose(actual, desired)
def test_combo_global_grid_combinations(self): """Test regridding with different global grid configurations.""" boolmix = [[True, True], [False, False], [True, False], [False, True]] keywords = dict(with_bounds=boolmix, wrapped=boolmix, resolution=[[3.0, 6.0], [6.0, 3.0], [3.0, 3.0]]) for ctr, k in enumerate(self.iter_product_keywords(keywords)): # print ctr, k # if ctr != 10: continue source_grid = self.get_spherical_global_grid( with_bounds=k.with_bounds[0], wrapped=k.wrapped[0], resolution=k.resolution[0] ) destination_grid = self.get_spherical_global_grid( with_bounds=k.with_bounds[1], wrapped=k.wrapped[1], resolution=k.resolution[1] ) source_field = self.get_regridding_field(source_grid, "source") destination_field = self.get_regridding_field(destination_grid, "destination") ro = RegridOperation(source_field, destination_field) res = ro.execute() actual = res.variables["source"].value desired = destination_field.variables["destination"].value try: self.assertNumpyAllClose(actual, desired) except AssertionError: # Without bounds, some data will be unmapped and masked around the exterior of the grid if the source # resolution is higher than the destination. self.assertEqual(k.resolution, [6.0, 3.0]) self.assertEqual(actual.mask.sum(), 356) self.assertTrue(np.all(np.isclose(actual.compressed(), 15.0)))
def test_system_global_grid_combinations(self): """Test regridding with different global grid configurations.""" from ocgis.regrid.base import RegridOperation boolmix = [[True, True], [False, False], [True, False], [False, True]] keywords = dict(with_bounds=boolmix, wrapped=boolmix, resolution=[[3.0, 6.0], [6.0, 3.0], [3.0, 3.0]]) for ctr, k in enumerate(self.iter_product_keywords(keywords)): # print ctr, k # if ctr != 10: continue source_grid = self.get_gridxy_global(with_bounds=k.with_bounds[0], wrapped=k.wrapped[0], resolution=k.resolution[0]) destination_grid = self.get_gridxy_global( with_bounds=k.with_bounds[1], wrapped=k.wrapped[1], resolution=k.resolution[1]) source_field = self.create_regridding_field(source_grid, 'source') destination_field = self.create_regridding_field( destination_grid, 'destination') ro = RegridOperation(source_field, destination_field) res = ro.execute() actual = res['source'].get_masked_value() targets = [actual.min(), actual.mean(), actual.max()] for t in targets: self.assertAlmostEqual(t, 15.0, places=5)
def smm(index_path, wd=None, data_variables='auto'): """ Run a chunked sparse matrix multiplication. :param str index_path: Path to chunked operation's index file. :param str wd: Path to the directory containing the chunk files. :param list data_variables: Optional list of data variables. Otherwise, auto-discovery is used. """ # Assume the current working directory if wd is None: wd = '' # Turn on auto-discovery if data_variables == 'auto': v = None else: v = data_variables # Get the attribute host index_field = RequestDataset(index_path).get() gs_index_v = index_field[GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE] # Collect the source filenames src_filenames = gs_index_v.attrs[GridChunkerConstants.IndexFile.NAME_SOURCE_VARIABLE] src_filenames = index_field[src_filenames] src_filenames = src_filenames.join_string_value() # Collect the destination file names dst_filenames = gs_index_v.attrs[GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE] dst_filenames = index_field[dst_filenames] dst_filenames = dst_filenames.join_string_value() # Collect the weight filenames wgt_filenames = gs_index_v.attrs[GridChunkerConstants.IndexFile.NAME_WEIGHTS_VARIABLE] wgt_filenames = index_field[wgt_filenames] wgt_filenames = wgt_filenames.join_string_value() # Main loop for executing the SMM for ii in range(src_filenames.size): ocgis_lh(msg="Running SMM {} of {}".format(ii + 1, src_filenames.size), level=10, logger='regrid.base') src_path = os.path.join(wd, src_filenames[ii]) src_field = RequestDataset(src_path, variable=v, decomp_type=DecompositionType.ESMF).create_field() src_field.load() dst_path = os.path.join(wd, dst_filenames[ii]) dst_field = RequestDataset(dst_path, decomp_type=DecompositionType.ESMF).create_field() dst_field.load() from ocgis.regrid.base import RegridOperation from ESMF.api.constants import RegridMethod # HACK: Note that weight filenames are stored with their full path. # HACK: Always use a bilinear regridding method to allows for the identity sparse matrix in the case of # equal grids. regrid_options = {'split': False, 'weights_in': wgt_filenames[ii], 'regrid_method': RegridMethod.BILINEAR} ro = RegridOperation(src_field, dst_field, regrid_options=regrid_options) regridded = ro.execute() regridded.write(dst_path)
def test_system(self): from ocgis.regrid.base import create_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects import ESMF yc = Variable(name='yc', value=np.arange(-90 + (45 / 2.), 90, 45), dimensions='ydim', dtype=float) xc = Variable(name='xc', value=np.arange(15, 360, 30), dimensions='xdim', dtype=float) ogrid = Grid(y=yc, x=xc, crs=Spherical()) ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds') np.random.seed(1) mask = np.random.rand(*ogrid.shape) mask = mask > 0.5 self.assertTrue(mask.sum() > 3) ogrid.set_mask(mask) egrid = create_esmf_grid(ogrid) actual_shape = egrid.size[0].tolist() desired_shape = np.flipud(ogrid.shape).tolist() self.assertEqual(actual_shape, desired_shape) desired = ogrid.get_value_stacked() desired = np.ma.array(desired, mask=False) desired.mask[0, :, :] = ogrid.get_mask() desired.mask[1, :, :] = ogrid.get_mask() desired = desired.sum() actual_col = egrid.get_coords(0) actual_row = egrid.get_coords(1) actual_mask = np.invert(egrid.mask[0].astype(bool)) actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array(actual_col, mask=actual_mask).sum() self.assertEqual(actual, desired) desired = 9900.0 corners = egrid.coords[ESMF.StaggerLoc.CORNER] actual = corners[0].sum() + corners[1].sum() self.assertEqual(actual, desired) ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical()) variable_name, efield, tidx = list(iter_esmf_fields(ofield, split=False))[0] desired_value = ofield['data'].get_value() self.assertAlmostEqual(efield.data.sum(), desired_value.sum(), places=3) destroy_esmf_objects([egrid, efield]) ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True) desired_value = ofield['data'].get_masked_value() keywords = dict(split=[False, True]) for k in self.iter_product_keywords(keywords): opts = {'split': k.split} dofield = ofield.deepcopy() dofield['data'].get_value().fill(0) ro = RegridOperation(ofield, dofield, regrid_options=opts) actual_field = ro.execute() actual_value = actual_field['data'].get_masked_value() self.assertAlmostEqual(0.0, np.abs(desired_value - actual_value).max())
def test_system_global_grid_combinations(self): """Test regridding with different global grid configurations.""" from ocgis.regrid.base import RegridOperation boolmix = [[True, True], [False, False], [True, False], [False, True]] keywords = dict(with_bounds=boolmix, wrapped=boolmix, resolution=[[3.0, 6.0], [6.0, 3.0], [3.0, 3.0]]) for ctr, k in enumerate(self.iter_product_keywords(keywords)): # print ctr, k # if ctr != 10: continue source_grid = self.get_gridxy_global(with_bounds=k.with_bounds[0], wrapped=k.wrapped[0], resolution=k.resolution[0]) destination_grid = self.get_gridxy_global(with_bounds=k.with_bounds[1], wrapped=k.wrapped[1], resolution=k.resolution[1]) source_field = self.create_regridding_field(source_grid, 'source') destination_field = self.create_regridding_field(destination_grid, 'destination') ro = RegridOperation(source_field, destination_field) res = ro.execute() actual = res['source'].get_masked_value() targets = [actual.min(), actual.mean(), actual.max()] for t in targets: self.assertAlmostEqual(t, 15.0, places=5)
def test_system_masking_with_smm(self): """Test masking with sparse matrix multiplication.""" from ocgis.regrid import RegridOperation grid = create_gridxy_global(with_bounds=False, crs=Spherical(), dist_dimname='x', resolution=5.0) src_field = create_exact_field(grid, 'exact', ntime=3) mask = src_field.grid.get_mask(create=True) mask[0:2, :] = True mask[:, -2:] = True mask[-2:, :] = True mask[:, 0:2] = True src_field.grid.set_mask(mask, cascade=True) src_field['exact'].set_value(src_field['exact'].mv().filled()) dst_field = deepcopy(src_field) dst_field.remove_variable('exact') weights = self.get_temporary_file_path('weights.nc', collective=True) weights = vm.bcast(weights) ro = RegridOperation(src_field, dst_field, regrid_options={ 'weights_out': weights, 'split': False }) _ = ro.execute() ro2 = RegridOperation(src_field, dst_field, regrid_options={ 'weights_in': weights, 'split': True }) result = ro2.execute() actual = result['exact'].mv() desired = src_field['exact'].mv() self.assertNumpyAllClose(actual, desired)
def test_system(self): from ocgis.regrid.base import get_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects import ESMF yc = Variable(name='yc', value=np.arange(-90 + (45 / 2.), 90, 45), dimensions='ydim', dtype=float) xc = Variable(name='xc', value=np.arange(15, 360, 30), dimensions='xdim', dtype=float) ogrid = Grid(y=yc, x=xc, crs=Spherical()) ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds') np.random.seed(1) mask = np.random.rand(*ogrid.shape) mask = mask > 0.5 self.assertTrue(mask.sum() > 3) ogrid.set_mask(mask) egrid = get_esmf_grid(ogrid) actual_shape = egrid.size[0].tolist() desired_shape = np.flipud(ogrid.shape).tolist() self.assertEqual(actual_shape, desired_shape) desired = ogrid.get_value_stacked() desired = np.ma.array(desired, mask=False) desired.mask[0, :, :] = ogrid.get_mask() desired.mask[1, :, :] = ogrid.get_mask() desired = desired.sum() actual_col = egrid.get_coords(0) actual_row = egrid.get_coords(1) actual_mask = np.invert(egrid.mask[0].astype(bool)) actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array( actual_col, mask=actual_mask).sum() self.assertEqual(actual, desired) desired = 9900.0 corners = egrid.coords[ESMF.StaggerLoc.CORNER] actual = corners[0].sum() + corners[1].sum() self.assertEqual(actual, desired) ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical()) variable_name, efield, tidx = list( iter_esmf_fields(ofield, split=False))[0] desired_value = ofield['data'].get_value() self.assertAlmostEqual(efield.data.sum(), desired_value.sum(), places=3) destroy_esmf_objects([egrid, efield]) ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True) desired_value = ofield['data'].get_masked_value() keywords = dict(split=[False, True]) for k in self.iter_product_keywords(keywords): opts = {'split': k.split} dofield = ofield.deepcopy() dofield['data'].get_value().fill(0) ro = RegridOperation(ofield, dofield, regrid_options=opts) actual_field = ro.execute() actual_value = actual_field['data'].get_masked_value() self.assertAlmostEqual(0.0, np.abs(desired_value - actual_value).max())
def smm(index_path, wd=None, data_variables='auto'): """ Run a chunked sparse matrix multiplication. :param str index_path: Path to chunked operation's index file. :param str wd: Path to the directory containing the chunk files. :param list data_variables: Optional list of data variables. Otherwise, auto-discovery is used. """ # Assume the current working directory if wd is None: wd = '' # Turn on auto-discovery if data_variables == 'auto': v = None else: v = data_variables # Get the attribute host index_field = RequestDataset(index_path).get() gs_index_v = index_field[ GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE] # Collect the source filenames src_filenames = gs_index_v.attrs[ GridChunkerConstants.IndexFile.NAME_SOURCE_VARIABLE] src_filenames = index_field[src_filenames] src_filenames = src_filenames.join_string_value() # Collect the destination file names dst_filenames = gs_index_v.attrs[ GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE] dst_filenames = index_field[dst_filenames] dst_filenames = dst_filenames.join_string_value() # Collect the weight filenames wgt_filenames = gs_index_v.attrs[ GridChunkerConstants.IndexFile.NAME_WEIGHTS_VARIABLE] wgt_filenames = index_field[wgt_filenames] wgt_filenames = wgt_filenames.join_string_value() # Main loop for executing the SMM for ii in range(src_filenames.size): ocgis_lh(msg="Running SMM {} of {}".format(ii + 1, src_filenames.size), level=10, logger='regrid.base') src_path = os.path.join(wd, src_filenames[ii]) src_field = RequestDataset( src_path, variable=v, decomp_type=DecompositionType.ESMF).create_field() src_field.load() dst_path = os.path.join(wd, dst_filenames[ii]) dst_field = RequestDataset( dst_path, decomp_type=DecompositionType.ESMF).create_field() dst_field.load() from ocgis.regrid.base import RegridOperation from ESMF.api.constants import RegridMethod # HACK: Note that weight filenames are stored with their full path. # HACK: Always use a bilinear regridding method to allows for the identity sparse matrix in the case of # equal grids. regrid_options = { 'split': False, 'weights_in': wgt_filenames[ii], 'regrid_method': RegridMethod.BILINEAR } ro = RegridOperation(src_field, dst_field, regrid_options=regrid_options) regridded = ro.execute() regridded.write(dst_path)