def _get_regridded_field_with_subset_(self,
sfield,
subset_field_for_regridding=None):
"""
Regrid ``sfield`` subsetting the regrid destination in the process.
:param sfield: The input field to regrid.
:type sfield: :class:`ocgis.Field`
:param subset_field_for_regridding: The original, unaltered spatial dimension to use for subsetting.
:type subset_field_for_regridding: :class:`ocgis.Field`
:rtype: :class:`~ocgis.Field`
"""
from ocgis.regrid.base import RegridOperation
ocgis_lh(logger=self._subset_log,
msg='Starting regrid operation...',
level=logging.INFO)
ro = RegridOperation(sfield,
self.ops.regrid_destination,
subset_field=subset_field_for_regridding,
regrid_options=self.ops.regrid_options)
sfield = ro.execute()
ocgis_lh(logger=self._subset_log,
msg='Regrid operation complete.',
level=logging.INFO)
return sfield
def test_system_masking_with_smm(self):
"""Test masking with sparse matrix multiplication."""
from ocgis.regrid import RegridOperation
grid = create_gridxy_global(with_bounds=False, crs=Spherical(), dist_dimname='x', resolution=5.0)
src_field = create_exact_field(grid, 'exact', ntime=3)
mask = src_field.grid.get_mask(create=True)
mask[0:2, :] = True
mask[:, -2:] = True
mask[-2:, :] = True
mask[:, 0:2] = True
src_field.grid.set_mask(mask, cascade=True)
src_field['exact'].set_value(src_field['exact'].mv().filled())
dst_field = deepcopy(src_field)
dst_field.remove_variable('exact')
weights = self.get_temporary_file_path('weights.nc', collective=True)
weights = vm.bcast(weights)
ro = RegridOperation(src_field, dst_field, regrid_options={'weights_out': weights, 'split': False})
_ = ro.execute()
ro2 = RegridOperation(src_field, dst_field, regrid_options={'weights_in': weights, 'split': True})
result = ro2.execute()
actual = result['exact'].mv()
desired = src_field['exact'].mv()
self.assertNumpyAllClose(actual, desired)
def test_combo_global_grid_combinations(self):
"""Test regridding with different global grid configurations."""
boolmix = [[True, True], [False, False], [True, False], [False, True]]
keywords = dict(with_bounds=boolmix, wrapped=boolmix, resolution=[[3.0, 6.0], [6.0, 3.0], [3.0, 3.0]])
for ctr, k in enumerate(self.iter_product_keywords(keywords)):
# print ctr, k
# if ctr != 10: continue
source_grid = self.get_spherical_global_grid(
with_bounds=k.with_bounds[0], wrapped=k.wrapped[0], resolution=k.resolution[0]
)
destination_grid = self.get_spherical_global_grid(
with_bounds=k.with_bounds[1], wrapped=k.wrapped[1], resolution=k.resolution[1]
)
source_field = self.get_regridding_field(source_grid, "source")
destination_field = self.get_regridding_field(destination_grid, "destination")
ro = RegridOperation(source_field, destination_field)
res = ro.execute()
actual = res.variables["source"].value
desired = destination_field.variables["destination"].value
try:
self.assertNumpyAllClose(actual, desired)
except AssertionError:
# Without bounds, some data will be unmapped and masked around the exterior of the grid if the source
# resolution is higher than the destination.
self.assertEqual(k.resolution, [6.0, 3.0])
self.assertEqual(actual.mask.sum(), 356)
self.assertTrue(np.all(np.isclose(actual.compressed(), 15.0)))
def test_system_global_grid_combinations(self):
"""Test regridding with different global grid configurations."""
from ocgis.regrid.base import RegridOperation
boolmix = [[True, True], [False, False], [True, False], [False, True]]
keywords = dict(with_bounds=boolmix,
wrapped=boolmix,
resolution=[[3.0, 6.0], [6.0, 3.0], [3.0, 3.0]])
for ctr, k in enumerate(self.iter_product_keywords(keywords)):
# print ctr, k
# if ctr != 10: continue
source_grid = self.get_gridxy_global(with_bounds=k.with_bounds[0],
wrapped=k.wrapped[0],
resolution=k.resolution[0])
destination_grid = self.get_gridxy_global(
with_bounds=k.with_bounds[1],
wrapped=k.wrapped[1],
resolution=k.resolution[1])
source_field = self.create_regridding_field(source_grid, 'source')
destination_field = self.create_regridding_field(
destination_grid, 'destination')
ro = RegridOperation(source_field, destination_field)
res = ro.execute()
actual = res['source'].get_masked_value()
targets = [actual.min(), actual.mean(), actual.max()]
for t in targets:
self.assertAlmostEqual(t, 15.0, places=5)
def smm(index_path, wd=None, data_variables='auto'):
"""
Run a chunked sparse matrix multiplication.
:param str index_path: Path to chunked operation's index file.
:param str wd: Path to the directory containing the chunk files.
:param list data_variables: Optional list of data variables. Otherwise, auto-discovery is used.
"""
# Assume the current working directory
if wd is None:
wd = ''
# Turn on auto-discovery
if data_variables == 'auto':
v = None
else:
v = data_variables
# Get the attribute host
index_field = RequestDataset(index_path).get()
gs_index_v = index_field[GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE]
# Collect the source filenames
src_filenames = gs_index_v.attrs[GridChunkerConstants.IndexFile.NAME_SOURCE_VARIABLE]
src_filenames = index_field[src_filenames]
src_filenames = src_filenames.join_string_value()
# Collect the destination file names
dst_filenames = gs_index_v.attrs[GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE]
dst_filenames = index_field[dst_filenames]
dst_filenames = dst_filenames.join_string_value()
# Collect the weight filenames
wgt_filenames = gs_index_v.attrs[GridChunkerConstants.IndexFile.NAME_WEIGHTS_VARIABLE]
wgt_filenames = index_field[wgt_filenames]
wgt_filenames = wgt_filenames.join_string_value()
# Main loop for executing the SMM
for ii in range(src_filenames.size):
ocgis_lh(msg="Running SMM {} of {}".format(ii + 1, src_filenames.size), level=10, logger='regrid.base')
src_path = os.path.join(wd, src_filenames[ii])
src_field = RequestDataset(src_path, variable=v, decomp_type=DecompositionType.ESMF).create_field()
src_field.load()
dst_path = os.path.join(wd, dst_filenames[ii])
dst_field = RequestDataset(dst_path, decomp_type=DecompositionType.ESMF).create_field()
dst_field.load()
from ocgis.regrid.base import RegridOperation
from ESMF.api.constants import RegridMethod
# HACK: Note that weight filenames are stored with their full path.
# HACK: Always use a bilinear regridding method to allows for the identity sparse matrix in the case of
# equal grids.
regrid_options = {'split': False, 'weights_in': wgt_filenames[ii], 'regrid_method': RegridMethod.BILINEAR}
ro = RegridOperation(src_field, dst_field, regrid_options=regrid_options)
regridded = ro.execute()
regridded.write(dst_path)
def test_system(self):
from ocgis.regrid.base import create_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects
import ESMF
yc = Variable(name='yc', value=np.arange(-90 + (45 / 2.), 90, 45), dimensions='ydim', dtype=float)
xc = Variable(name='xc', value=np.arange(15, 360, 30), dimensions='xdim', dtype=float)
ogrid = Grid(y=yc, x=xc, crs=Spherical())
ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds')
np.random.seed(1)
mask = np.random.rand(*ogrid.shape)
mask = mask > 0.5
self.assertTrue(mask.sum() > 3)
ogrid.set_mask(mask)
egrid = create_esmf_grid(ogrid)
actual_shape = egrid.size[0].tolist()
desired_shape = np.flipud(ogrid.shape).tolist()
self.assertEqual(actual_shape, desired_shape)
desired = ogrid.get_value_stacked()
desired = np.ma.array(desired, mask=False)
desired.mask[0, :, :] = ogrid.get_mask()
desired.mask[1, :, :] = ogrid.get_mask()
desired = desired.sum()
actual_col = egrid.get_coords(0)
actual_row = egrid.get_coords(1)
actual_mask = np.invert(egrid.mask[0].astype(bool))
actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array(actual_col, mask=actual_mask).sum()
self.assertEqual(actual, desired)
desired = 9900.0
corners = egrid.coords[ESMF.StaggerLoc.CORNER]
actual = corners[0].sum() + corners[1].sum()
self.assertEqual(actual, desired)
ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical())
variable_name, efield, tidx = list(iter_esmf_fields(ofield, split=False))[0]
desired_value = ofield['data'].get_value()
self.assertAlmostEqual(efield.data.sum(), desired_value.sum(), places=3)
destroy_esmf_objects([egrid, efield])
ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True)
desired_value = ofield['data'].get_masked_value()
keywords = dict(split=[False, True])
for k in self.iter_product_keywords(keywords):
opts = {'split': k.split}
dofield = ofield.deepcopy()
dofield['data'].get_value().fill(0)
ro = RegridOperation(ofield, dofield, regrid_options=opts)
actual_field = ro.execute()
actual_value = actual_field['data'].get_masked_value()
self.assertAlmostEqual(0.0, np.abs(desired_value - actual_value).max())
def _get_regridded_field_with_subset_(self, sfield, subset_field_for_regridding=None):
"""
Regrid ``sfield`` subsetting the regrid destination in the process.
:param sfield: The input field to regrid.
:type sfield: :class:`ocgis.Field`
:param subset_field_for_regridding: The original, unaltered spatial dimension to use for subsetting.
:type subset_field_for_regridding: :class:`ocgis.Field`
:rtype: :class:`~ocgis.Field`
"""
from ocgis.regrid.base import RegridOperation
ocgis_lh(logger=self._subset_log, msg='Starting regrid operation...', level=logging.INFO)
ro = RegridOperation(sfield, self.ops.regrid_destination, subset_field=subset_field_for_regridding,
regrid_options=self.ops.regrid_options)
sfield = ro.execute()
ocgis_lh(logger=self._subset_log, msg='Regrid operation complete.', level=logging.INFO)
return sfield
def test_system_global_grid_combinations(self):
"""Test regridding with different global grid configurations."""
from ocgis.regrid.base import RegridOperation
boolmix = [[True, True], [False, False], [True, False], [False, True]]
keywords = dict(with_bounds=boolmix, wrapped=boolmix, resolution=[[3.0, 6.0], [6.0, 3.0], [3.0, 3.0]])
for ctr, k in enumerate(self.iter_product_keywords(keywords)):
# print ctr, k
# if ctr != 10: continue
source_grid = self.get_gridxy_global(with_bounds=k.with_bounds[0], wrapped=k.wrapped[0],
resolution=k.resolution[0])
destination_grid = self.get_gridxy_global(with_bounds=k.with_bounds[1], wrapped=k.wrapped[1],
resolution=k.resolution[1])
source_field = self.create_regridding_field(source_grid, 'source')
destination_field = self.create_regridding_field(destination_grid, 'destination')
ro = RegridOperation(source_field, destination_field)
res = ro.execute()
actual = res['source'].get_masked_value()
targets = [actual.min(), actual.mean(), actual.max()]
for t in targets:
self.assertAlmostEqual(t, 15.0, places=5)
def test_system_masking_with_smm(self):
"""Test masking with sparse matrix multiplication."""
from ocgis.regrid import RegridOperation
grid = create_gridxy_global(with_bounds=False,
crs=Spherical(),
dist_dimname='x',
resolution=5.0)
src_field = create_exact_field(grid, 'exact', ntime=3)
mask = src_field.grid.get_mask(create=True)
mask[0:2, :] = True
mask[:, -2:] = True
mask[-2:, :] = True
mask[:, 0:2] = True
src_field.grid.set_mask(mask, cascade=True)
src_field['exact'].set_value(src_field['exact'].mv().filled())
dst_field = deepcopy(src_field)
dst_field.remove_variable('exact')
weights = self.get_temporary_file_path('weights.nc', collective=True)
weights = vm.bcast(weights)
ro = RegridOperation(src_field,
dst_field,
regrid_options={
'weights_out': weights,
'split': False
})
_ = ro.execute()
ro2 = RegridOperation(src_field,
dst_field,
regrid_options={
'weights_in': weights,
'split': True
})
result = ro2.execute()
actual = result['exact'].mv()
desired = src_field['exact'].mv()
self.assertNumpyAllClose(actual, desired)
def test_system(self):
from ocgis.regrid.base import get_esmf_grid, iter_esmf_fields, RegridOperation, destroy_esmf_objects
import ESMF
yc = Variable(name='yc',
value=np.arange(-90 + (45 / 2.), 90, 45),
dimensions='ydim',
dtype=float)
xc = Variable(name='xc',
value=np.arange(15, 360, 30),
dimensions='xdim',
dtype=float)
ogrid = Grid(y=yc, x=xc, crs=Spherical())
ogrid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds')
np.random.seed(1)
mask = np.random.rand(*ogrid.shape)
mask = mask > 0.5
self.assertTrue(mask.sum() > 3)
ogrid.set_mask(mask)
egrid = get_esmf_grid(ogrid)
actual_shape = egrid.size[0].tolist()
desired_shape = np.flipud(ogrid.shape).tolist()
self.assertEqual(actual_shape, desired_shape)
desired = ogrid.get_value_stacked()
desired = np.ma.array(desired, mask=False)
desired.mask[0, :, :] = ogrid.get_mask()
desired.mask[1, :, :] = ogrid.get_mask()
desired = desired.sum()
actual_col = egrid.get_coords(0)
actual_row = egrid.get_coords(1)
actual_mask = np.invert(egrid.mask[0].astype(bool))
actual = np.ma.array(actual_row, mask=actual_mask).sum() + np.ma.array(
actual_col, mask=actual_mask).sum()
self.assertEqual(actual, desired)
desired = 9900.0
corners = egrid.coords[ESMF.StaggerLoc.CORNER]
actual = corners[0].sum() + corners[1].sum()
self.assertEqual(actual, desired)
ofield = create_exact_field(ogrid, 'data', ntime=3, crs=Spherical())
variable_name, efield, tidx = list(
iter_esmf_fields(ofield, split=False))[0]
desired_value = ofield['data'].get_value()
self.assertAlmostEqual(efield.data.sum(),
desired_value.sum(),
places=3)
destroy_esmf_objects([egrid, efield])
ofield.grid.set_mask(ofield.grid.get_mask(), cascade=True)
desired_value = ofield['data'].get_masked_value()
keywords = dict(split=[False, True])
for k in self.iter_product_keywords(keywords):
opts = {'split': k.split}
dofield = ofield.deepcopy()
dofield['data'].get_value().fill(0)
ro = RegridOperation(ofield, dofield, regrid_options=opts)
actual_field = ro.execute()
actual_value = actual_field['data'].get_masked_value()
self.assertAlmostEqual(0.0,
np.abs(desired_value - actual_value).max())
def smm(index_path, wd=None, data_variables='auto'):
"""
Run a chunked sparse matrix multiplication.
:param str index_path: Path to chunked operation's index file.
:param str wd: Path to the directory containing the chunk files.
:param list data_variables: Optional list of data variables. Otherwise, auto-discovery is used.
"""
# Assume the current working directory
if wd is None:
wd = ''
# Turn on auto-discovery
if data_variables == 'auto':
v = None
else:
v = data_variables
# Get the attribute host
index_field = RequestDataset(index_path).get()
gs_index_v = index_field[
GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE]
# Collect the source filenames
src_filenames = gs_index_v.attrs[
GridChunkerConstants.IndexFile.NAME_SOURCE_VARIABLE]
src_filenames = index_field[src_filenames]
src_filenames = src_filenames.join_string_value()
# Collect the destination file names
dst_filenames = gs_index_v.attrs[
GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE]
dst_filenames = index_field[dst_filenames]
dst_filenames = dst_filenames.join_string_value()
# Collect the weight filenames
wgt_filenames = gs_index_v.attrs[
GridChunkerConstants.IndexFile.NAME_WEIGHTS_VARIABLE]
wgt_filenames = index_field[wgt_filenames]
wgt_filenames = wgt_filenames.join_string_value()
# Main loop for executing the SMM
for ii in range(src_filenames.size):
ocgis_lh(msg="Running SMM {} of {}".format(ii + 1, src_filenames.size),
level=10,
logger='regrid.base')
src_path = os.path.join(wd, src_filenames[ii])
src_field = RequestDataset(
src_path, variable=v,
decomp_type=DecompositionType.ESMF).create_field()
src_field.load()
dst_path = os.path.join(wd, dst_filenames[ii])
dst_field = RequestDataset(
dst_path, decomp_type=DecompositionType.ESMF).create_field()
dst_field.load()
from ocgis.regrid.base import RegridOperation
from ESMF.api.constants import RegridMethod
# HACK: Note that weight filenames are stored with their full path.
# HACK: Always use a bilinear regridding method to allows for the identity sparse matrix in the case of
# equal grids.
regrid_options = {
'split': False,
'weights_in': wgt_filenames[ii],
'regrid_method': RegridMethod.BILINEAR
}
ro = RegridOperation(src_field,
dst_field,
regrid_options=regrid_options)
regridded = ro.execute()
regridded.write(dst_path)