def test_create_merged_weight_file(self):
import ESMF
path_src = self.get_temporary_file_path('src.nc')
path_dst = self.get_temporary_file_path('dst.nc')
src_grid = create_gridxy_global(resolution=30.0,
wrapped=False,
crs=Spherical())
dst_grid = create_gridxy_global(resolution=35.0,
wrapped=False,
crs=Spherical())
src_grid.write(path_src)
dst_grid.write(path_dst)
# Split source and destination grids ---------------------------------------------------------------------------
gs = GridChunker(src_grid,
dst_grid, (2, 2),
check_contains=False,
allow_masked=True,
paths=self.fixture_paths,
genweights=True)
gs.write_chunks()
# Merge weight files -------------------------------------------------------------------------------------------
merged_weight_filename = self.get_temporary_file_path(
'merged_weights.nc')
gs.create_merged_weight_file(merged_weight_filename)
# Generate a global weight file using ESMF ---------------------------------------------------------------------
global_weights_filename = self.get_temporary_file_path(
'global_weights.nc')
srcgrid = ESMF.Grid(filename=path_src,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
dstgrid = ESMF.Grid(filename=path_dst,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
srcfield = ESMF.Field(grid=srcgrid)
dstfield = ESMF.Field(grid=dstgrid)
_ = ESMF.Regrid(srcfield=srcfield,
dstfield=dstfield,
filename=global_weights_filename,
regrid_method=ESMF.RegridMethod.CONSERVE)
# Test merged and global weight files are equivalent -----------------------------------------------------------
self.assertWeightFilesEquivalent(global_weights_filename,
merged_weight_filename)
def test_create_merged_weight_file(self):
path_src = self.get_temporary_file_path('src.nc')
path_dst = self.get_temporary_file_path('dst.nc')
src_grid = create_gridxy_global(resolution=30.0, wrapped=False, crs=Spherical())
dst_grid = create_gridxy_global(resolution=35.0, wrapped=False, crs=Spherical())
src_grid.write(path_src)
dst_grid.write(path_dst)
# Split source and destination grids ---------------------------------------------------------------------------
gs = GridSplitter(src_grid, dst_grid, (2, 2), check_contains=False, allow_masked=True, paths=self.fixture_paths)
gs.write_subsets()
# Load the grid splitter index file ----------------------------------------------------------------------------
index_filename = gs.create_full_path_from_template('index_file')
ifile = RequestDataset(uri=index_filename).get()
ifile.load()
gidx = ifile[GridSplitterConstants.IndexFile.NAME_INDEX_VARIABLE].attrs
source_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_SOURCE_VARIABLE]]
sv = source_filename.join_string_value()
destination_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_DESTINATION_VARIABLE]]
dv = destination_filename.join_string_value()
# Create weight files for each subset --------------------------------------------------------------------------
for ii, sfn in enumerate(sv):
esp = os.path.join(self.current_dir_output, sfn)
edp = os.path.join(self.current_dir_output, dv[ii])
ewp = gs.create_full_path_from_template('wgt_template', index=ii + 1)
cmd = ['ESMF_RegridWeightGen', '-s', esp, '--src_type', 'GRIDSPEC', '-d', edp, '--dst_type',
'GRIDSPEC', '-w', ewp, '--method', 'conserve', '-r', '--no_log']
subprocess.check_call(cmd)
# Merge weight files -------------------------------------------------------------------------------------------
merged_weight_filename = self.get_temporary_file_path('merged_weights.nc')
gs.create_merged_weight_file(merged_weight_filename)
# Generate a global weight file using ESMF ---------------------------------------------------------------------
global_weights_filename = self.get_temporary_file_path('global_weights.nc')
cmd = ['ESMF_RegridWeightGen', '-s', path_src, '--src_type', 'GRIDSPEC', '-d', path_dst, '--dst_type',
'GRIDSPEC', '-w', global_weights_filename, '--method', 'conserve', '--weight-only', '--no_log']
subprocess.check_call(cmd)
# Test merged and global weight files are equivalent -----------------------------------------------------------
self.assertWeightFilesEquivalent(global_weights_filename, merged_weight_filename)
def test_system_multiple_netcdf_files(self):
"""Test subsetting multiple netCDF files and returning a spatial collection."""
grid = create_gridxy_global(resolution=3.0)
vars = ['ocgis_example_tasmin', 'ocgis_example_tas', 'ocgis_example_tasmax']
paths = [self.get_temporary_file_path('{}.nc'.format(ii)) for ii in vars]
geom_select_uid = [16, 23]
field_names = ['tasmin', 'tas', 'tasmax']
for ctr, (path, var) in enumerate(zip(paths, vars), start=1):
field = create_exact_field(grid.copy(), var, ntime=3)
field.data_variables[0].get_value()[:] = 10 * ctr
field.write(path)
rds = [RequestDataset(uri=uri, variable=var, field_name=field_name) for uri, var, field_name in
zip(paths, vars, field_names)]
ops = OcgOperations(dataset=rds, spatial_operation='clip', aggregate=True, geom=self.path_state_boundaries,
geom_select_uid=geom_select_uid)
ret = ops.execute()
self.assertAsSetEqual(ret.children.keys(), geom_select_uid)
for geom_uid in geom_select_uid:
actual = ret.children[geom_uid].children.keys()
self.assertAsSetEqual(actual, field_names)
for idx, field_name in enumerate(field_names):
actual = ret.get_element(container_ugid=geom_uid, field_name=field_names[idx], variable_name=vars[idx])
actual = actual.get_value()
actual = actual == (idx + 1) * 10
self.assertTrue(np.all(actual))
def test_iter_spatial_decomposition(self):
self.add_barrier = False
if vm.size not in [1, 4]:
raise SkipTest('vm.size not in [1, 4]')
grid = create_gridxy_global(resolution=10.,
wrapped=False,
crs=Spherical())
splits = (2, 3)
actual = []
for sub, slc in iter_spatial_decomposition(grid,
splits,
optimized_bbox_subset=True):
root = vm.get_live_ranks_from_object(sub)[0]
with vm.scoped_by_emptyable('test extent', sub):
if vm.is_null:
extent_global = None
else:
extent_global = sub.extent_global
extent_global = vm.bcast(extent_global, root=root)
actual.append(extent_global)
desired = [(0.0, -90.0, 120.0, 0.0), (120.0, -90.0, 240.0, 0.0),
(240.0, -90.0, 360.0, 0.0), (0.0, 0.0, 120.0, 90.0),
(120.0, 0.0, 240.0, 90.0), (240.0, 0.0, 360.0, 90.0)]
self.assertEqual(actual, desired)
def test_get_ocgis_field_from_esmf_field(self):
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
ogrid = create_gridxy_global(crs=Spherical())
ofield = create_exact_field(ogrid, 'foo', ntime=3)
ogrid = ofield.grid
ogrid_mask = ogrid.get_mask(create=True)
ogrid_mask[1, 0] = True
ogrid.set_mask(ogrid_mask)
efield = get_esmf_field_from_ocgis_field(ofield)
self.assertEqual(efield.data.shape, (360, 180, 3))
ofield_actual = get_ocgis_field_from_esmf_field(efield, field=ofield)
actual_dv_mask = ofield_actual.data_variables[0].get_mask()
self.assertTrue(np.all(actual_dv_mask[:, 1, 0]))
self.assertEqual(actual_dv_mask.sum(), 3)
self.assertNumpyAll(ofield.data_variables[0].get_value(),
ofield_actual.data_variables[0].get_value())
self.assertEqual(ofield.data_variables[0].name, efield.name)
self.assertNumpyAll(ofield.time.get_value(),
ofield_actual.time.get_value())
def test_system_masking_with_smm(self):
"""Test masking with sparse matrix multiplication."""
from ocgis.regrid import RegridOperation
grid = create_gridxy_global(with_bounds=False, crs=Spherical(), dist_dimname='x', resolution=5.0)
src_field = create_exact_field(grid, 'exact', ntime=3)
mask = src_field.grid.get_mask(create=True)
mask[0:2, :] = True
mask[:, -2:] = True
mask[-2:, :] = True
mask[:, 0:2] = True
src_field.grid.set_mask(mask, cascade=True)
src_field['exact'].set_value(src_field['exact'].mv().filled())
dst_field = deepcopy(src_field)
dst_field.remove_variable('exact')
weights = self.get_temporary_file_path('weights.nc', collective=True)
weights = vm.bcast(weights)
ro = RegridOperation(src_field, dst_field, regrid_options={'weights_out': weights, 'split': False})
_ = ro.execute()
ro2 = RegridOperation(src_field, dst_field, regrid_options={'weights_in': weights, 'split': True})
result = ro2.execute()
actual = result['exact'].mv()
desired = src_field['exact'].mv()
self.assertNumpyAllClose(actual, desired)
def test_system_scrip_destination_splitting(self):
"""Test splitting a SCRIP destination grid."""
src_grid = create_gridxy_global()
dst_grid = self.fixture_driver_scrip_netcdf_field().grid
gs = GridChunker(src_grid, dst_grid, (3,), paths={'wd': self.current_dir_output})
gs.write_chunks()
self.assertEqual(len(os.listdir(self.current_dir_output)), 7)
def test_transform_coordinates(self):
desired_min_maxes = [[-0.9330127018922193, 0.93301270189221941], [-0.93301270189221941, 0.93301270189221941],
[-0.96592582628906831, 0.96592582628906831]]
keywords = {'wrapped': [
False,
True
],
'angular_units': [
OcgisUnits.DEGREES,
OcgisUnits.RADIANS
],
'other_crs': [
Cartesian(),
WGS84()
]
}
for k in self.iter_product_keywords(keywords):
spherical = Spherical(angular_units=k.angular_units)
tp = Tripole(spherical=spherical)
grid = create_gridxy_global(resolution=30.0, wrapped=k.wrapped)
if not k.wrapped:
x_value = grid.x.get_value()
select = x_value > 180.
x_value[select] -= 360.
grid.expand()
x = grid.x.get_value()
y = grid.y.get_value()
if k.angular_units == OcgisUnits.RADIANS:
x *= ConversionFactor.DEG_TO_RAD
y *= ConversionFactor.DEG_TO_RAD
z = np.ones(x.shape, dtype=x.dtype)
desired = (x, y, z)
try:
as_cart = tp.transform_coordinates(k.other_crs, x, y, z)
except CRSNotEquivalenError:
self.assertNotEqual(k.other_crs, Cartesian())
continue
x_cart, y_cart, z_cart = as_cart
for idx, ii in enumerate(as_cart):
actual_min_max = [ii.min(), ii.max()]
self.assertNumpyAllClose(np.array(actual_min_max), np.array(desired_min_maxes[idx]))
actual = tp.transform_coordinates(k.other_crs, x_cart, y_cart, z_cart, inverse=True)
for a, d in zip(actual, desired):
are = np.abs(a - d)
self.assertLessEqual(are.max(), 1e-6)
def test_get_esmf_grid_periodicity(self):
"""Test periodicity parameters generate reasonable output."""
from ocgis.regrid.base import create_esmf_grid
ogrid = create_gridxy_global(resolution=10.0, crs=Spherical(), with_bounds=False, dist_dimname='x')
egrid = create_esmf_grid(ogrid)
self.assertEqual(egrid.periodic_dim, 0)
self.assertEqual(egrid.num_peri_dims, 1)
self.assertEqual(egrid.pole_dim, 1)
def test_to_xarray(self):
grid = create_gridxy_global(crs=Spherical())
field = create_exact_field(grid, 'foo', ntime=3)
field.attrs['i_am_global'] = 'confirm'
field.grid.abstraction = Topology.POINT
field.set_abstraction_geom()
field.time.set_extrapolated_bounds('time_bounds', 'bounds')
xr = field.to_xarray()
self.assertEqual(xr.attrs['i_am_global'], 'confirm')
self.assertGreater(len(xr.coords), 0)
def test_system_user_geometry_identifier_typed_appropriately(self):
"""Test UGID is typed appropriately according to the data model."""
ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'}
grid = create_gridxy_global(resolution=3.0)
field = create_exact_field(grid, 'foo', crs=Spherical())
ops = OcgOperations(dataset=field, output_format_options=ofo, geom=[-100, 30, -90, 40], aggregate=True)
actual = ops.execute()
actual = actual[1]
self.assertEqual(actual.geom.ugid.dtype, np.int32)
def test_system_scrip_destination_splitting(self):
"""Test splitting a SCRIP destination grid."""
src_grid = create_gridxy_global()
dst_grid = self.fixture_driver_scrip_netcdf_field().grid
gc = GridChunker(src_grid,
dst_grid, (3, ),
paths={'wd': self.current_dir_output})
gc.write_chunks()
self.assertEqual(len(os.listdir(self.current_dir_output)), 7)
def test_transform_coordinates(self):
desired_min_maxes = [[-0.9330127018922193, 0.93301270189221941],
[-0.93301270189221941, 0.93301270189221941],
[-0.96592582628906831, 0.96592582628906831]]
keywords = {
'wrapped': [False, True],
'angular_units': [OcgisUnits.DEGREES, OcgisUnits.RADIANS],
'other_crs': [Cartesian(), WGS84()]
}
for k in self.iter_product_keywords(keywords):
spherical = Spherical(angular_units=k.angular_units)
tp = Tripole(spherical=spherical)
grid = create_gridxy_global(resolution=30.0, wrapped=k.wrapped)
if not k.wrapped:
x_value = grid.x.get_value()
select = x_value > 180.
x_value[select] -= 360.
grid.expand()
x = grid.x.get_value()
y = grid.y.get_value()
if k.angular_units == OcgisUnits.RADIANS:
x *= ConversionFactor.DEG_TO_RAD
y *= ConversionFactor.DEG_TO_RAD
z = np.ones(x.shape, dtype=x.dtype)
desired = (x, y, z)
try:
as_cart = tp.transform_coordinates(k.other_crs, x, y, z)
except CRSNotEquivalenError:
self.assertNotEqual(k.other_crs, Cartesian())
continue
x_cart, y_cart, z_cart = as_cart
for idx, ii in enumerate(as_cart):
actual_min_max = [ii.min(), ii.max()]
self.assertNumpyAllClose(np.array(actual_min_max),
np.array(desired_min_maxes[idx]))
actual = tp.transform_coordinates(k.other_crs,
x_cart,
y_cart,
z_cart,
inverse=True)
for a, d in zip(actual, desired):
are = np.abs(a - d)
self.assertLessEqual(are.max(), 1e-6)
def test_get_esmf_grid_periodicity(self):
"""Test periodicity parameters generate reasonable output."""
from ocgis.regrid.base import create_esmf_grid
ogrid = create_gridxy_global(resolution=10.0,
crs=Spherical(),
with_bounds=False,
dist_dimname='x')
egrid = create_esmf_grid(ogrid)
self.assertEqual(egrid.periodic_dim, 0)
self.assertEqual(egrid.num_peri_dims, 1)
self.assertEqual(egrid.pole_dim, 1)
def test_system_geometry_identifier_typed_appropriately(self):
"""Test GID is typed appropriately according to the data model."""
ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'}
grid = create_gridxy_global(resolution=3.0)
field = create_exact_field(grid, 'foo', crs=Spherical())
ops = OcgOperations(dataset=field, output_format_options=ofo, geom=[-100, 30, -90, 40], aggregate=True)
actual = ops.execute()
actual = actual.get_element(container_ugid=1)
self.assertEqual(actual.geom.ugid.dtype, np.int32)
# Test data model is retrieved appropriately from file.
ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'}
grid = create_gridxy_global(resolution=3.0)
field = create_exact_field(grid, 'foo', crs=Spherical())
ops = OcgOperations(dataset=field, output_format_options=ofo, output_format='nc')
ret = ops.execute()
rd = RequestDataset(uri=ret)
ops = OcgOperations(dataset=rd, geom=[-100, 30, -90, 40], aggregate=True)
actual = ops.execute()
actual = actual.get_element(container_ugid=1)
self.assertEqual(actual.geom.ugid.dtype, np.int32)
def test_create_merged_weight_file(self):
import ESMF
path_src = self.get_temporary_file_path('src.nc')
path_dst = self.get_temporary_file_path('dst.nc')
src_grid = create_gridxy_global(resolution=30.0, wrapped=False, crs=Spherical())
dst_grid = create_gridxy_global(resolution=35.0, wrapped=False, crs=Spherical())
src_grid.write(path_src)
dst_grid.write(path_dst)
# Split source and destination grids ---------------------------------------------------------------------------
gs = GridChunker(src_grid, dst_grid, (2, 2), check_contains=False, allow_masked=True, paths=self.fixture_paths,
genweights=True)
gs.write_chunks()
# Merge weight files -------------------------------------------------------------------------------------------
merged_weight_filename = self.get_temporary_file_path('merged_weights.nc')
gs.create_merged_weight_file(merged_weight_filename)
# Generate a global weight file using ESMF ---------------------------------------------------------------------
global_weights_filename = self.get_temporary_file_path('global_weights.nc')
srcgrid = ESMF.Grid(filename=path_src, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True)
dstgrid = ESMF.Grid(filename=path_dst, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True)
srcfield = ESMF.Field(grid=srcgrid)
dstfield = ESMF.Field(grid=dstgrid)
_ = ESMF.Regrid(srcfield=srcfield, dstfield=dstfield, filename=global_weights_filename,
regrid_method=ESMF.RegridMethod.CONSERVE)
# Test merged and global weight files are equivalent -----------------------------------------------------------
self.assertWeightFilesEquivalent(global_weights_filename, merged_weight_filename)
def test_chunked_rwg_spatial_subset(self):
env.CLOBBER_UNITS_ON_BOUNDS = False
src_grid = create_gridxy_global(crs=Spherical())
src_field = create_exact_field(src_grid, 'foo')
xvar = Variable(name='x', value=[-90., -80.], dimensions='xdim')
yvar = Variable(name='y', value=[40., 50.], dimensions='ydim')
dst_grid = Grid(x=xvar, y=yvar, crs=Spherical())
if ocgis.vm.rank == 0:
source = self.get_temporary_file_path('source.nc')
else:
source = None
source = ocgis.vm.bcast(source)
src_field.write(source)
if ocgis.vm.rank == 0:
destination = self.get_temporary_file_path('destination.nc')
else:
destination = None
destination = ocgis.vm.bcast(destination)
dst_grid.parent.write(destination)
wd = os.path.join(self.current_dir_output, 'chunks')
weight = os.path.join(self.current_dir_output, 'weights.nc')
spatial_subset = os.path.join(self.current_dir_output,
'spatial_subset.nc')
runner = CliRunner()
cli_args = [
'chunked-rwg', '--source', source, '--destination', destination,
'--wd', wd, '--spatial_subset', '--spatial_subset_path',
spatial_subset, '--weight', weight, '--esmf_regrid_method',
'BILINEAR', '--persist'
]
result = runner.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
actual = RequestDataset(uri=spatial_subset).create_field()
actual_ymean = actual.grid.get_value_stacked()[0].mean()
actual_xmean = actual.grid.get_value_stacked()[1].mean()
self.assertEqual(actual_ymean, 45.)
self.assertEqual(actual_xmean, -85.)
self.assertEqual(actual.grid.shape, (14, 14))
self.assertTrue(os.path.exists(weight))
actual = RequestDataset(weight, driver='netcdf').create_field()
self.assertIn('history', actual.attrs)
def test_get_dimension_map_2d_spatial_coordinates(self):
grid = create_gridxy_global()
grid.expand()
path = self.get_temporary_file_path('foo.nc')
f = Field(grid=grid)
f.write(path)
rd = RequestDataset(path)
field = rd.get()
sub = field.get_field_slice({'y': 10, 'x': 5})
self.assertEqual(sub.grid.x.shape, (1, 1))
actual = f.dimension_map.get_dimension(DimensionMapKey.Y)
self.assertEqual(actual, ['y'])
actual = f.dimension_map.get_dimension(DimensionMapKey.X)
self.assertEqual(actual, ['x'])
def test_convert_to_geometry_coordinates_polygons(self):
grid = create_gridxy_global(resolution=45.0)
geom = grid.get_abstraction_geometry()
geom.reshape(Dimension('n_elements', geom.size))
keywords = dict(pack=[True, False], repeat_last_node=[False, True], start_index=[0, 1])
for k in self.iter_product_keywords(keywords):
actual = geom.convert_to(pack=k.pack, repeat_last_node=k.repeat_last_node, start_index=k.start_index)
self.assertEqual(actual.cindex.attrs['start_index'], k.start_index)
self.assertEqual(actual.start_index, k.start_index)
self.assertEqual(actual.cindex.get_value()[0].min(), k.start_index)
self.assertEqual(actual.packed, k.pack)
self.assertIsNotNone(actual.cindex)
for actual_geom, desired_geom in zip(actual.get_geometry_iterable(), geom.get_value().flat):
self.assertEqual(actual_geom[1], desired_geom)
def test_create_dimension_map_2d_spatial_coordinates(self):
grid = create_gridxy_global()
grid.expand()
path = self.get_temporary_file_path('foo.nc')
f = Field(grid=grid)
f.write(path)
rd = RequestDataset(path)
field = rd.get()
sub = field.get_field_slice({'y': 10, 'x': 5})
self.assertEqual(sub.grid.x.shape, (1, 1))
actual = f.dimension_map.get_dimension(DimensionMapKey.Y)
self.assertEqual(actual, ['y'])
actual = f.dimension_map.get_dimension(DimensionMapKey.X)
self.assertEqual(actual, ['x'])
def test_system_merge_geometries_across_shapefiles(self):
geoms_to_union = []
state_names = ('Nebraska', 'South Dakota', 'North Dakota')
gci = GeomCabinetIterator(path=self.path_state_boundaries)
for row in gci:
if row['properties']['STATE_NAME'] in state_names:
geoms_to_union.append(row['geom'])
self.assertEqual(len(geoms_to_union), 3)
unioned = cascaded_union(geoms_to_union)
grid = create_gridxy_global()
field = create_exact_field(grid, 'data', crs=WGS84())
original_shape = field.grid.shape
ops = OcgOperations(dataset=field, geom=unioned)
ret = ops.execute()
actual_shape = ret.get_element().grid.shape
self.assertNotEqual(actual_shape, original_shape)
def test_system_merge_geometries_across_shapefiles(self):
geoms_to_union = []
state_names = ('Nebraska', 'South Dakota', 'North Dakota')
gci = GeomCabinetIterator(path=self.path_state_boundaries)
for row in gci:
if row['properties']['STATE_NAME'] in state_names:
geoms_to_union.append(row['geom'])
self.assertEqual(len(geoms_to_union), 3)
unioned = cascaded_union(geoms_to_union)
grid = create_gridxy_global()
field = create_exact_field(grid, 'data', crs=WGS84())
original_shape = field.grid.shape
ops = OcgOperations(dataset=field, geom=unioned)
ret = ops.execute()
actual_shape = ret.get_element().grid.shape
self.assertNotEqual(actual_shape, original_shape)
def test_system_masking_with_smm(self):
"""Test masking with sparse matrix multiplication."""
from ocgis.regrid import RegridOperation
grid = create_gridxy_global(with_bounds=False,
crs=Spherical(),
dist_dimname='x',
resolution=5.0)
src_field = create_exact_field(grid, 'exact', ntime=3)
mask = src_field.grid.get_mask(create=True)
mask[0:2, :] = True
mask[:, -2:] = True
mask[-2:, :] = True
mask[:, 0:2] = True
src_field.grid.set_mask(mask, cascade=True)
src_field['exact'].set_value(src_field['exact'].mv().filled())
dst_field = deepcopy(src_field)
dst_field.remove_variable('exact')
weights = self.get_temporary_file_path('weights.nc', collective=True)
weights = vm.bcast(weights)
ro = RegridOperation(src_field,
dst_field,
regrid_options={
'weights_out': weights,
'split': False
})
_ = ro.execute()
ro2 = RegridOperation(src_field,
dst_field,
regrid_options={
'weights_in': weights,
'split': True
})
result = ro2.execute()
actual = result['exact'].mv()
desired = src_field['exact'].mv()
self.assertNumpyAllClose(actual, desired)
def test_chunked_rwg_spatial_subset(self):
env.CLOBBER_UNITS_ON_BOUNDS = False
src_grid = create_gridxy_global(crs=Spherical())
src_field = create_exact_field(src_grid, 'foo')
xvar = Variable(name='x', value=[-90., -80.], dimensions='xdim')
yvar = Variable(name='y', value=[40., 50.], dimensions='ydim')
dst_grid = Grid(x=xvar, y=yvar, crs=Spherical())
if ocgis.vm.rank == 0:
source = self.get_temporary_file_path('source.nc')
else:
source = None
source = ocgis.vm.bcast(source)
src_field.write(source)
if ocgis.vm.rank == 0:
destination = self.get_temporary_file_path('destination.nc')
else:
destination = None
destination = ocgis.vm.bcast(destination)
dst_grid.parent.write(destination)
wd = os.path.join(self.current_dir_output, 'chunks')
weight = os.path.join(self.current_dir_output, 'weights.nc')
runner = CliRunner()
cli_args = ['chunked-rwg', '--source', source, '--destination', destination, '--wd', wd, '--spatial_subset',
'--weight', weight, '--esmf_regrid_method', 'BILINEAR', '--persist']
result = runner.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
dst_path = os.path.join(wd, 'spatial_subset.nc')
self.assertTrue(os.path.exists(weight))
actual = RequestDataset(uri=dst_path).create_field()
actual_ymean = actual.grid.get_value_stacked()[0].mean()
actual_xmean = actual.grid.get_value_stacked()[1].mean()
self.assertEqual(actual_ymean, 45.)
self.assertEqual(actual_xmean, -85.)
self.assertEqual(actual.grid.shape, (14, 14))
def test_get_dimension_map_with_spatial_mask(self):
path = self.get_temporary_file_path('foo.nc')
grid = create_gridxy_global()
gmask = grid.get_mask(create=True)
gmask[1, 1] = True
grid.set_mask(gmask)
grid.parent.write(path)
rd = RequestDataset(path)
driver = DriverNetcdfCF(rd)
dmap = driver.get_dimension_map(driver.metadata_source)
self.assertIsNotNone(dmap.get_spatial_mask())
field = rd.get()
self.assertEqual(field.grid.get_mask().sum(), 1)
# Test mask variable is blown away if set to None during a read.
rd = RequestDataset(path)
rd.dimension_map.set_spatial_mask(None)
self.assertIsNone(rd.dimension_map.get_spatial_mask())
# rd.dimension_map.pprint()
field = rd.get()
self.assertIsNone(field.grid.get_mask())
def test_create_dimension_map_with_spatial_mask(self):
path = self.get_temporary_file_path('foo.nc')
grid = create_gridxy_global()
gmask = grid.get_mask(create=True)
gmask[1, 1] = True
grid.set_mask(gmask)
grid.parent.write(path)
rd = RequestDataset(path)
driver = DriverNetcdfCF(rd)
dmap = driver.create_dimension_map(driver.metadata_source)
self.assertIsNotNone(dmap.get_spatial_mask())
field = rd.get()
self.assertEqual(field.grid.get_mask().sum(), 1)
# Test mask variable is blown away if set to None during a read.
rd = RequestDataset(path)
rd.dimension_map.set_spatial_mask(None)
self.assertIsNone(rd.dimension_map.get_spatial_mask())
# rd.dimension_map.pprint()
field = rd.get()
self.assertIsNone(field.grid.get_mask())
def test_iter_spatial_decomposition(self):
self.add_barrier = False
if vm.size not in [1, 4]:
raise SkipTest('vm.size not in [1, 4]')
grid = create_gridxy_global(resolution=10., wrapped=False, crs=Spherical())
splits = (2, 3)
actual = []
for sub, slc in iter_spatial_decomposition(grid, splits, optimized_bbox_subset=True):
root = vm.get_live_ranks_from_object(sub)[0]
with vm.scoped_by_emptyable('test extent', sub):
if vm.is_null:
extent_global = None
else:
extent_global = sub.extent_global
extent_global = vm.bcast(extent_global, root=root)
actual.append(extent_global)
desired = [(0.0, -90.0, 120.0, 0.0), (120.0, -90.0, 240.0, 0.0), (240.0, -90.0, 360.0, 0.0),
(0.0, 0.0, 120.0, 90.0), (120.0, 0.0, 240.0, 90.0), (240.0, 0.0, 360.0, 90.0)]
self.assertEqual(actual, desired)
def test_update_crs_to_cartesian(self):
"""Test a spherical to cartesian CRS update."""
grid = create_gridxy_global(resolution=30.0,
wrapped=True,
crs=Spherical())
z_value = np.ones(grid.shape, dtype=grid.dtype)
zvar = Variable('ocgis_zc', value=z_value, dimensions=grid.dimensions)
grid.z = zvar
z_bounds_value = np.ones((list(zvar.shape) + [4]), dtype=grid.dtype)
zvar_bounds = Variable('ocgis_zc_bounds', z_bounds_value,
list(grid.dimensions) + ['corners'])
grid.z.set_bounds(zvar_bounds)
original_grid = deepcopy(grid)
original_grid.expand()
grid.update_crs(Cartesian())
self.assertIsInstance(grid.crs, Cartesian)
self.assertNotAlmostEquals(original_grid.x.get_value().max(),
grid.x.get_value().max())
self.assertNotAlmostEquals(original_grid.y.get_value().max(),
grid.y.get_value().max())
self.assertNotAlmostEquals(original_grid.x.bounds.get_value().max(),
grid.x.bounds.get_value().max())
self.assertNotAlmostEquals(original_grid.y.bounds.get_value().max(),
grid.y.bounds.get_value().max())
grid.update_crs(Spherical())
self.assertEqual(grid.crs, Spherical())
self.assertNumpyAllClose(original_grid.x.get_value(),
grid.x.get_value())
self.assertNumpyAllClose(original_grid.y.get_value(),
grid.y.get_value())
self.assertNumpyAllClose(original_grid.x.bounds.get_value(),
grid.x.bounds.get_value())
self.assertNumpyAllClose(original_grid.y.bounds.get_value(),
grid.y.bounds.get_value())
self.assertEqual(grid.z.get_value().sum(), 72)
def test_convert_to_geometry_coordinates_polygons(self):
grid = create_gridxy_global(resolution=45.0)
geom = grid.get_abstraction_geometry()
geom.reshape(Dimension('n_elements', geom.size))
keywords = dict(pack=[True, False],
repeat_last_node=[False, True],
start_index=[0, 1])
for k in self.iter_product_keywords(keywords):
actual = geom.convert_to(pack=k.pack,
repeat_last_node=k.repeat_last_node,
start_index=k.start_index)
self.assertEqual(actual.cindex.attrs['start_index'], k.start_index)
self.assertEqual(actual.start_index, k.start_index)
self.assertEqual(actual.cindex.get_value()[0].min(), k.start_index)
self.assertEqual(actual.packed, k.pack)
self.assertIsNotNone(actual.cindex)
for actual_geom, desired_geom in zip(
actual.get_geometry_iterable(),
geom.get_value().flat):
self.assertEqual(actual_geom[1], desired_geom)
def test_get_ocgis_field_from_esmf_field(self):
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
ogrid = create_gridxy_global(crs=Spherical())
ofield = create_exact_field(ogrid, 'foo', ntime=3)
ogrid = ofield.grid
ogrid_mask = ogrid.get_mask(create=True)
ogrid_mask[1, 0] = True
ogrid.set_mask(ogrid_mask)
efield = get_esmf_field_from_ocgis_field(ofield)
self.assertEqual(efield.data.shape, (360, 180, 3))
ofield_actual = get_ocgis_field_from_esmf_field(efield, field=ofield)
actual_dv_mask = ofield_actual.data_variables[0].get_mask()
self.assertTrue(np.all(actual_dv_mask[:, 1, 0]))
self.assertEqual(actual_dv_mask.sum(), 3)
self.assertNumpyAll(ofield.data_variables[0].get_value(), ofield_actual.data_variables[0].get_value())
self.assertEqual(ofield.data_variables[0].name, efield.name)
self.assertNumpyAll(ofield.time.get_value(), ofield_actual.time.get_value())
import ocgis
from ocgis.test.base import create_gridxy_global, create_exact_field
# Name of the variable to subset.
VAR_TAS = 'tas'
# Make it easy to switch to non-snippet requests.
SNIPPET = True
# Set output directory for shapefile and keyed formats. (MAKE SURE IT EXISTS!)
ocgis.env.DIR_OUTPUT = tempfile.mkdtemp()
print ocgis.env.DIR_OUTPUT
# The bounding box coordinates [minx, miny, maxx, maxy] for the state of Colorado in WGS84 latitude/longitude
# coordinates.
BBOX = [-109.1, 36.9, -102.0, 41.0]
# Create synthetic data for this example.
grid = create_gridxy_global(resolution=5.0)
field = create_exact_field(grid, VAR_TAS, ntime=31)
data_path = os.path.join(ocgis.env.DIR_OUTPUT,
'ocgis_example_simple_subset.nc')
field.write(data_path)
# This object will be reused so just build it once. Variable names are typically auto-discovered.
rd = ocgis.RequestDataset(data_path, VAR_TAS)
########################################################################################################################
# Returning an OCGIS spatial collection
ret = ocgis.OcgOperations(dataset=rd, geom=BBOX, snippet=SNIPPET).execute()
########################################################################################################################
# Returning conversions
import time
import ocgis
from ocgis.test.base import create_gridxy_global, create_exact_field
ocgis.env.ADD_OPS_MPI_BARRIER = False
ocgis.env.OVERWRITE = True
# Path to the output netCDf file.
PATH = 'foo.nc'
tic = time.time()
# Create a test grid.
grid = create_gridxy_global()
# Create an exact field on the grid.
field = create_exact_field(grid, 'foo')
# Write the field to disk.
field.write(PATH)
rd = ocgis.RequestDataset(PATH)
# Calculate a monthly mean.
ops = ocgis.OcgOperations(dataset=rd,
calc=[{
'func': 'mean',
'name': 'mean'
}],
calc_grouping=['month'],
prefix='mean',
dir_output='.',
output_format='nc')
def test_system_spatial_averaging_through_operations_state_boundaries(self):
if MPI_SIZE != 8:
raise SkipTest('MPI_SIZE != 8')
ntime = 3
# Get the exact field value for the state's representative center.
with vm.scoped([0]):
if MPI_RANK == 0:
states = RequestDataset(self.path_state_boundaries, driver='vector').get()
states.update_crs(env.DEFAULT_COORDSYS)
fill = np.zeros((states.geom.shape[0], 2))
for idx, geom in enumerate(states.geom.get_value().flat):
centroid = geom.centroid
fill[idx, :] = centroid.x, centroid.y
exact_states = create_exact_field_value(fill[:, 0], fill[:, 1])
state_ugid = states['UGID'].get_value()
area = states.geom.area
keywords = {
'spatial_operation': [
'clip',
'intersects'
],
'aggregate': [
True,
False
],
'wrapped': [True, False],
'output_format': [
OutputFormatName.OCGIS,
'csv',
'csv-shp',
'shp'
],
}
# total_iterations = len(list(self.iter_product_keywords(keywords)))
for ctr, k in enumerate(self.iter_product_keywords(keywords)):
# barrier_print(k)
# if ctr % 1 == 0:
# if vm.is_root:
# print('Iteration {} of {}...'.format(ctr + 1, total_iterations))
with vm.scoped([0]):
if vm.is_root:
grid = create_gridxy_global(resolution=1.0, dist=False, wrapped=k.wrapped)
field = create_exact_field(grid, 'foo', ntime=ntime)
path = self.get_temporary_file_path('foo.nc')
field.write(path)
else:
path = None
path = MPI_COMM.bcast(path)
rd = RequestDataset(path)
ops = OcgOperations(dataset=rd, geom='state_boundaries', spatial_operation=k.spatial_operation,
aggregate=k.aggregate, output_format=k.output_format, prefix=str(ctr),
# geom_select_uid=[8]
)
ret = ops.execute()
# Test area is preserved for a problem element during union. The union's geometry was not fully represented
# in the output.
if k.output_format == 'shp' and k.aggregate and k.spatial_operation == 'clip':
with vm.scoped([0]):
if vm.is_root:
inn = RequestDataset(ret).get()
inn_ugid_idx = np.where(inn['UGID'].get_value() == 8)[0][0]
ugid_idx = np.where(state_ugid == 8)[0][0]
self.assertAlmostEqual(inn.geom.get_value()[inn_ugid_idx].area, area[ugid_idx], places=2)
# Test the overview geometry shapefile is written.
if k.output_format == 'shp':
directory = os.path.split(ret)[0]
contents = os.listdir(directory)
actual = ['_ugid.shp' in c for c in contents]
self.assertTrue(any(actual))
elif k.output_format == 'csv-shp':
directory = os.path.split(ret)[0]
directory = os.path.join(directory, 'shp')
contents = os.listdir(directory)
actual = ['_ugid.shp' in c for c in contents]
self.assertTrue(any(actual))
if not k.aggregate:
actual = ['_gid.shp' in c for c in contents]
self.assertTrue(any(actual))
if k.output_format == OutputFormatName.OCGIS:
geom_keys = ret.children.keys()
all_geom_keys = vm.gather(np.array(geom_keys))
if vm.is_root:
all_geom_keys = hgather(all_geom_keys)
self.assertEqual(len(np.unique(all_geom_keys)), 51)
if k.aggregate:
actual = Dict()
for field, container in ret.iter_fields(yield_container=True):
if not field.is_empty:
ugid = container.geom.ugid.get_value()[0]
actual[ugid]['actual'] = field.data_variables[0].get_value()
actual[ugid]['area'] = container.geom.area[0]
actual = vm.gather(actual)
if vm.is_root:
actual = dgather(actual)
ares = []
actual_areas = []
for ugid_key, v in actual.items():
ugid_idx = np.where(state_ugid == ugid_key)[0][0]
desired = exact_states[ugid_idx]
actual_areas.append(v['area'])
for tidx in range(ntime):
are = np.abs((desired + ((tidx + 1) * 10)) - v['actual'][tidx, 0])
ares.append(are)
if k.spatial_operation == 'clip':
diff = np.abs(np.array(area) - np.array(actual_areas))
self.assertLess(np.max(diff), 1e-6)
self.assertLess(np.mean(diff), 1e-6)
# Test relative errors.
self.assertLess(np.max(ares), 0.031)
self.assertLess(np.mean(ares), 0.009)
import os
import subprocess
import tempfile
import ocgis
from ocgis.test.base import create_gridxy_global
DATADIR = tempfile.mkdtemp(prefix='ocgis_chunked_rwg_')
SRC_CFGRID = os.path.join(DATADIR, 'src.nc')
DST_CFGRID = os.path.join(DATADIR, 'dst.nc')
WEIGHT = os.path.join(DATADIR, 'esmf_weights.nc')
# Write the source and destination grids. The destination grid has a slightly lower spatial resolution. ----------------
srcgrid = create_gridxy_global(crs=ocgis.crs.Spherical())
srcgrid.parent.write(SRC_CFGRID)
dstgrid = create_gridxy_global(resolution=1.33, crs=ocgis.crs.Spherical())
dstgrid.parent.write(DST_CFGRID)
# ----------------------------------------------------------------------------------------------------------------------
# Construct the chunked regrid weight generation command and execute in a subprocess.
cmd = ['mpirun', '-n', str(4), 'ocli', 'chunked-rwg', '-s', SRC_CFGRID, '-d', DST_CFGRID, '-w', WEIGHT, '-n', '5,5']
print(' '.join(cmd))
# Command line looks like:
# mpirun -n 4 ocli chunked-rwg -s src.nc -d dst.nc -w esmf_weights.nc -n 5,5
subprocess.check_call(cmd)
# Inspect the weight file output.
import subprocess
from ocgis import env
from ocgis.test.base import create_gridxy_global
from ocgis.variable.crs import Spherical
SRC_PATH = '/home/benkoziol/htmp/rwg_test_src.nc'
DST_PATH = '/home/benkoziol/htmp/rwg_test_dst.nc'
WGT_PATH = '/home/benkoziol/htmp/rwg_test_weights.nc'
env.CLOBBER_UNITS_ON_BOUNDS = False
src_grid = create_gridxy_global(resolution=10.0, wrapped=False, crs=Spherical())
dst_grid = create_gridxy_global(resolution=20.0, wrapped=False, crs=Spherical())
src_grid.write(SRC_PATH)
dst_grid.write(DST_PATH)
cmd = ['ESMF_RegridWeightGen', '-s', SRC_PATH, '--src_type', 'GRIDSPEC', '-d', DST_PATH, '--dst_type', 'GRIDSPEC',
'-w', WGT_PATH, '--method', 'conserve', '--weight-only']
subprocess.check_call(cmd)
from shapely.geometry import box
import ocgis
from ocgis.test.base import create_gridxy_global, create_exact_field
assert ocgis.vm.size_global == 2
# Create synthetic data for this example. Create and write the data on a single process. Subcommunicators are always
# given a name.
with ocgis.vm.scoped('serial write', [0]):
# There will be null communicators for the duration of the context manager.
if not ocgis.vm.is_null:
grid = create_gridxy_global(resolution=5.0, dist=False)
field = create_exact_field(grid, 'exact', ntime=31)
field.write('ocgis_parallel_example.nc')
# Place a barrier so write can finish. Race conditions can occur with subcommunicators. Generally, barriers are managed
# by the operations.
ocgis.vm.barrier()
# This is our subset geometry.
bbox = [150, 30, 170, 50]
bbox = box(*bbox)
# Load the data from file.
rd = ocgis.RequestDataset('ocgis_parallel_example.nc')
field = rd.get()
# By default, data is distributed along the largest spatial dimension. The x or longitude dimension in this case.
distributed_dimension = field.grid.dimensions[1]
bounds_local = {0: (0, 36), 1: (36, 72)}
import os
import subprocess
import tempfile
import ocgis
from ocgis.test.base import create_gridxy_global
DATADIR = tempfile.mkdtemp(prefix='ocgis_chunked_rwg_')
SRC_CFGRID = os.path.join(DATADIR, 'src.nc')
DST_CFGRID = os.path.join(DATADIR, 'dst.nc')
WEIGHT = os.path.join(DATADIR, 'esmf_weights.nc')
# Write the source and destination grids. The destination grid has a much smaller spatial extent. ----------------------
srcgrid = create_gridxy_global(crs=ocgis.crs.Spherical())
srcgrid.parent.write(SRC_CFGRID)
# Write the destination grid. Slice the grid first to create a single cell.
dstgrid = create_gridxy_global(resolution=5, crs=ocgis.crs.Spherical())
dstgrid = dstgrid[18, 36]
dstgrid.parent.write(DST_CFGRID)
# ----------------------------------------------------------------------------------------------------------------------
# Construct the chunked regrid weight generation command and execute in a subprocess.
cmd = [
'ocli', 'chunked-rwg', '-s', SRC_CFGRID, '-d', DST_CFGRID, '-w', WEIGHT,
'--spatial_subset'
]
print(' '.join(cmd))
# Command looks like:
def test_system_chunked_versus_global(self):
"""Test weight files are equivalent using the chunked versus global weight generation and SMM approach."""
if ocgis.vm.size not in [1, 4]:
raise SkipTest('ocgis.vm.size not in [1, 4]')
import ESMF
# Do not put units on bounds variables.
env.CLOBBER_UNITS_ON_BOUNDS = False
# Create source and destination files. -------------------------------------------------------------------------
src_grid = create_gridxy_global(resolution=15, dist_dimname='x')
dst_grid = create_gridxy_global(resolution=12, dist_dimname='x')
src_field = create_exact_field(src_grid, 'foo', crs=Spherical(), dtype=np.float64)
dst_field = create_exact_field(dst_grid, 'foo', crs=Spherical(), dtype=np.float64)
if ocgis.vm.rank == 0:
source = self.get_temporary_file_path('source.nc')
else:
source = None
source = ocgis.vm.bcast(source)
src_field.write(source)
if ocgis.vm.rank == 0:
destination = self.get_temporary_file_path('destination.nc')
else:
destination = None
destination = ocgis.vm.bcast(destination)
dst_field['foo'].v()[:] = -9999
dst_field.write(destination)
# --------------------------------------------------------------------------------------------------------------
# Directory for output grid chunks.
wd = os.path.join(self.current_dir_output, 'chunks')
# Path to the merged weight file.
weight = self.get_temporary_file_path('merged_weights.nc')
# Generate the source and destination chunks and a merged weight file.
runner = CliRunner()
cli_args = ['chunked-rwg', '--source', source, '--destination', destination, '--nchunks_dst', '2,3', '--wd',
wd, '--weight', weight, '--persist']
result = runner.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
self.assertTrue(len(os.listdir(wd)) > 3)
# Also apply the sparse matrix
runner2 = CliRunner()
cli_args = ['chunked-smm', '--wd', wd, '--insert_weighted', '--destination', destination]
result = runner2.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
# Create a standard ESMF weights file from the original grid files.
esmf_weights_path = self.get_temporary_file_path('esmf_desired_weights.nc')
# Generate weights using ESMF command line interface.
# cmd = ['ESMF_RegridWeightGen', '-s', source, '--src_type', 'GRIDSPEC', '-d', destination, '--dst_type',
# 'GRIDSPEC', '-w', esmf_weights_path, '--method', 'conserve', '--no-log']
# subprocess.check_call(cmd)
# Create a weights file using the ESMF Python interface.
srcgrid = ESMF.Grid(filename=source, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True)
dstgrid = ESMF.Grid(filename=destination, filetype=ESMF.FileFormat.GRIDSPEC, add_corner_stagger=True)
srcfield = ESMF.Field(grid=srcgrid, typekind=ESMF.TypeKind.R8)
srcfield.data[:] = np.swapaxes(np.squeeze(src_field['foo'].v()), 0, 1)
dstfield = ESMF.Field(grid=dstgrid, typekind=ESMF.TypeKind.R8)
_ = ESMF.Regrid(srcfield=srcfield, dstfield=dstfield, filename=esmf_weights_path,
regrid_method=ESMF.RegridMethod.CONSERVE)
if ocgis.vm.rank == 0:
# Assert the weight files are equivalent using chunked versus global creation.
self.assertWeightFilesEquivalent(esmf_weights_path, weight)
actual_dst = RequestDataset(uri=destination, decomp_type=DecompositionType.ESMF).create_field()['foo'].v()
actual_dst = np.swapaxes(np.squeeze(actual_dst), 0, 1)
desired_dst = dstfield.data
self.assertNumpyAllClose(actual_dst, desired_dst)
select_y1 = py >= bbox[1]
select_y2 = py <= bbox[3]
select_y = np.logical_and(select_y1, select_y2)
select = np.logical_and(select_x, select_y)
if initial is not None:
select = np.logical_or(select, initial)
return select
if __name__ == '__main__':
# ------------------------------------------------------------------------------------------------------------------
# Grid splitter implementation
resolution = 1. / 111.
# resolution = 1.
grid = create_gridxy_global(resolution=resolution, wrapped=False, crs=ocgis.crs.Spherical())
field = create_exact_field(grid, 'exact', ntime=3, fill_data_var=False, crs=ocgis.crs.Spherical())
field.write(os.path.join(OUTDIR, 'dst_field_1km.nc'))
gs = GridChunker(grid, grid, (10, 10))
ctr = 1
for grid_sub in gs.iter_dst_grid_subsets():
subset_filename = os.path.join(OUTDIR, 'src_subset_{}.nc'.format(ctr))
dst_subset_filename = os.path.join(OUTDIR, 'dst_subset_{}.nc'.format(ctr))
if vm.rank == 0:
print 'creating subset:', subset_filename
def test_write_esmf_weights(self):
# Create source and destination fields. This is the identity test, so the source and destination fields are
# equivalent.
src_grid = create_gridxy_global(resolution=3.0, crs=Spherical())
# Only test masking in serial to make indexing easier...just being lazy
if vm.size == 1:
mask = src_grid.get_mask(create=True)
mask[4, 5] = True
mask[25, 27] = True
src_grid.set_mask(mask)
self.assertEqual(src_grid.get_mask().sum(), 2)
src_field = create_exact_field(src_grid, 'foo', ntime=3)
dst_field = deepcopy(src_field)
# Write the fields to disk for use in global file reconstruction and testing.
if vm.rank == 0:
master_path = self.get_temporary_file_path('foo.nc')
src_field_path = self.get_temporary_file_path('src_field.nc')
else:
master_path = None
src_field_path = None
master_path = vm.bcast(master_path)
src_field_path = vm.bcast(src_field_path)
assert not os.path.exists(master_path)
dst_field.write(master_path)
src_field.write(src_field_path)
# Remove the destination data variable to test its creation and filling
dst_field.remove_variable('foo')
# Chunk the fields and generate weights
paths = {'wd': self.current_dir_output}
gc = GridChunker(src_field,
dst_field,
nchunks_dst=(2, 2),
genweights=True,
paths=paths,
esmf_kwargs={'regrid_method': 'BILINEAR'})
gc.write_chunks()
# This is the path to the index file describing how to reconstruct the grid file
index_path = os.path.join(self.current_dir_output,
gc.paths['index_file'])
# Execute the sparse matrix multiplication using weights read from file
gc.smm(index_path, paths['wd'])
with vm.scoped('index and reconstruct', [0]):
if not vm.is_null:
# Reconstruct the global destination file
gc.insert_weighted(index_path, self.current_dir_output,
master_path)
# Load the actual values from file (destination)
actual_field = RequestDataset(master_path).create_field()
actual = actual_field.data_variables[0].mv()
# Load the desired data from file (original values in the source field)
desired = RequestDataset(
src_field_path).create_field().data_variables[0].mv()
if vm.size_global == 1: # Masking is only tested in serial
self.assertEqual(actual_field.grid.get_mask().sum(), 2)
else:
self.assertIsNone(actual_field.grid.get_mask())
self.assertNumpyAll(actual, desired)
from shapely.geometry import box
import ocgis
from ocgis.test.base import create_gridxy_global, create_exact_field
assert ocgis.vm.size_global == 2
# Create synthetic data for this example. Create and write the data on a single process. Subcommunicators are always
# given a name.
with ocgis.vm.scoped('serial write', [0]):
# There will be null communicators for the duration of the context manager.
if not ocgis.vm.is_null:
grid = create_gridxy_global(resolution=5.0, dist=False)
field = create_exact_field(grid, 'exact', ntime=31)
field.write('ocgis_parallel_example.nc')
# Place a barrier so write can finish. Race conditions can occur with subcommunicators. Generally, barriers are managed
# by the operations.
ocgis.vm.barrier()
# This is our subset geometry.
bbox = [150, 30, 170, 50]
bbox = box(*bbox)
# Load the data from file.
rd = ocgis.RequestDataset('ocgis_parallel_example.nc')
field = rd.get()
# By default, data is distributed along the largest spatial dimension. The x or longitude dimension in this case.
distributed_dimension = field.grid.dimensions[1]
bounds_local = {0: (0, 36), 1: (36, 72)}
def test_system_negative_values_in_spherical_grid(self):
original_dir = os.getcwd()
try:
xcn = np.arange(-10, 350, step=10, dtype=float)
xc = np.arange(0, 360, step=10, dtype=float)
yc = np.arange(-90, 100, step=10, dtype=float)
xvn = Variable("lon", xcn, dimensions=["lon"])
xv = Variable("lon", xc, dimensions=["lon"])
yv = Variable("lat", yc, dimensions=["lat"])
gridn = Grid(x=xvn.copy(), y=yv.copy(), crs=Spherical())
gridu = Grid(x=xv.copy(), y=yv.copy(), crs=Spherical())
gridw = create_gridxy_global(5, with_bounds=False, crs=Spherical())
grids = [gridn, gridu, gridw]
for ctr, (src, dst) in enumerate(itertools.product(grids, grids)):
os.chdir(self.current_dir_output)
gdirname = "grid-ctr-{}".format(ctr)
self.dprint(gdirname)
griddir = os.path.join(self.current_dir_output, gdirname)
os.mkdir(gdirname)
os.chdir(gdirname)
srcgridname = "gridn.nc"
src.parent.write(srcgridname)
dstgridname = "grid.nc"
dst.parent.write(dstgridname)
nchunks_dst = [(4, 1), (3, 1), (2, 1), (1, 1)]
for ctr, n in enumerate(nchunks_dst):
os.chdir(griddir)
dirname = 'ctr-{}'.format(ctr)
os.mkdir(dirname)
os.chdir(dirname)
wd = os.getcwd()
self.dprint("current chunks", n)
g = GridChunker(src,
dst,
nchunks_dst=n,
genweights=True,
paths={'wd': wd},
esmf_kwargs={'regrid_method': 'BILINEAR'})
if not g.is_one_chunk:
g.write_chunks()
g.create_merged_weight_file(
os.path.join(griddir, "ctr-{}".format(ctr),
"merged-weights.nc"))
else:
g.write_esmf_weights(
os.path.join(griddir, srcgridname),
os.path.join(griddir, dstgridname),
os.path.join(griddir, "global-weights.nc"))
os.chdir(griddir)
for ctr in range(0, len(nchunks_dst) - 1):
src_filename = os.path.join(griddir, "ctr-{}".format(ctr),
"merged-weights.nc")
dst_filename = os.path.join(griddir, "global-weights.nc")
self.assertWeightFilesEquivalent(src_filename,
dst_filename)
finally:
os.chdir(original_dir)
def run_create_merged_weight_file(self, filemode):
import ESMF
esmf_filemode = getattr(ESMF.FileMode, filemode)
path_src = self.get_temporary_file_path('src.nc')
path_dst = self.get_temporary_file_path('dst.nc')
src_grid = create_gridxy_global(resolution=30.0,
wrapped=False,
crs=Spherical())
dst_grid = create_gridxy_global(resolution=35.0,
wrapped=False,
crs=Spherical())
src_grid.write(path_src)
dst_grid.write(path_dst)
# Split source and destination grids ---------------------------------------------------------------------------
src_rd = RequestDataset(path_src, driver='netcdf-cf')
dst_rd = RequestDataset(path_dst, driver='netcdf-cf')
gs = GridChunker(src_rd,
dst_rd, (2, 2),
check_contains=False,
allow_masked=True,
paths=self.fixture_paths,
genweights=True,
filemode=filemode)
gs.write_chunks()
if filemode == "WITHAUX":
weightfile = self.get_temporary_file_path('esmf_weights_1.nc')
vc = RequestDataset(weightfile, driver='netcdf').create_field()
self.assertGreater(len(vc.keys()), 3)
weightfile = self.get_temporary_file_path('esmf_weights_2.nc')
vc = RequestDataset(weightfile, driver='netcdf').get()
self.assertEqual(len(vc.keys()), 3)
# Merge weight files -------------------------------------------------------------------------------------------
merged_weight_filename = self.get_temporary_file_path(
'merged_weights.nc')
gs.create_merged_weight_file(merged_weight_filename)
nvars = len(
RequestDataset(merged_weight_filename,
driver='netcdf').get().keys())
if filemode == "WITHAUX":
self.assertGreater(nvars, 3)
else:
self.assertEqual(nvars, 3)
# Generate a global weight file using ESMF ---------------------------------------------------------------------
global_weights_filename = self.get_temporary_file_path(
'global_weights.nc')
srcgrid = ESMF.Grid(filename=path_src,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
dstgrid = ESMF.Grid(filename=path_dst,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
srcfield = ESMF.Field(grid=srcgrid)
dstfield = ESMF.Field(grid=dstgrid)
_ = ESMF.Regrid(srcfield=srcfield,
dstfield=dstfield,
filename=global_weights_filename,
regrid_method=ESMF.RegridMethod.CONSERVE,
filemode=esmf_filemode,
src_file=path_src,
dst_file=path_dst,
src_file_type=ESMF.FileFormat.GRIDSPEC,
dst_file_type=ESMF.FileFormat.GRIDSPEC)
# Test merged and global weight files are equivalent -----------------------------------------------------------
self.assertWeightFilesEquivalent(global_weights_filename,
merged_weight_filename)
def test_write_esmf_weights(self):
# Create source and destination fields. This is the identity test, so the source and destination fields are
# equivalent.
src_grid = create_gridxy_global(resolution=3.0, crs=Spherical())
# Only test masking in serial to make indexing easier...just being lazy
if vm.size == 1:
mask = src_grid.get_mask(create=True)
mask[4, 5] = True
mask[25, 27] = True
src_grid.set_mask(mask)
self.assertEqual(src_grid.get_mask().sum(), 2)
src_field = create_exact_field(src_grid, 'foo', ntime=3)
dst_field = deepcopy(src_field)
# Write the fields to disk for use in global file reconstruction and testing.
if vm.rank == 0:
master_path = self.get_temporary_file_path('foo.nc')
src_field_path = self.get_temporary_file_path('src_field.nc')
else:
master_path = None
src_field_path = None
master_path = vm.bcast(master_path)
src_field_path = vm.bcast(src_field_path)
assert not os.path.exists(master_path)
dst_field.write(master_path)
src_field.write(src_field_path)
# Remove the destination data variable to test its creation and filling
dst_field.remove_variable('foo')
# Chunk the fields and generate weights
paths = {'wd': self.current_dir_output}
gc = GridChunker(src_field, dst_field, nchunks_dst=(2, 2), genweights=True, paths=paths,
esmf_kwargs={'regrid_method': 'BILINEAR'})
gc.write_chunks()
# This is the path to the index file describing how to reconstruct the grid file
index_path = os.path.join(self.current_dir_output, gc.paths['index_file'])
# Execute the sparse matrix multiplication using weights read from file
gc.smm(index_path, paths['wd'])
with vm.scoped('index and reconstruct', [0]):
if not vm.is_null:
# Reconstruct the global destination file
gc.insert_weighted(index_path, self.current_dir_output, master_path)
# Load the actual values from file (destination)
actual_field = RequestDataset(master_path).create_field()
actual = actual_field.data_variables[0].mv()
# Load the desired data from file (original values in the source field)
desired = RequestDataset(src_field_path).create_field().data_variables[0].mv()
if vm.size_global == 1: # Masking is only tested in serial
self.assertEqual(actual_field.grid.get_mask().sum(), 2)
else:
self.assertIsNone(actual_field.grid.get_mask())
self.assertNumpyAll(actual, desired)
def test_system_chunked_versus_global(self):
"""Test weight files are equivalent using the chunked versus global weight generation approach."""
if ocgis.vm.size not in [1, 4]:
raise SkipTest('ocgis.vm.size not in [1, 4]')
import ESMF
# Do not put units on bounds variables.
env.CLOBBER_UNITS_ON_BOUNDS = False
# Create source and destination files. -------------------------------------------------------------------------
src_grid = create_gridxy_global(resolution=15)
dst_grid = create_gridxy_global(resolution=12)
src_field = create_exact_field(src_grid, 'foo', crs=Spherical())
dst_field = create_exact_field(dst_grid, 'foo', crs=Spherical())
if ocgis.vm.rank == 0:
source = self.get_temporary_file_path('source.nc')
else:
source = None
source = ocgis.vm.bcast(source)
src_field.write(source)
if ocgis.vm.rank == 0:
destination = self.get_temporary_file_path('destination.nc')
else:
destination = None
destination = ocgis.vm.bcast(destination)
dst_field.write(destination)
# --------------------------------------------------------------------------------------------------------------
# Directory for output grid chunks.
wd = os.path.join(self.current_dir_output, 'chunks')
# Path to the merged weight file.
weight = self.get_temporary_file_path('merged_weights.nc')
# Generate the source and destination chunks and a merged weight file.
runner = CliRunner()
cli_args = [
'chunked_rwg', '--source', source, '--destination', destination,
'--nchunks_dst', '2,3', '--wd', wd, '--weight', weight, '--persist'
]
result = runner.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
self.assertTrue(len(os.listdir(wd)) > 3)
# Create a standard ESMF weights file from the original grid files.
esmf_weights_path = self.get_temporary_file_path(
'esmf_desired_weights.nc')
# Generate weights using ESMF command line interface.
# cmd = ['ESMF_RegridWeightGen', '-s', source, '--src_type', 'GRIDSPEC', '-d', destination, '--dst_type',
# 'GRIDSPEC', '-w', esmf_weights_path, '--method', 'conserve', '--no-log']
# subprocess.check_call(cmd)
# Create a weights file using the ESMF Python interface.
srcgrid = ESMF.Grid(filename=source,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
dstgrid = ESMF.Grid(filename=destination,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
srcfield = ESMF.Field(grid=srcgrid)
dstfield = ESMF.Field(grid=dstgrid)
_ = ESMF.Regrid(srcfield=srcfield,
dstfield=dstfield,
filename=esmf_weights_path,
regrid_method=ESMF.RegridMethod.CONSERVE)
if ocgis.vm.rank == 0:
# Assert the weight files are equivalent using chunked versus global creation.
self.assertWeightFilesEquivalent(esmf_weights_path, weight)
""" Provides a general overview of the OCGIS spatial collection object. """ from ocgis import SpatialCollection, GeometryVariable, Variable, crs from ocgis.test.base import create_gridxy_global, create_exact_field from ocgis.util.helpers import pprint_dict from shapely.geometry import box # Spatial collections are hierarchical and may contain groups (children). In OCGIS, the first spatial collection groups # are subset geometries. The children or groups within the subset geometry groups contain the data subset by the # geometry. This is easiest to understand by walking through how OCGIS subsets and assembles a spatial collection inside # operations. # Create a test grid. grid = create_gridxy_global() # Create an exact field on the grid. field = create_exact_field(grid, 'foo') # Create a subset geometry. subset_geom = box(30., 20., 40., 25.) # Subset the field using the geometry. Note we subset the grid and return its parent (the field). sub = field.grid.get_intersects(subset_geom).parent # Subset geometries are themselves treated as fields. Convert the geometry to an OCGIS geometry variable. gvar = GeometryVariable.from_shapely(subset_geom, ugid=11, crs=crs.Spherical()) # Add some descriptive variables. info = Variable(name='desc', value=['random bbox'], dtype=str, dimensions=gvar.dimensions[0])
import subprocess
from ocgis import env
from ocgis.test.base import create_gridxy_global
from ocgis.variable.crs import Spherical
SRC_PATH = '/home/benkoziol/htmp/rwg_test_src.nc'
DST_PATH = '/home/benkoziol/htmp/rwg_test_dst.nc'
WGT_PATH = '/home/benkoziol/htmp/rwg_test_weights.nc'
env.CLOBBER_UNITS_ON_BOUNDS = False
src_grid = create_gridxy_global(resolution=10.0,
wrapped=False,
crs=Spherical())
dst_grid = create_gridxy_global(resolution=20.0,
wrapped=False,
crs=Spherical())
src_grid.write(SRC_PATH)
dst_grid.write(DST_PATH)
cmd = [
'ESMF_RegridWeightGen', '-s', SRC_PATH, '--src_type', 'GRIDSPEC', '-d',
DST_PATH, '--dst_type', 'GRIDSPEC', '-w', WGT_PATH, '--method', 'conserve',
'--weight-only'
]
subprocess.check_call(cmd)
from ocgis import RequestDataset
from ocgis.test.base import create_gridxy_global, create_exact_field
from ocgis.variable.crs import Spherical
# Create synthetic data for this example.
grid = create_gridxy_global(resolution=5.0)
field = create_exact_field(grid, 'exact', ntime=31, crs=Spherical())
field.write('ocgis_example_inspect_request_dataset.nc')
# Create the request dataset object.
rd = RequestDataset('ocgis_example_inspect_request_dataset.nc')
# Provides a metadata dump for the request dataset.
rd.inspect()
# These are the auto-discovered data variable names.
assert rd.variable == 'exact'
# The dimension map provides information on how OCGIS will interpret the dataset.
rd.dimension_map.pprint()