def test_update_crs_to_cartesian(self):
"""Test a spherical to cartesian CRS update."""
bbox = box(-170., 40., 150., 80.)
original_bounds = deepcopy(bbox.bounds)
geom = GeometryVariable(name='geom',
value=[bbox],
dimensions='geom',
crs=Spherical())
other_crs = Cartesian()
geom.update_crs(other_crs)
actual = geom.get_value()[0].bounds
desired = (-0.7544065067354889, -0.13302222155948895,
-0.15038373318043535, 0.38302222155948895)
self.assertNumpyAllClose(np.array(actual), np.array(desired))
self.assertIsInstance(geom.crs, Cartesian)
other_crs = Spherical()
geom.update_crs(other_crs)
self.assertEqual(geom.crs, Spherical())
actual = geom.get_value()[0].bounds
self.assertNumpyAllClose(np.array(original_bounds), np.array(actual))
# Test data may not be wrapped.
bbox = box(0, 40, 270, 80)
geom = GeometryVariable(name='geom',
value=[bbox],
dimensions='geom',
crs=Spherical())
other_crs = Cartesian()
with self.assertRaises(ValueError):
geom.update_crs(other_crs)
def test_system_spatial_subsetting(self):
"""Test spatial subsetting ESMF Unstructured format."""
bbox = shapely.geometry.box(*[-119.2, 61.7, -113.2, 62.7])
gvar = GeometryVariable(name='geom',
value=bbox,
is_bbox=True,
dimensions='ngeom',
crs=Spherical())
gvar.unwrap()
rd = RequestDataset(uri=self.path_esmf_unstruct,
driver=DriverESMFUnstruct,
crs=Spherical(),
grid_abstraction='point',
grid_is_isomorphic=True)
field = rd.create_field()
sub, slc = field.grid.get_intersects(gvar,
optimized_bbox_subset=True,
return_slice=True)
desired_extent = np.array(
(240.890625, 61.8046875, 246.796875, 62.6484375))
self.assertGreaterEqual(len(vm.get_live_ranks_from_object(sub)), 1)
with vm.scoped_by_emptyable('reduction', sub):
if not vm.is_null:
red = sub.reduce_global()
self.assertNumpyAllClose(desired_extent,
np.array(red.extent_global))
path = self.get_temporary_file_path('foo.nc', collective=True)
with vm.scoped_by_emptyable('write', sub):
if not vm.is_null:
red.parent.write(path)
def test_regrid_field_different_grid_shapes(self):
"""Test regridding a downscaled dataset to GCM output. The input and output grids have different shapes."""
downscaled = self.test_data.get_rd('maurer_2010_tas')
downscaled.time_region = {'month': [2], 'year': [1990]}
downscaled = downscaled.get()
poly = make_poly([37, 43], [-104, -94])
downscaled = downscaled.grid.get_intersects(poly).parent
downscaled.unwrap()
downscaled.set_crs(Spherical())
gcm = self.test_data.get_rd('cancm4_tas')
gcm = gcm.get()
poly = make_poly([37, 43], [-104 + 360, -94 + 360])
gcm = gcm.grid.get_intersects(poly).parent
gcm.set_crs(Spherical())
self.assertIsNone(downscaled.grid.get_mask())
self.assertIsNone(gcm.grid.get_mask())
from ocgis.regrid.base import regrid_field
regridded = regrid_field(downscaled, gcm)
dv = regridded.data_variables[0]
self.assertEqual(dv.shape, (28, 3, 5))
self.assertEqual(dv.name, 'tas')
vmask = dv.get_mask()
self.assertEqual(vmask.sum(), 252)
def test_crs(self):
"""Test overloading by geometry and grid."""
field = Field()
self.assertIsNone(field.crs)
geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g', crs=Spherical())
field = Field(geom=geom)
self.assertEqual(field.crs, geom.crs)
grid = self.get_gridxy_global(crs=Spherical())
field = Field(grid=grid)
self.assertEqual(field.crs, grid.crs)
grid = self.get_gridxy_global(crs=Spherical())
# Grid and field coordinate systems do not match.
with self.assertRaises(ValueError):
Field(grid=grid, crs=WGS84())
geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g', crs=Spherical())
with self.assertRaises(ValueError):
Field(geom=geom, crs=WGS84())
geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g')
grid = self.get_gridxy_global()
field = Field(geom=geom, grid=grid, crs=WGS84())
self.assertEqual(field.crs, WGS84())
self.assertEqual(field.geom.crs, WGS84())
self.assertEqual(field.grid.crs, WGS84())
g = self.get_gridxy_global()
f = Field(grid=g, crs=Spherical())
self.assertIn('standard_name', f.grid.x.attrs)
self.assertIn('standard_name', f.grid.y.attrs)
def test_system_regridding_crs(self):
"""Test with coordinate systems."""
dest_crs = WGS84()
grid_spherical = self.get_gridxy_global(resolution=10.0, wrapped=False, crs=Spherical())
self.assertEqual(grid_spherical.crs, Spherical())
coords = grid_spherical.get_value_stacked()
data_value = self.get_exact_field_value(coords[1], coords[0])
desired = data_value.copy()
data_var = Variable(name='data_src', value=data_value, dimensions=grid_spherical.dimensions)
source = Field(grid=grid_spherical, is_data=data_var, crs=grid_spherical.crs)
self.assertEqual(source.crs, Spherical())
destination = deepcopy(source)
destination.update_crs(dest_crs)
source_expanded = deepcopy(source.grid)
source_expanded.expand()
diff = np.abs(destination.y.get_value() - source_expanded.y.get_value())
self.assertAlmostEqual(diff.max(), 0.19231511439)
for output_crs in [None, WGS84()]:
ops = OcgOperations(dataset=source, regrid_destination=destination, output_crs=output_crs)
ret = ops.execute()
actual = ret.get_element(variable_name=data_var.name)
if output_crs is None:
self.assertEqual(actual.parent.crs, Spherical())
else:
self.assertEqual(actual.parent.crs, WGS84())
actual = actual.get_value()
diff = np.abs(actual - desired)
self.assertTrue(diff.max() < 1e-5)
def test_create_merged_weight_file_unstructured(self):
self.remove_dir = False
ufile = self.get_temporary_file_path('ugrid.nc')
resolution = 10.
self.fixture_regular_ugrid_file(ufile, resolution, crs=Spherical())
src_grid = RequestDataset(ufile, driver=DriverNetcdfUGRID, grid_abstraction='point').get().grid
self.assertEqual(src_grid.abstraction, 'point')
dst_grid = self.get_gridxy_global(resolution=20., crs=Spherical())
dst_path = self.get_temporary_file_path('dst.nc')
dst_grid.parent.write(dst_path)
gs = GridSplitter(src_grid, dst_grid, (3, 3), check_contains=False, src_grid_resolution=10.,
paths=self.fixture_paths)
gs.write_subsets()
# Load the grid splitter index file ----------------------------------------------------------------------------
index_path = gs.create_full_path_from_template('index_file')
ifile = RequestDataset(uri=index_path).get()
ifile.load()
gidx = ifile[GridSplitterConstants.IndexFile.NAME_INDEX_VARIABLE].attrs
source_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_SOURCE_VARIABLE]]
sv = source_filename.join_string_value()
destination_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_DESTINATION_VARIABLE]]
dv = destination_filename.join_string_value()
# Create weight files for each subset --------------------------------------------------------------------------
for ii, sfn in enumerate(sv):
esp = os.path.join(self.current_dir_output, sfn)
edp = os.path.join(self.current_dir_output, dv[ii])
ewp = gs.create_full_path_from_template('wgt_template', index=ii + 1)
cmd = ['ESMF_RegridWeightGen', '-s', esp, '--src_type', 'UGRID', '--src_meshname',
VariableName.UGRID_HOST_VARIABLE, '-d', edp, '--dst_type', 'GRIDSPEC', '-w', ewp, '--method',
'conserve', '-r', '--no_log']
subprocess.check_call(cmd)
# Merge weight files -------------------------------------------------------------------------------------------
mwf = self.get_temporary_file_path('merged_weight_file.nc')
gs.create_merged_weight_file(mwf)
# Generate a global weight file using ESMF ---------------------------------------------------------------------
global_weights_filename = self.get_temporary_file_path('global_weights.nc')
cmd = ['ESMF_RegridWeightGen', '-s', ufile, '--src_type', 'UGRID', '-d', dst_path, '--dst_type',
'GRIDSPEC', '-w', global_weights_filename, '--method', 'conserve', '--weight-only', '--no_log',
'--src_meshname', VariableName.UGRID_HOST_VARIABLE]
subprocess.check_call(cmd)
# Test merged and global weight files are equivalent -----------------------------------------------------------
self.assertWeightFilesEquivalent(global_weights_filename, mwf)
def test_deepcopy(self):
gvar = GeometryVariable(value=Point(1, 2),
crs=Spherical(),
dimensions='d')
self.assertEqual(gvar.crs, Spherical())
d = gvar.deepcopy()
d.crs = WGS84()
self.assertEqual(gvar.crs, Spherical())
self.assertFalse(np.may_share_memory(gvar.get_value(), d.get_value()))
self.assertIsNotNone(d.crs)
def test_create_merged_weight_file(self):
import ESMF
path_src = self.get_temporary_file_path('src.nc')
path_dst = self.get_temporary_file_path('dst.nc')
src_grid = create_gridxy_global(resolution=30.0,
wrapped=False,
crs=Spherical())
dst_grid = create_gridxy_global(resolution=35.0,
wrapped=False,
crs=Spherical())
src_grid.write(path_src)
dst_grid.write(path_dst)
# Split source and destination grids ---------------------------------------------------------------------------
gs = GridChunker(src_grid,
dst_grid, (2, 2),
check_contains=False,
allow_masked=True,
paths=self.fixture_paths,
genweights=True)
gs.write_chunks()
# Merge weight files -------------------------------------------------------------------------------------------
merged_weight_filename = self.get_temporary_file_path(
'merged_weights.nc')
gs.create_merged_weight_file(merged_weight_filename)
# Generate a global weight file using ESMF ---------------------------------------------------------------------
global_weights_filename = self.get_temporary_file_path(
'global_weights.nc')
srcgrid = ESMF.Grid(filename=path_src,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
dstgrid = ESMF.Grid(filename=path_dst,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
srcfield = ESMF.Field(grid=srcgrid)
dstfield = ESMF.Field(grid=dstgrid)
_ = ESMF.Regrid(srcfield=srcfield,
dstfield=dstfield,
filename=global_weights_filename,
regrid_method=ESMF.RegridMethod.CONSERVE)
# Test merged and global weight files are equivalent -----------------------------------------------------------
self.assertWeightFilesEquivalent(global_weights_filename,
merged_weight_filename)
def test_system_cf_data(self):
rd = self.test_data.get_rd('cancm4_tas')
path = self.get_temporary_file_path('grid.shp')
field = rd.get()[{'time': slice(3, 6), 'lat': slice(10, 20), 'lon': slice(21, 27)}]
variable_names = ['time', 'lat', 'lon', 'tas']
field.set_abstraction_geom()
field.write(path, driver=DriverVector, variable_names=variable_names)
read = RequestDataset(path).get()
self.assertEqual(len(list(read.dimensions.values())[0]), 3 * 10 * 6)
self.assertEqual(read.crs, Spherical())
self.assertEqual(rd.get().crs, Spherical())
def test_system_spatial_wrapping_and_reorder(self):
if sys.version_info.major == 3 and sys.version_info.minor == 5:
raise SkipTest('undefined behavior with Python 3.5')
keywords = {'spatial_wrapping': list(SpatialWrapping.iter_possible()),
'crs': [None, Spherical(), CoordinateReferenceSystem(epsg=2136)],
'unwrapped': [True, False],
'spatial_reorder': [False, True]}
for ctr, k in enumerate(self.iter_product_keywords(keywords)):
field = self.get_wrap_field(crs=k.crs, unwrapped=k.unwrapped)
ops = OcgOperations(dataset=field, spatial_wrapping=k.spatial_wrapping, spatial_reorder=k.spatial_reorder)
ret = ops.execute()
actual_field = ret.get_element()
with vm.scoped_by_emptyable('wrapped state', actual_field):
if not vm.is_null:
actual = actual_field.wrapped_state
else:
actual = None
actual_x = actual_field.grid.x.get_value()
if not actual_field.is_empty:
self.assertLessEqual(actual_x.max(), 360.)
if k.spatial_reorder and k.unwrapped and k.spatial_wrapping == 'wrap' and k.crs == Spherical():
actual_data_value = actual_field.data_variables[0].get_value()
desired_reordered = [None] * actual_data_value.shape[1]
for idx in range(actual_data_value.shape[1]):
desired_reordered[idx] = [3.0, 4.0, 0.0, 1.0, 2.0]
for tidx in range(actual_data_value.shape[0]):
time_data_value = actual_data_value[tidx]
self.assertEqual(time_data_value.tolist(), desired_reordered)
if k.spatial_reorder and not k.unwrapped and not k.spatial_wrapping:
self.assertTrue(actual_x[0] < actual_x[-1])
if actual is None or k.crs != Spherical():
desired = None
else:
p = k.spatial_wrapping
if p is None:
if k.unwrapped:
desired = WrappedState.UNWRAPPED
else:
desired = WrappedState.WRAPPED
elif p == 'wrap':
desired = WrappedState.WRAPPED
else:
desired = WrappedState.UNWRAPPED
self.assertEqual(actual, desired)
def test_create_merged_weight_file(self):
path_src = self.get_temporary_file_path('src.nc')
path_dst = self.get_temporary_file_path('dst.nc')
src_grid = create_gridxy_global(resolution=30.0, wrapped=False, crs=Spherical())
dst_grid = create_gridxy_global(resolution=35.0, wrapped=False, crs=Spherical())
src_grid.write(path_src)
dst_grid.write(path_dst)
# Split source and destination grids ---------------------------------------------------------------------------
gs = GridSplitter(src_grid, dst_grid, (2, 2), check_contains=False, allow_masked=True, paths=self.fixture_paths)
gs.write_subsets()
# Load the grid splitter index file ----------------------------------------------------------------------------
index_filename = gs.create_full_path_from_template('index_file')
ifile = RequestDataset(uri=index_filename).get()
ifile.load()
gidx = ifile[GridSplitterConstants.IndexFile.NAME_INDEX_VARIABLE].attrs
source_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_SOURCE_VARIABLE]]
sv = source_filename.join_string_value()
destination_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_DESTINATION_VARIABLE]]
dv = destination_filename.join_string_value()
# Create weight files for each subset --------------------------------------------------------------------------
for ii, sfn in enumerate(sv):
esp = os.path.join(self.current_dir_output, sfn)
edp = os.path.join(self.current_dir_output, dv[ii])
ewp = gs.create_full_path_from_template('wgt_template', index=ii + 1)
cmd = ['ESMF_RegridWeightGen', '-s', esp, '--src_type', 'GRIDSPEC', '-d', edp, '--dst_type',
'GRIDSPEC', '-w', ewp, '--method', 'conserve', '-r', '--no_log']
subprocess.check_call(cmd)
# Merge weight files -------------------------------------------------------------------------------------------
merged_weight_filename = self.get_temporary_file_path('merged_weights.nc')
gs.create_merged_weight_file(merged_weight_filename)
# Generate a global weight file using ESMF ---------------------------------------------------------------------
global_weights_filename = self.get_temporary_file_path('global_weights.nc')
cmd = ['ESMF_RegridWeightGen', '-s', path_src, '--src_type', 'GRIDSPEC', '-d', path_dst, '--dst_type',
'GRIDSPEC', '-w', global_weights_filename, '--method', 'conserve', '--weight-only', '--no_log']
subprocess.check_call(cmd)
# Test merged and global weight files are equivalent -----------------------------------------------------------
self.assertWeightFilesEquivalent(global_weights_filename, merged_weight_filename)
def test_get_wrapped_state(self):
if sys.version_info.major == 3 and sys.version_info.minor == 5:
raise SkipTest('undefined behavior with Python 3.5')
ompi = OcgDist()
ompi.create_dimension('x', 5, dist=True)
ompi.create_dimension('y', 1)
ompi.update_dimension_bounds()
values = [{'value': [-179, -90, 0, 90, 180], 'desired': WrappedState.WRAPPED},
{'value': [0, 90, 180, 270, 360], 'desired': WrappedState.UNWRAPPED},
{'value': [1, 2, 3, 4, 5], 'desired': WrappedState.UNKNOWN}]
kwds = {'values': values, 'crs': [Spherical(), None]}
for k in self.iter_product_keywords(kwds):
ompi = deepcopy(ompi)
if MPI_RANK == 0:
vx = Variable(name='x', value=k.values['value'], dimensions='x')
vy = Variable(name='y', value=[0], dimensions='y')
else:
vx, vy = [None] * 2
vx = variable_scatter(vx, ompi)
vy = variable_scatter(vy, ompi)
grid = Grid(vx, vy)
field = Field(grid=grid, crs=k.crs)
with vm.scoped_by_emptyable('wrap', field):
if not vm.is_null:
wrapped_state = field.wrapped_state
else:
wrapped_state = None
if not field.is_empty:
if k.crs is None:
self.assertIsNone(wrapped_state)
else:
self.assertIsNotNone(wrapped_state)
if k.crs is None or field.is_empty:
self.assertIsNone(wrapped_state)
else:
self.assertEqual(wrapped_state, k.values['desired'])
# Test with masked geometries.
values = [Point(350, 2), Point(-90, 5), Point(340, 5)]
mask = [True, False, True]
gvar = GeometryVariable(name='geom', value=values, mask=mask, dimensions='ngeom')
crs = Spherical()
wrapped_state = crs.get_wrapped_state(gvar)
self.assertEqual(wrapped_state, WrappedState.WRAPPED)
def test_chunked_rwg_spatial_subset(self):
env.CLOBBER_UNITS_ON_BOUNDS = False
src_grid = create_gridxy_global(crs=Spherical())
src_field = create_exact_field(src_grid, 'foo')
xvar = Variable(name='x', value=[-90., -80.], dimensions='xdim')
yvar = Variable(name='y', value=[40., 50.], dimensions='ydim')
dst_grid = Grid(x=xvar, y=yvar, crs=Spherical())
if ocgis.vm.rank == 0:
source = self.get_temporary_file_path('source.nc')
else:
source = None
source = ocgis.vm.bcast(source)
src_field.write(source)
if ocgis.vm.rank == 0:
destination = self.get_temporary_file_path('destination.nc')
else:
destination = None
destination = ocgis.vm.bcast(destination)
dst_grid.parent.write(destination)
wd = os.path.join(self.current_dir_output, 'chunks')
weight = os.path.join(self.current_dir_output, 'weights.nc')
spatial_subset = os.path.join(self.current_dir_output,
'spatial_subset.nc')
runner = CliRunner()
cli_args = [
'chunked-rwg', '--source', source, '--destination', destination,
'--wd', wd, '--spatial_subset', '--spatial_subset_path',
spatial_subset, '--weight', weight, '--esmf_regrid_method',
'BILINEAR', '--persist'
]
result = runner.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
actual = RequestDataset(uri=spatial_subset).create_field()
actual_ymean = actual.grid.get_value_stacked()[0].mean()
actual_xmean = actual.grid.get_value_stacked()[1].mean()
self.assertEqual(actual_ymean, 45.)
self.assertEqual(actual_xmean, -85.)
self.assertEqual(actual.grid.shape, (14, 14))
self.assertTrue(os.path.exists(weight))
actual = RequestDataset(weight, driver='netcdf').create_field()
self.assertIn('history', actual.attrs)
def _get_regrid_source_(self):
# Update the source coordinate system to spherical.
if self.field_src.crs != Spherical():
ocgis_lh(
logger='regrid',
msg=
'updating regrid source to spherical. regrid source crs is: {}'
.format(self.field_src.crs),
level=logging.DEBUG)
backtransform_crs = deepcopy(self.field_src.crs)
self.field_src.update_crs(Spherical())
else:
backtransform_crs = None
return self.field_src, backtransform_crs
def test_create_merged_weight_file_unstructured(self):
import ESMF
# Create an isomorphic source UGRID file.
ufile = self.get_temporary_file_path('ugrid.nc')
resolution = 10.
self.fixture_regular_ugrid_file(ufile, resolution, crs=Spherical())
src_grid = RequestDataset(ufile,
driver=DriverNetcdfUGRID,
grid_abstraction='point').get().grid
self.assertEqual(src_grid.abstraction, 'point')
# Create a logically rectangular destination grid file.
dst_grid = self.get_gridxy_global(resolution=20., crs=Spherical())
dst_path = self.get_temporary_file_path('dst.nc')
dst_grid.parent.write(dst_path)
# Create the grid chunks.
gs = GridChunker(src_grid,
dst_grid, (3, 3),
check_contains=False,
src_grid_resolution=10.,
paths=self.fixture_paths,
genweights=True)
gs.write_chunks()
# Merge weight files.
mwf = self.get_temporary_file_path('merged_weight_file.nc')
gs.create_merged_weight_file(mwf)
# Generate a global weight file using ESMF.
global_weights_filename = self.get_temporary_file_path(
'global_weights.nc')
srcgrid = ESMF.Mesh(filename=ufile,
filetype=ESMF.FileFormat.UGRID,
meshname=VariableName.UGRID_HOST_VARIABLE)
dstgrid = ESMF.Grid(filename=dst_path,
filetype=ESMF.FileFormat.GRIDSPEC,
add_corner_stagger=True)
srcfield = ESMF.Field(grid=srcgrid, meshloc=ESMF.MeshLoc.ELEMENT)
dstfield = ESMF.Field(grid=dstgrid)
_ = ESMF.Regrid(srcfield=srcfield,
dstfield=dstfield,
filename=global_weights_filename,
regrid_method=ESMF.RegridMethod.CONSERVE)
# Test merged and global weight files are equivalent.
self.assertWeightFilesEquivalent(global_weights_filename, mwf)
def test_get_ocgis_field_from_esmf_field(self):
from ocgis.regrid.base import get_esmf_field_from_ocgis_field
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
ogrid = create_gridxy_global(crs=Spherical())
ofield = create_exact_field(ogrid, 'foo', ntime=3)
ogrid = ofield.grid
ogrid_mask = ogrid.get_mask(create=True)
ogrid_mask[1, 0] = True
ogrid.set_mask(ogrid_mask)
efield = get_esmf_field_from_ocgis_field(ofield)
self.assertEqual(efield.data.shape, (360, 180, 3))
ofield_actual = get_ocgis_field_from_esmf_field(efield, field=ofield)
actual_dv_mask = ofield_actual.data_variables[0].get_mask()
self.assertTrue(np.all(actual_dv_mask[:, 1, 0]))
self.assertEqual(actual_dv_mask.sum(), 3)
self.assertNumpyAll(ofield.data_variables[0].get_value(),
ofield_actual.data_variables[0].get_value())
self.assertEqual(ofield.data_variables[0].name, efield.name)
self.assertNumpyAll(ofield.time.get_value(),
ofield_actual.time.get_value())
def run_system_splitting_unstructured(self, genweights):
env.CLOBBER_UNITS_ON_BOUNDS = False
ufile = self.get_temporary_file_path('ugrid.nc')
resolution = 10.
self.fixture_regular_ugrid_file(ufile, resolution)
src_rd = RequestDataset(ufile,
driver=DriverNetcdfUGRID,
grid_abstraction='point')
# src_rd.inspect()
src_grid = src_rd.get().grid
self.assertEqual(src_grid.abstraction, 'point')
dst_grid = self.get_gridxy_global(resolution=20., crs=Spherical())
gs = GridChunker(src_grid,
dst_grid, (3, 3),
check_contains=False,
src_grid_resolution=10.,
paths=self.fixture_paths,
genweights=genweights,
use_spatial_decomp=True)
gs.write_chunks()
actual = gs.create_full_path_from_template('src_template', index=1)
actual = RequestDataset(actual).get()
self.assertIn(GridChunkerConstants.IndexFile.NAME_SRCIDX_GUID, actual)
def test_system_subset(self):
"""Test data may be subsetted and that coordinate transformations return the same value arrays."""
ops = OcgOperations(dataset=self.rd, output_format=constants.OutputFormatName.OCGIS, snippet=True,
geom='world_countries', select_ugid=[69])
ret = ops.execute()
# Assert some of the geometry values are masked
actual = ret.get_element().grid.get_mask()
self.assertTrue(actual.any())
# Perform the operations but change the output coordinate system. The value arrays should be equivalent
# regardless of coordinate transformation.
ops2 = OcgOperations(dataset=self.rd, output_format=constants.OutputFormatName.OCGIS, snippet=True,
geom='world_countries', select_ugid=[69], output_crs=Spherical())
ret2 = ops2.execute()
# Value arrays should be the same
ret_value = ret.get_element(variable_name='pr').get_value()
ret2_value = ret2.get_element(variable_name='pr').get_value()
self.assertNumpyAll(ret_value, ret2_value)
# Grid coordinates should not be the same.
ret_grid_value = ret.get_element().grid.get_value_stacked()
ret2_grid_value = ret2.get_element().grid.get_value_stacked()
diff = np.abs(ret_grid_value - ret2_grid_value)
select = diff > 1
self.assertTrue(select.all())
def test_iter_spatial_decomposition(self):
self.add_barrier = False
if vm.size not in [1, 4]:
raise SkipTest('vm.size not in [1, 4]')
grid = create_gridxy_global(resolution=10.,
wrapped=False,
crs=Spherical())
splits = (2, 3)
actual = []
for sub, slc in iter_spatial_decomposition(grid,
splits,
optimized_bbox_subset=True):
root = vm.get_live_ranks_from_object(sub)[0]
with vm.scoped_by_emptyable('test extent', sub):
if vm.is_null:
extent_global = None
else:
extent_global = sub.extent_global
extent_global = vm.bcast(extent_global, root=root)
actual.append(extent_global)
desired = [(0.0, -90.0, 120.0, 0.0), (120.0, -90.0, 240.0, 0.0),
(240.0, -90.0, 360.0, 0.0), (0.0, 0.0, 120.0, 90.0),
(120.0, 0.0, 240.0, 90.0), (240.0, 0.0, 360.0, 90.0)]
self.assertEqual(actual, desired)
def test_system_process_geometries(self):
"""Test multiple geometries with coordinate system update."""
a = 'POLYGON((-105.21347987288135073 40.21514830508475313,-104.39928495762711691 40.21514830508475313,-104.3192002118643984 39.5677966101694949,-102.37047139830508513 39.61451271186440692,-102.12354343220337682 37.51896186440677639,-105.16009004237288593 37.51896186440677639,-105.21347987288135073 40.21514830508475313))'
b = 'POLYGON((-104.15235699152542281 39.02722457627118757,-103.71189088983049942 39.44099576271186436,-102.71750529661017026 39.28082627118644155,-102.35712394067796538 37.63908898305084705,-104.13900953389830306 37.63241525423728717,-104.15235699152542281 39.02722457627118757))'
geom = [{'geom': wkt.loads(xx), 'properties': {'UGID': ugid}} for ugid, xx in enumerate([a, b])]
grid_value = [
[[37.0, 37.0, 37.0, 37.0], [38.0, 38.0, 38.0, 38.0], [39.0, 39.0, 39.0, 39.0], [40.0, 40.0, 40.0, 40.0]],
[[-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0],
[-105.0, -104.0, -103.0, -102.0]]]
output_crs = CoordinateReferenceSystem(
value={'a': 6370997, 'lon_0': -100, 'y_0': 0, 'no_defs': True, 'proj': 'laea', 'x_0': 0, 'units': 'm',
'b': 6370997, 'lat_0': 45})
x = Variable('x', grid_value[1], dimensions=['lat', 'lon'])
y = Variable('y', grid_value[0], dimensions=['lat', 'lon'])
grid = Grid(x, y)
field = Field(grid=grid, crs=Spherical())
ops = OcgOperations(dataset=field, geom=geom, output_crs=output_crs)
ret = ops.execute()
expected = {0: -502052.79407259845,
1: -510391.37909706926}
for field, container in ret.iter_fields(yield_container=True):
self.assertAlmostEqual(field.grid.get_value_stacked().mean(),
expected[container.geom.ugid.get_value()[0]])
def test_narccap_point_subset_small(self):
dmap = {DimensionMapKey.X: {DimensionMapKey.VARIABLE: 'xc'},
DimensionMapKey.Y: {DimensionMapKey.VARIABLE: 'yc'},
DimensionMapKey.TIME: {DimensionMapKey.VARIABLE: 'time'}}
rd = self.test_data.get_rd('narccap_pr_wrfg_ncep', kwds={'dimension_map': dmap})
field = rd.get()
self.assertIsInstance(field.crs, CFLambertConformal)
self.assertIsNotNone(field.time)
geom = [-97.74278, 30.26694]
calc = [{'func': 'mean', 'name': 'mean'},
{'func': 'median', 'name': 'median'},
{'func': 'max', 'name': 'max'},
{'func': 'min', 'name': 'min'}]
calc_grouping = ['month', 'year']
ops = ocgis.OcgOperations(dataset=rd, calc=calc, calc_grouping=calc_grouping,
output_format=constants.OutputFormatName.OCGIS, geom=geom, abstraction='point',
snippet=False, allow_empty=False, output_crs=Spherical(),
search_radius_mult=2.0)
ret = ops.execute()
ref = ret.get_element()
actual = set(get_variable_names(ref.data_variables))
self.assertEqual(actual, {'mean', 'median', 'max', 'min'})
def test_regrid_field(self):
"""Test with equivalent input and output expectations. The shapes of the grids are equal."""
source = self.get_ofield()
source.set_crs(Spherical())
destination = deepcopy(source)
desired = deepcopy(source)
keywords = dict(split=[False, True])
for k in itr_products_keywords(keywords, as_namedtuple=True):
from ocgis.regrid.base import regrid_field
regridded = regrid_field(source, destination, split=k.split)
self.assertIsInstance(regridded, Field)
self.assertNumpyAll(regridded.grid.get_value_stacked(),
desired.grid.get_value_stacked())
self.assertEqual(regridded.crs, source.crs)
for variable in regridded.data_variables:
self.assertGreater(variable.get_value().mean(), 2.0)
self.assertNumpyAll(
variable.get_value(),
source[variable.name].get_value().squeeze())
self.assertFalse(
np.may_share_memory(variable.get_value(),
source[variable.name].get_value()))
def test_get_esmf_grid_with_mask(self):
"""Test with masked data."""
from ocgis.regrid.base import get_esmf_grid
x = Variable(name='x', value=[1, 2, 3], dimensions='x')
y = Variable(name='y', value=[4, 5, 6], dimensions='y')
grid = Grid(x, y, crs=Spherical())
gmask = grid.get_mask(create=True)
gmask[1, 1] = True
grid.set_mask(gmask)
self.assertEqual(grid.get_mask().sum(), 1)
egrid = get_esmf_grid(grid)
egrid_mask_inverted = np.invert(np.array(egrid.mask[0], dtype=bool))
self.assertNumpyAll(grid.get_mask(), egrid_mask_inverted)
# Test with a value mask.
value_mask = np.zeros(grid.shape, dtype=bool)
value_mask[-1, -1] = True
egrid = get_esmf_grid(grid, value_mask=value_mask)
egrid_mask_inverted = np.invert(np.array(egrid.mask[0], dtype=bool))
self.assertNumpyAll(egrid_mask_inverted,
np.logical_or(grid.get_mask(), value_mask))
def test_update_crs(self):
f = self.fixture(crs=Spherical())
to_crs = WGS84()
orig_diff = np.sum(np.diff(f.y.get_value()))
f.update_crs(to_crs)
actual_diff = np.sum(np.diff(f.y.get_value()))
self.assertGreater(np.abs(orig_diff - actual_diff), 0.01)
def test_set_geom(self):
f = Field()
self.assertIsNone(f.crs)
g = GeometryVariable(value=[Point(1, 2)], dimensions='geom', crs=Spherical())
f.set_geom(g)
def test_update_crs(self):
# Test copying allows the CRS to be updated on the copy w/out changing the source CRS.
desired = Spherical()
gvar = GeometryVariable(value=[Point(1, 2)], name='geom', dimensions='geom')
field = Field(crs=desired, geom=gvar)
cfield = field.copy()
self.assertEqual(cfield.crs, desired)
new_crs = CoordinateReferenceSystem(name='i_am_new', epsg=4326)
cfield.update_crs(new_crs)
self.assertEqual(field.crs, desired)
# Test geometry crs update is called.
mfield = Mock(Field)
mfield.is_empty = False
mfield.dimension_map = Mock(DimensionMap)
mfield.grid = Mock(Grid)
mfield.geom = Mock(GeometryVariable)
mcrs = Mock(Spherical)
Field.update_crs(mfield, mcrs)
mfield.grid.update_crs.assert_called_once_with(mcrs, from_crs=mfield.crs)
mfield.geom.update_crs.assert_called_once_with(mcrs, from_crs=mfield.crs)
from_crs = Mock(WGS84)
mfield.grid.update_crs.reset_mock()
mfield.geom.update_crs.reset_mock()
Field.update_crs(mfield, mcrs, from_crs=from_crs)
mfield.grid.update_crs.assert_called_once_with(mcrs, from_crs=from_crs)
mfield.geom.update_crs.assert_called_once_with(mcrs, from_crs=from_crs)
def test_place_prime_meridian_array(self):
arr = np.array([123, 180, 200, 180], dtype=float)
ret = Spherical._place_prime_meridian_array_(arr)
self.assertNumpyAll(ret, np.array([False, True, False, True]))
self.assertNumpyAll(
arr,
np.array([
123., constants.MERIDIAN_180TH, 200., constants.MERIDIAN_180TH
]))
def test_as_shapely(self):
coords = (12, 3, 15, 4)
bbox = box(*coords)
gvar = GeometryVariable(value=bbox,
is_bbox=True,
dimensions='geom',
crs=Spherical())
geom = gvar.as_shapely()
self.assertEqual(geom.bounds, coords)
def test_wrap(self):
geom = box(195, -40, 225, -30)
gvar = GeometryVariable(name='geoms',
value=geom,
crs=Spherical(),
dimensions='geoms')
gvar.wrap()
self.assertEqual(gvar.get_value()[0].bounds,
(-165.0, -40.0, -135.0, -30.0))
def create_regridding_field(grid, name_variable):
col_shape = grid.shape[1]
row_shape = grid.shape[0]
value = np.ones(col_shape * row_shape, dtype=float).reshape(
grid.shape) * 15.
variable = Variable(name=name_variable,
value=value,
dimensions=grid.dimensions)
field = Field(is_data=variable, grid=grid, crs=Spherical())
return field
def test_check_fields_for_regridding(self):
from ESMF import RegridMethod
from ocgis.regrid.base import check_fields_for_regridding
source = self.get_ofield()
destination = deepcopy(source)
# Test non-spherical coordinate systems.
new_source = deepcopy(source)
new_source.update_crs(WGS84())
with self.assertRaises(RegriddingError):
check_fields_for_regridding(new_source, destination)
# Change coordinate systems to spherical.
source.update_crs(Spherical())
destination.update_crs(Spherical())
# Test with different parameters of the sphere.
new_source = deepcopy(source)
new_source.update_crs(Spherical(semi_major_axis=100))
with self.assertRaises(RegriddingError):
check_fields_for_regridding(new_source, destination)
# Test corners not available on all inputs #####################################################################
# Test corners not on one of the sources
new_source = deepcopy(source)
new_source.grid.x.set_bounds(None)
new_source.grid.y.set_bounds(None)
self.assertFalse(new_source.grid.has_bounds)
for regrid_method in ['auto', RegridMethod.CONSERVE]:
if regrid_method == RegridMethod.CONSERVE:
with self.assertRaises(CornersInconsistentError):
check_fields_for_regridding(new_source,
destination,
regrid_method=regrid_method)
else:
check_fields_for_regridding(new_source,
destination,
regrid_method=regrid_method)
def test_place_prime_meridian_array(self):
arr = np.array([123, 180, 200, 180], dtype=float)
ret = Spherical._place_prime_meridian_array_(arr)
self.assertNumpyAll(ret, np.array([False, True, False, True]))
self.assertNumpyAll(arr, np.array([123., constants.MERIDIAN_180TH, 200., constants.MERIDIAN_180TH]))
