def _get_regrid_destination_(self):
"""
Prepare destination field for regridding.
:rtype: (:class:`~ocgis.Field`, :class:`~ocgis.CoordinateReferenceSystem` or ``None``)
"""
# Transform the coordinate system of the regrid destination. ###################################################
# Update the regrid destination coordinate system must be updated to match the source.
if self.field_dst.crs != Spherical():
ocgis_lh(logger='regrid',
msg='updating regrid destination to spherical. regrid destination crs is: {}'.format(
self.field_dst.crs), level=logging.DEBUG)
backtransform_crs = deepcopy(self.field_dst.crs)
self.field_dst.update_crs(Spherical())
else:
backtransform_crs = None
# Spatially subset the regrid destination. #####################################################################
if self.subset_field is None:
ocgis_lh(logger='regrid', msg='no spatial subsetting', level=logging.DEBUG)
regrid_destination = self.field_dst
else:
ss = SpatialSubsetOperation(self.field_dst)
regrid_destination = ss.get_spatial_subset('intersects', self.subset_field.geom,
use_spatial_index=env.USE_SPATIAL_INDEX,
select_nearest=False)
return regrid_destination, backtransform_crs
def _write_spatial_subset_(rd_src, rd_dst, spatial_subset_path, src_resmax=None):
src_field = rd_src.create_field()
dst_field = rd_dst.create_field()
sso = SpatialSubsetOperation(src_field)
with grid_abstraction_scope(dst_field.grid, Topology.POLYGON):
dst_field_extent = dst_field.grid.extent_global
subset_geom = GeometryVariable.from_shapely(box(*dst_field_extent), crs=dst_field.crs, is_bbox=True)
if src_resmax is None:
src_resmax = src_field.grid.resolution_max
buffer_value = GridChunkerConstants.BUFFER_RESOLUTION_MODIFIER * src_resmax
sub_src = sso.get_spatial_subset('intersects', subset_geom, buffer_value=buffer_value, optimized_bbox_subset=True)
# Try to reduce the coordinate indexing for unstructured grids.
with ocgis.vm.scoped_by_emptyable('subset reduce/write', sub_src):
if not ocgis.vm.is_null:
# Attempt to reindex the subset.
try:
reduced = sub_src.grid.reduce_global()
except AttributeError:
pass
else:
sub_src = reduced.parent
# Write the subset to file.
sub_src.write(spatial_subset_path)
def test_prepare_geometry(self):
for geometry_record in self.get_subset_geometries():
for ss, k in self:
gvar = GeometryVariable(value=geometry_record['geom'],
dimensions='dim',
crs=WGS84())
self.assertIsNotNone(gvar.crs)
prepared = ss._prepare_geometry_(gvar)
self.assertNotEqual(gvar.get_value()[0].bounds,
prepared.get_value()[0].bounds)
self.assertFalse(
np.may_share_memory(gvar.get_value(),
prepared.get_value()))
try:
self.assertEqual(prepared.crs, ss.field.crs)
except AssertionError:
# Rotated pole on the fields become spherical.
self.assertEqual(prepared.crs, CFSpherical())
self.assertIsInstance(ss.field.crs, CFRotatedPole)
# Test nebraska against an unwrapped dataset.
nebraska = GeometryVariable(value=self.nebraska['geom'],
dimensions='d',
crs=WGS84())
field = self.test_data.get_rd('cancm4_tas').get()
ss = SpatialSubsetOperation(field)
prepared = ss._prepare_geometry_(nebraska)
self.assertEqual(prepared.wrapped_state, WrappedState.UNWRAPPED)
def test_get_spatial_subset_rotated_pole(self):
"""Test input has rotated pole with now output CRS."""
rd = self.rd_rotated_pole
ss = SpatialSubsetOperation(rd.get())
ret = ss.get_spatial_subset('intersects', self.germany['geom'], geom_crs=WGS84())
self.assertEqual(ret.crs, rd.get().crs)
self.assertAlmostEqual(ret.grid.get_value_stacked().mean(), -2.1699999954751132)
def test_get_spatial_subset_circular_geometries(self):
"""Test circular geometries. They were causing wrapping errors."""
geoms = TestGeom.get_geometry_dictionaries()
rd = self.test_data.get_rd('cancm4_tas')
buffered = [element['geom'].buffer(rd.get().grid.resolution * 2) for element in geoms]
for buff in buffered:
ss = SpatialSubsetOperation(rd.get(), wrap=False)
gvar = GeometryVariable(value=buff, name='geom', dimensions='dim', crs=WGS84())
ret = ss.get_spatial_subset('intersects', gvar)
self.assertTrue(np.all(ret.grid.x.get_value() >= 0))
def test_get_spatial_subset_rotated_pole(self):
"""Test a spatial subset with rotated pole data."""
rd = self.fixture_rd_rotated_pole(
**{KeywordArgument.ROTATED_POLE_PRIORITY: True})
ss = SpatialSubsetOperation(rd.get())
ret = ss.get_spatial_subset('intersects',
self.germany['geom'],
geom_crs=WGS84())
self.assertEqual(ret.crs, rd.get().crs)
self.assertAlmostEqual(ret.grid.get_value_stacked().mean(),
-2.1699999954751132)
def test_should_update_crs(self):
# no output crs provided
target = self.test_data.get_rd('cancm4_tas').get()
ss = SpatialSubsetOperation(target)
self.assertFalse(ss.should_update_crs)
# output crs different than input
ss = SpatialSubsetOperation(target, output_crs=CoordinateReferenceSystem(epsg=2136))
self.assertTrue(ss.should_update_crs)
# same output crs as input
ss = SpatialSubsetOperation(target, output_crs=ss.field.crs)
self.assertFalse(ss.should_update_crs)
def _get_spatially_subsetted_field_(self, alias, field, subset_field,
subset_ugid):
"""
Spatially subset a field with a selection field.
:param str alias: The request data alias currently being processed.
:param field: Target field to subset.
:type field: :class:`ocgis.Field`
:param subset_field: The field to use for subsetting.
:type subset_field: :class:`ocgis.Field`
:rtype: :class:`ocgis.Field`
:raises: AssertionError, ExtentError
"""
assert subset_field is not None
ocgis_lh('executing spatial subset operation',
self._subset_log,
level=logging.DEBUG,
alias=alias,
ugid=subset_ugid)
sso = SpatialSubsetOperation(field)
try:
# Execute the spatial subset and return the subsetted field.
sfield = sso.get_spatial_subset(
self.ops.spatial_operation,
subset_field.geom,
select_nearest=self.ops.select_nearest,
optimized_bbox_subset=self.ops.optimized_bbox_subset)
except EmptySubsetError as e:
if self.ops.allow_empty:
ocgis_lh(
alias=alias,
ugid=subset_ugid,
msg='Empty geometric operation but empty returns allowed.',
level=logging.WARN)
sfield = Field(name=field.name, is_empty=True)
else:
msg = ' This typically means the selection geometry falls outside the spatial domain of the target ' \
'dataset.'
msg = str(e) + msg
ocgis_lh(exc=ExtentError(message=msg),
alias=alias,
logger=self._subset_log)
# If the subset geometry is unwrapped and the vector wrap option is true, wrap the subset geometry.
if self.ops.vector_wrap:
if subset_field.wrapped_state == WrappedState.UNWRAPPED:
subset_field.wrap()
return sfield
def test_get_spatial_subset_rotated_pole(self):
"""Test a spatial subset with rotated pole data."""
try:
rd = self.fixture_rd_rotated_pole(**{KeywordArgument.ROTATED_POLE_PRIORITY: True})
ss = SpatialSubsetOperation(rd.get())
ret = ss.get_spatial_subset('intersects', self.germany['geom'], geom_crs=WGS84())
self.assertEqual(ret.crs, rd.get().crs)
self.assertAlmostEqual(ret.grid.get_value_stacked().mean(), -2.1699999954751132)
except RuntimeError as e:
if "HDF error" in str(e):
raise SkipTest('HDF sometimes has trouble reading the dataset')
else:
raise
def test_get_buffered_geometry(self):
proj4 = '+proj=aea +lat_1=20 +lat_2=60 +lat_0=40 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +units=m +no_defs'
buffer_crs_list = [None, CoordinateReferenceSystem(proj4=proj4)]
poly = make_poly((36, 44), (-104, -95))
for buffer_crs in buffer_crs_list:
gvar = GeometryVariable(value=poly, name='geoms', dimensions='dim', crs=WGS84())
self.assertEqual(gvar.crs, WGS84())
if buffer_crs is None:
buffer_value = 1
else:
buffer_value = 10
ret = SpatialSubsetOperation._get_buffered_geometry_(gvar, buffer_value, buffer_crs=buffer_crs)
ref = ret.get_value()[0]
if buffer_crs is None:
self.assertEqual(ref.bounds, (-105.0, 35.0, -94.0, 45.0))
else:
self.assertNumpyAllClose(np.array(ref.bounds), np.array(
(-104.00013263459613, 35.9999147913708, -94.99986736540386, 44.00008450528758)))
self.assertEqual(gvar.crs, ret.crs)
# check deepcopy
ret.get_value()[0] = make_poly((1, 2), (3, 4))
ref_buffered = ret.get_value()[0]
ref_original = gvar.get_value()[0]
with self.assertRaises(AssertionError):
self.assertNumpyAllClose(np.array(ref_buffered.bounds), np.array(ref_original.bounds))
def __iter__(self):
keywords = dict(target=self.target,
output_crs=self.get_output_crs(),
wrap=[None, True, False])
for k in itr_products_keywords(keywords, as_namedtuple=True):
kwargs = k._asdict()
target = kwargs.pop('target')
ss = SpatialSubsetOperation(list(target.values())[0], **kwargs)
yield (ss, k)
def _write_spatial_subset_(rd_src, rd_dst, spatial_subset_path):
src_field = rd_src.create_field()
dst_field = rd_dst.create_field()
sso = SpatialSubsetOperation(src_field)
with grid_abstraction_scope(dst_field.grid, Topology.POLYGON):
dst_field_extent = dst_field.grid.extent_global
subset_geom = GeometryVariable.from_shapely(box(*dst_field_extent), crs=dst_field.crs, is_bbox=True)
buffer_value = GridChunkerConstants.BUFFER_RESOLUTION_MODIFIER * src_field.grid.resolution_max
sub_src = sso.get_spatial_subset('intersects', subset_geom, buffer_value=buffer_value, optimized_bbox_subset=True)
# Try to reduce the coordinate indexing for unstructured grids.
try:
reduced = sub_src.grid.reduce_global()
except AttributeError:
pass
else:
sub_src = reduced.parent
sub_src.write(spatial_subset_path)
def _write_spatial_subset_(rd_src,
rd_dst,
spatial_subset_path,
src_resmax=None):
src_field = rd_src.create_field()
dst_field = rd_dst.create_field()
sso = SpatialSubsetOperation(src_field)
with grid_abstraction_scope(dst_field.grid, Topology.POLYGON):
dst_field_extent = dst_field.grid.extent_global
subset_geom = GeometryVariable.from_shapely(box(*dst_field_extent),
crs=dst_field.crs,
is_bbox=True)
if src_resmax is None:
src_resmax = src_field.grid.resolution_max
buffer_value = GridChunkerConstants.BUFFER_RESOLUTION_MODIFIER * src_resmax
sub_src = sso.get_spatial_subset('intersects',
subset_geom,
buffer_value=buffer_value,
optimized_bbox_subset=True)
# No empty spatial subsets allowed through CLI. There will be nothing for ESMF to do.
raise_if_empty(sub_src, check_current=True)
# Try to reduce the coordinate indexing for unstructured grids.
with ocgis.vm.scoped_by_emptyable('subset reduce/write', sub_src):
if not ocgis.vm.is_null:
# Attempt to reindex the subset.
try:
reduced = sub_src.grid.reduce_global()
except AttributeError:
pass
except ValueError:
if sub_src.driver.__class__ == DriverNetcdfUGRID:
raise
else:
sub_src = reduced.parent
# Write the subset to file.
sub_src.write(spatial_subset_path)
def _get_spatially_subsetted_field_(self, alias, field, subset_field, subset_ugid):
"""
Spatially subset a field with a selection field.
:param str alias: The request data alias currently being processed.
:param field: Target field to subset.
:type field: :class:`ocgis.Field`
:param subset_field: The field to use for subsetting.
:type subset_field: :class:`ocgis.Field`
:rtype: :class:`ocgis.Field`
:raises: AssertionError, ExtentError
"""
assert subset_field is not None
ocgis_lh('executing spatial subset operation', self._subset_log, level=logging.DEBUG, alias=alias,
ugid=subset_ugid)
sso = SpatialSubsetOperation(field)
try:
# Execute the spatial subset and return the subsetted field.
sfield = sso.get_spatial_subset(self.ops.spatial_operation, subset_field.geom,
select_nearest=self.ops.select_nearest,
optimized_bbox_subset=self.ops.optimized_bbox_subset)
except EmptySubsetError as e:
if self.ops.allow_empty:
ocgis_lh(alias=alias, ugid=subset_ugid, msg='Empty geometric operation but empty returns allowed.',
level=logging.WARN)
sfield = Field(name=field.name, is_empty=True)
else:
msg = ' This typically means the selection geometry falls outside the spatial domain of the target ' \
'dataset.'
msg = str(e) + msg
ocgis_lh(exc=ExtentError(message=msg), alias=alias, logger=self._subset_log)
# If the subset geometry is unwrapped and the vector wrap option is true, wrap the subset geometry.
if self.ops.vector_wrap:
if subset_field.wrapped_state == WrappedState.UNWRAPPED:
subset_field.wrap()
return sfield
def test_system_spherical_unwrapped_grid_negative_values(self):
"""Test with negative values in an unwrapped spherical dataset."""
xc = np.arange(-10, 350, step=10, dtype=float)
yc = np.arange(-90, 100, step=10, dtype=float)
xv = Variable("lon", xc, dimensions=["lon"])
yv = Variable("lat", yc, dimensions=["lat"])
grid = Grid(x=xv, y=yv, crs=Spherical())
self.assertEqual(grid.wrapped_state, WrappedState.UNWRAPPED)
self.assertEqual(grid._wrapped_state, "auto")
subbox = box(240, -10, 360, 10)
subbox = GeometryVariable(name='subset',
value=subbox,
is_bbox=True,
dimensions="ngeom",
crs=Spherical())
so = SpatialSubsetOperation(grid.parent)
subfield = so.get_spatial_subset("intersects", subbox)
self.assertTrue(grid.is_vectorized)
self.assertFalse(subfield.grid.get_mask()[0, 0])
self.assertEqual(grid.x.v()[0], -10.0)
self.assertIsNone(so._transformed_unwrapped_select)
def _get_regrid_destination_(self):
"""
Prepare destination field for regridding.
:rtype: (:class:`~ocgis.Field`, :class:`~ocgis.CoordinateReferenceSystem` or ``None``)
"""
# Transform the coordinate system of the regrid destination. ###################################################
# Update the regrid destination coordinate system must be updated to match the source.
if self.field_dst.crs != Spherical():
ocgis_lh(
logger='regrid',
msg=
'updating regrid destination to spherical. regrid destination crs is: {}'
.format(self.field_dst.crs),
level=logging.DEBUG)
backtransform_crs = deepcopy(self.field_dst.crs)
self.field_dst.update_crs(Spherical())
else:
backtransform_crs = None
# Spatially subset the regrid destination. #####################################################################
if self.subset_field is None:
ocgis_lh(logger='regrid',
msg='no spatial subsetting',
level=logging.DEBUG)
regrid_destination = self.field_dst
else:
ss = SpatialSubsetOperation(self.field_dst)
regrid_destination = ss.get_spatial_subset(
'intersects',
self.subset_field.geom,
use_spatial_index=env.USE_SPATIAL_INDEX,
select_nearest=False)
return regrid_destination, backtransform_crs
def test_get_spatial_subset_wrap(self):
"""Test subsetting with wrap set to a boolean value."""
rd = self.test_data.get_rd('cancm4_tas')
self.assertEqual(rd.get().wrapped_state, WrappedState.UNWRAPPED)
ss = SpatialSubsetOperation(rd.get(), wrap=True)
ret = ss.get_spatial_subset('intersects', self.nebraska['geom'], geom_crs=WGS84())
self.assertEqual(ret.wrapped_state, WrappedState.WRAPPED)
self.assertAlmostEqual(ret.grid.get_value_stacked()[1].mean(), -99.84375)
# Test with wrap false.
ss = SpatialSubsetOperation(rd.get(), wrap=False)
ret = ss.get_spatial_subset('intersects', self.nebraska['geom'], geom_crs=WGS84())
self.assertEqual(ret.wrapped_state, WrappedState.UNWRAPPED)
self.assertAlmostEqual(ret.grid.get_value_stacked()[1].mean(), 260.15625)
def test_get_spatial_subset_output_crs(self):
"""Test subsetting with an output CRS."""
# test with default crs converting to north american lambert
proj4 = '+proj=aea +lat_1=20 +lat_2=60 +lat_0=40 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +units=m +no_defs'
output_crs = CoordinateReferenceSystem(proj4=proj4)
rd = self.test_data.get_rd('cancm4_tas')
ss = SpatialSubsetOperation(rd.get(), output_crs=output_crs)
ret = ss.get_spatial_subset('intersects', self.nebraska['geom'], geom_crs=WGS84())
self.assertEqual(ret.crs, output_crs)
self.assertAlmostEqual(ret.grid.get_value_stacked().mean(), -23341.955070198124)
# test with an input rotated pole coordinate system
rd = self.rd_rotated_pole
ss = SpatialSubsetOperation(rd.get(), output_crs=env.DEFAULT_COORDSYS)
ret = ss.get_spatial_subset('intersects', self.germany['geom'], geom_crs=WGS84())
self.assertEqual(ret.crs, env.DEFAULT_COORDSYS)
def test_get_should_wrap(self):
# A 360 dataset.
field_360 = self.test_data.get_rd('cancm4_tas').get()
ss = SpatialSubsetOperation(field_360, wrap=True)
self.assertTrue(ss._get_should_wrap_(ss.field))
ss = SpatialSubsetOperation(field_360, wrap=False)
self.assertFalse(ss._get_should_wrap_(ss.field))
ss = SpatialSubsetOperation(field_360, wrap=None)
self.assertFalse(ss._get_should_wrap_(ss.field))
# A wrapped dataset.
field_360.wrap()
ss = SpatialSubsetOperation(field_360, wrap=True)
self.assertFalse(ss._get_should_wrap_(ss.field))
ss = SpatialSubsetOperation(field_360, wrap=False)
self.assertFalse(ss._get_should_wrap_(ss.field))
ss = SpatialSubsetOperation(field_360, wrap=None)
self.assertFalse(ss._get_should_wrap_(ss.field))
