def _write_spatial_subset_(rd_src, rd_dst, spatial_subset_path, src_resmax=None):
src_field = rd_src.create_field()
dst_field = rd_dst.create_field()
sso = SpatialSubsetOperation(src_field)
with grid_abstraction_scope(dst_field.grid, Topology.POLYGON):
dst_field_extent = dst_field.grid.extent_global
subset_geom = GeometryVariable.from_shapely(box(*dst_field_extent), crs=dst_field.crs, is_bbox=True)
if src_resmax is None:
src_resmax = src_field.grid.resolution_max
buffer_value = GridChunkerConstants.BUFFER_RESOLUTION_MODIFIER * src_resmax
sub_src = sso.get_spatial_subset('intersects', subset_geom, buffer_value=buffer_value, optimized_bbox_subset=True)
# Try to reduce the coordinate indexing for unstructured grids.
with ocgis.vm.scoped_by_emptyable('subset reduce/write', sub_src):
if not ocgis.vm.is_null:
# Attempt to reindex the subset.
try:
reduced = sub_src.grid.reduce_global()
except AttributeError:
pass
else:
sub_src = reduced.parent
# Write the subset to file.
sub_src.write(spatial_subset_path)
def _write_spatial_subset_(rd_src, rd_dst, spatial_subset_path):
src_field = rd_src.create_field()
dst_field = rd_dst.create_field()
sso = SpatialSubsetOperation(src_field)
with grid_abstraction_scope(dst_field.grid, Topology.POLYGON):
dst_field_extent = dst_field.grid.extent_global
subset_geom = GeometryVariable.from_shapely(box(*dst_field_extent), crs=dst_field.crs, is_bbox=True)
buffer_value = GridChunkerConstants.BUFFER_RESOLUTION_MODIFIER * src_field.grid.resolution_max
sub_src = sso.get_spatial_subset('intersects', subset_geom, buffer_value=buffer_value, optimized_bbox_subset=True)
# Try to reduce the coordinate indexing for unstructured grids.
try:
reduced = sub_src.grid.reduce_global()
except AttributeError:
pass
else:
sub_src = reduced.parent
sub_src.write(spatial_subset_path)
def _write_spatial_subset_(rd_src,
rd_dst,
spatial_subset_path,
src_resmax=None):
src_field = rd_src.create_field()
dst_field = rd_dst.create_field()
sso = SpatialSubsetOperation(src_field)
with grid_abstraction_scope(dst_field.grid, Topology.POLYGON):
dst_field_extent = dst_field.grid.extent_global
subset_geom = GeometryVariable.from_shapely(box(*dst_field_extent),
crs=dst_field.crs,
is_bbox=True)
if src_resmax is None:
src_resmax = src_field.grid.resolution_max
buffer_value = GridChunkerConstants.BUFFER_RESOLUTION_MODIFIER * src_resmax
sub_src = sso.get_spatial_subset('intersects',
subset_geom,
buffer_value=buffer_value,
optimized_bbox_subset=True)
# No empty spatial subsets allowed through CLI. There will be nothing for ESMF to do.
raise_if_empty(sub_src, check_current=True)
# Try to reduce the coordinate indexing for unstructured grids.
with ocgis.vm.scoped_by_emptyable('subset reduce/write', sub_src):
if not ocgis.vm.is_null:
# Attempt to reindex the subset.
try:
reduced = sub_src.grid.reduce_global()
except AttributeError:
pass
except ValueError:
if sub_src.driver.__class__ == DriverNetcdfUGRID:
raise
else:
sub_src = reduced.parent
# Write the subset to file.
sub_src.write(spatial_subset_path)
def iter_src_grid_subsets(self, yield_dst=False, yield_idx=None):
"""
Yield source grid subset using the extent of its associated destination grid subset.
:param bool yield_dst: If ``True``, yield the destination subset as well as the source grid subset.
:param int yield_idx: If a zero-based integer, only yield for this chunk index and skip everything else.
:rtype: tuple(:class:`ocgis.spatial.grid.AbstractGrid`, `slice-like`)
"""
if yield_dst:
yield_slice = True
else:
yield_slice = False
buffer_value = self.buffer_value
dst_grid_wrapped_state = self.dst_grid.wrapped_state
dst_grid_crs = self.dst_grid.crs
# Use a destination grid iterator if provided.
if self.iter_dst is not None:
iter_dst = self.iter_dst(self,
yield_slice=yield_slice,
yield_idx=yield_idx)
else:
iter_dst = self.iter_dst_grid_subsets(yield_slice=yield_slice,
yield_idx=yield_idx)
# Loop over each destination grid subset.
ocgis_lh(logger=_LOCAL_LOGGER,
msg='starting "for yld in iter_dst"',
level=logging.DEBUG)
for iter_dst_ctr, yld in enumerate(iter_dst, start=1):
ocgis_lh(msg=["iter_dst_ctr", iter_dst_ctr], level=logging.DEBUG)
if yield_slice:
dst_grid_subset, dst_slice = yld
else:
dst_grid_subset = yld
# All masked destinations are very problematic for ESMF
with vm.scoped_by_emptyable('global mask', dst_grid_subset):
if not vm.is_null:
if dst_grid_subset.has_mask_global:
if dst_grid_subset.has_mask and dst_grid_subset.has_masked_values:
all_masked = dst_grid_subset.get_mask().all()
else:
all_masked = False
all_masked_gather = vm.gather(all_masked)
if vm.rank == 0:
if all(all_masked_gather):
exc = ValueError(
"Destination subset all masked")
try:
raise exc
finally:
vm.abort(exc=exc)
dst_box = None
with vm.scoped_by_emptyable('extent_global', dst_grid_subset):
if not vm.is_null:
# Use the extent of the polygon for determining the bounding box. This ensures conservative
# regridding will be fully mapped.
if isinstance(dst_grid_subset,
AbstractGeometryCoordinates):
target_grid = dst_grid_subset.parent.grid
else:
target_grid = dst_grid_subset
# Try to reduce the coordinates in the case of unstructured grid data. Ensure the data also has a
# coordinate index. SCRIP grid files, for example, do not have a coordinate index like UGRID.
if hasattr(
target_grid, 'reduce_global'
) and Topology.POLYGON in target_grid.abstractions_available and target_grid.cindex is not None:
ocgis_lh(
logger=_LOCAL_LOGGER,
msg='starting reduce_global for dst_grid_subset',
level=logging.DEBUG)
target_grid = target_grid.reduce_global()
ocgis_lh(
logger=_LOCAL_LOGGER,
msg='finished reduce_global for dst_grid_subset',
level=logging.DEBUG)
extent_global = target_grid.parent.attrs.get(
'extent_global')
if extent_global is None:
with grid_abstraction_scope(target_grid,
Topology.POLYGON):
extent_global = target_grid.extent_global
# HACK: Bad corner coordinates can lead to bad extents. In this case, the lower bound on the
# x-coordinate is unreasonable and breaks wrapping code. Set to 0.0 which is a reasonable
# lower x-coordate for unwrapped datasets.
if (isinstance(target_grid.crs, Spherical)) and \
dst_grid_wrapped_state == WrappedState.UNWRAPPED and \
extent_global[0] < 0.0:
e = list(extent_global)
e[0] = 0.0
extent_global = tuple(e)
if self.check_contains:
dst_box = box(*target_grid.extent_global)
sub_box = box(*extent_global)
if buffer_value is not None:
# Use the envelope! A buffer returns "fancy" borders. We just want to expand the bounding box.
sub_box = sub_box.buffer(buffer_value).envelope
ocgis_lh(msg=str(sub_box.bounds),
level=logging.DEBUG,
logger=_LOCAL_LOGGER)
else:
sub_box, dst_box = [None, None]
live_ranks = vm.get_live_ranks_from_object(dst_grid_subset)
sub_box = vm.bcast(sub_box, root=live_ranks[0])
if self.check_contains:
dst_box = vm.bcast(dst_box, root=live_ranks[0])
sub_box = GeometryVariable.from_shapely(
sub_box,
is_bbox=True,
wrapped_state=dst_grid_wrapped_state,
crs=dst_grid_crs)
# Prepare geometry to match coordinate system and wrapping of the subset target
sub_box = sub_box.prepare(archetype=self.src_grid)
ocgis_lh(logger=_LOCAL_LOGGER,
msg='prepared geometry',
level=logging.DEBUG)
ocgis_lh(logger=_LOCAL_LOGGER,
msg='starting "self.src_grid.get_intersects"',
level=logging.DEBUG)
src_grid_subset, src_grid_slice = self.src_grid.get_intersects(
sub_box,
keep_touches=False,
cascade=False,
optimized_bbox_subset=self.optimized_bbox_subset,
return_slice=True)
ocgis_lh(logger=_LOCAL_LOGGER,
msg='finished "self.src_grid.get_intersects"',
level=logging.DEBUG)
# Reload the data using a new source index distribution.
if hasattr(src_grid_subset,
'reduce_global') and src_grid_subset.cindex is not None:
# Only redistribute if we have one live rank.
if self.redistribute and len(
vm.get_live_ranks_from_object(src_grid_subset)) > 0:
ocgis_lh(logger=_LOCAL_LOGGER,
msg='starting redistribute',
level=logging.DEBUG)
topology = src_grid_subset.abstractions_available[
Topology.POLYGON]
cindex = topology.cindex
redist_dimname = self.src_grid.abstractions_available[
Topology.POLYGON].element_dim.name
if src_grid_subset.is_empty:
redist_dim = None
else:
redist_dim = topology.element_dim
redistribute_by_src_idx(cindex, redist_dimname, redist_dim)
ocgis_lh(logger=_LOCAL_LOGGER,
msg='finished redistribute',
level=logging.DEBUG)
with vm.scoped_by_emptyable('src_grid_subset', src_grid_subset):
if not vm.is_null:
if not self.allow_masked:
gmask = src_grid_subset.get_mask()
if gmask is not None and gmask.any():
raise ValueError(
'Masked values in source grid subset.')
if self.check_contains:
src_box = box(*src_grid_subset.extent_global)
if not does_contain(src_box, dst_box):
raise ValueError('Contains check failed.')
# Try to reduce the coordinates in the case of unstructured grid data.
if hasattr(src_grid_subset, 'reduce_global'
) and src_grid_subset.cindex is not None:
ocgis_lh(logger=_LOCAL_LOGGER,
msg='starting reduce_global',
level=logging.DEBUG)
src_grid_subset = src_grid_subset.reduce_global()
ocgis_lh(logger=_LOCAL_LOGGER,
msg='finished reduce_global',
level=logging.DEBUG)
else:
pass
# src_grid_subset = VariableCollection(is_empty=True)
if src_grid_subset.is_empty:
src_grid_slice = None
else:
src_grid_slice = {
src_grid_subset.dimensions[ii].name: src_grid_slice[ii]
for ii in range(src_grid_subset.ndim)
}
if yield_dst:
yld = (src_grid_subset, src_grid_slice, dst_grid_subset,
dst_slice)
else:
yld = src_grid_subset, src_grid_slice
yield yld
def iter_src_grid_subsets(self, yield_dst=False, yield_idx=None):
"""
Yield source grid subset using the extent of its associated destination grid subset.
:param bool yield_dst: If ``True``, yield the destination subset as well as the source grid subset.
:param int yield_idx: If a zero-based integer, only yield for this chunk index and skip everything else.
:rtype: tuple(:class:`ocgis.spatial.grid.AbstractGrid`, `slice-like`)
"""
if yield_dst:
yield_slice = True
else:
yield_slice = False
buffer_value = self.buffer_value
dst_grid_wrapped_state = self.dst_grid.wrapped_state
dst_grid_crs = self.dst_grid.crs
# Use a destination grid iterator if provided.
if self.iter_dst is not None:
iter_dst = self.iter_dst(self,
yield_slice=yield_slice,
yield_idx=yield_idx)
else:
iter_dst = self.iter_dst_grid_subsets(yield_slice=yield_slice,
yield_idx=yield_idx)
# Loop over each destination grid subset.
ocgis_lh(logger='grid_chunker',
msg='starting "for yld in iter_dst"',
level=logging.DEBUG)
for yld in iter_dst:
if yield_slice:
dst_grid_subset, dst_slice = yld
else:
dst_grid_subset = yld
dst_box = None
with vm.scoped_by_emptyable('extent_global', dst_grid_subset):
if not vm.is_null:
# Use the extent of the polygon for determining the bounding box. This ensures conservative
# regridding will be fully mapped.
if isinstance(dst_grid_subset,
AbstractGeometryCoordinates):
target_grid = dst_grid_subset.parent.grid
else:
target_grid = dst_grid_subset
extent_global = target_grid.parent.attrs.get(
'extent_global')
if extent_global is None:
with grid_abstraction_scope(target_grid,
Topology.POLYGON):
extent_global = target_grid.extent_global
if self.check_contains:
dst_box = box(*target_grid.extent_global)
sub_box = box(*extent_global)
if buffer_value is not None:
# Use the envelope! A buffer returns "fancy" borders. We just want to expand the bounding box.
sub_box = sub_box.buffer(buffer_value).envelope
ocgis_lh(msg=str(sub_box.bounds),
level=logging.DEBUG,
logger='grid_chunker')
else:
sub_box, dst_box = [None, None]
live_ranks = vm.get_live_ranks_from_object(dst_grid_subset)
sub_box = vm.bcast(sub_box, root=live_ranks[0])
if self.check_contains:
dst_box = vm.bcast(dst_box, root=live_ranks[0])
sub_box = GeometryVariable.from_shapely(
sub_box,
is_bbox=True,
wrapped_state=dst_grid_wrapped_state,
crs=dst_grid_crs)
ocgis_lh(logger='grid_chunker',
msg='starting "self.src_grid.get_intersects"',
level=logging.DEBUG)
src_grid_subset, src_grid_slice = self.src_grid.get_intersects(
sub_box,
keep_touches=False,
cascade=False,
optimized_bbox_subset=self.optimized_bbox_subset,
return_slice=True)
ocgis_lh(logger='grid_chunker',
msg='finished "self.src_grid.get_intersects"',
level=logging.DEBUG)
# Reload the data using a new source index distribution.
if hasattr(src_grid_subset,
'reduce_global') and src_grid_subset.cindex is not None:
# Only redistribute if we have one live rank.
if self.redistribute and len(
vm.get_live_ranks_from_object(src_grid_subset)) > 0:
ocgis_lh(logger='grid_chunker',
msg='starting redistribute',
level=logging.DEBUG)
topology = src_grid_subset.abstractions_available[
Topology.POLYGON]
cindex = topology.cindex
redist_dimname = self.src_grid.abstractions_available[
Topology.POLYGON].element_dim.name
if src_grid_subset.is_empty:
redist_dim = None
else:
redist_dim = topology.element_dim
redistribute_by_src_idx(cindex, redist_dimname, redist_dim)
ocgis_lh(logger='grid_chunker',
msg='finished redistribute',
level=logging.DEBUG)
with vm.scoped_by_emptyable('src_grid_subset', src_grid_subset):
if not vm.is_null:
if not self.allow_masked:
gmask = src_grid_subset.get_mask()
if gmask is not None and gmask.any():
raise ValueError(
'Masked values in source grid subset.')
if self.check_contains:
src_box = box(*src_grid_subset.extent_global)
if not does_contain(src_box, dst_box):
raise ValueError('Contains check failed.')
# Try to reduce the coordinates in the case of unstructured grid data.
if hasattr(src_grid_subset, 'reduce_global'
) and src_grid_subset.cindex is not None:
ocgis_lh(logger='grid_chunker',
msg='starting reduce_global',
level=logging.DEBUG)
src_grid_subset = src_grid_subset.reduce_global()
ocgis_lh(logger='grid_chunker',
msg='finished reduce_global',
level=logging.DEBUG)
else:
pass
# src_grid_subset = VariableCollection(is_empty=True)
if src_grid_subset.is_empty:
src_grid_slice = None
else:
src_grid_slice = {
src_grid_subset.dimensions[ii].name: src_grid_slice[ii]
for ii in range(src_grid_subset.ndim)
}
if yield_dst:
yld = (src_grid_subset, src_grid_slice, dst_grid_subset,
dst_slice)
else:
yld = src_grid_subset, src_grid_slice
yield yld
def iter_src_grid_subsets(self, yield_dst=False, yield_idx=None):
"""
Yield source grid subset using the extent of its associated destination grid subset.
:param bool yield_dst: If ``True``, yield the destination subset as well as the source grid subset.
:param int yield_idx: If a zero-based integer, only yield for this chunk index and skip everything else.
:rtype: tuple(:class:`ocgis.spatial.grid.AbstractGrid`, `slice-like`)
"""
if yield_dst:
yield_slice = True
else:
yield_slice = False
buffer_value = self.buffer_value
dst_grid_wrapped_state = self.dst_grid.wrapped_state
dst_grid_crs = self.dst_grid.crs
# Use a destination grid iterator if provided.
if self.iter_dst is not None:
iter_dst = self.iter_dst(self, yield_slice=yield_slice, yield_idx=yield_idx)
else:
iter_dst = self.iter_dst_grid_subsets(yield_slice=yield_slice, yield_idx=yield_idx)
# Loop over each destination grid subset.
ocgis_lh(logger='grid_chunker', msg='starting "for yld in iter_dst"', level=logging.DEBUG)
for iter_dst_ctr, yld in enumerate(iter_dst, start=1):
ocgis_lh(msg=["iter_dst_ctr", iter_dst_ctr], level=logging.DEBUG)
if yield_slice:
dst_grid_subset, dst_slice = yld
else:
dst_grid_subset = yld
# All masked destinations are very problematic for ESMF
with vm.scoped_by_emptyable('global mask', dst_grid_subset):
if not vm.is_null:
if dst_grid_subset.has_mask_global:
if dst_grid_subset.has_mask and dst_grid_subset.has_masked_values:
all_masked = dst_grid_subset.get_mask().all()
else:
all_masked = False
all_masked_gather = vm.gather(all_masked)
if vm.rank == 0:
if all(all_masked_gather):
exc = ValueError("Destination subset all masked")
try:
raise exc
finally:
vm.abort(exc=exc)
dst_box = None
with vm.scoped_by_emptyable('extent_global', dst_grid_subset):
if not vm.is_null:
# Use the extent of the polygon for determining the bounding box. This ensures conservative
# regridding will be fully mapped.
if isinstance(dst_grid_subset, AbstractGeometryCoordinates):
target_grid = dst_grid_subset.parent.grid
else:
target_grid = dst_grid_subset
# Try to reduce the coordinates in the case of unstructured grid data.
if hasattr(target_grid, 'reduce_global') and Topology.POLYGON in target_grid.abstractions_available:
ocgis_lh(logger='grid_chunker', msg='starting reduce_global for dst_grid_subset',
level=logging.DEBUG)
target_grid = target_grid.reduce_global()
ocgis_lh(logger='grid_chunker', msg='finished reduce_global for dst_grid_subset',
level=logging.DEBUG)
extent_global = target_grid.parent.attrs.get('extent_global')
if extent_global is None:
with grid_abstraction_scope(target_grid, Topology.POLYGON):
extent_global = target_grid.extent_global
if self.check_contains:
dst_box = box(*target_grid.extent_global)
sub_box = box(*extent_global)
if buffer_value is not None:
# Use the envelope! A buffer returns "fancy" borders. We just want to expand the bounding box.
sub_box = sub_box.buffer(buffer_value).envelope
ocgis_lh(msg=str(sub_box.bounds), level=logging.DEBUG, logger='grid_chunker')
else:
sub_box, dst_box = [None, None]
live_ranks = vm.get_live_ranks_from_object(dst_grid_subset)
sub_box = vm.bcast(sub_box, root=live_ranks[0])
if self.check_contains:
dst_box = vm.bcast(dst_box, root=live_ranks[0])
sub_box = GeometryVariable.from_shapely(sub_box, is_bbox=True, wrapped_state=dst_grid_wrapped_state,
crs=dst_grid_crs)
ocgis_lh(logger='grid_chunker', msg='starting "self.src_grid.get_intersects"', level=logging.DEBUG)
src_grid_subset, src_grid_slice = self.src_grid.get_intersects(sub_box, keep_touches=False, cascade=False,
optimized_bbox_subset=self.optimized_bbox_subset,
return_slice=True)
ocgis_lh(logger='grid_chunker', msg='finished "self.src_grid.get_intersects"', level=logging.DEBUG)
# Reload the data using a new source index distribution.
if hasattr(src_grid_subset, 'reduce_global') and src_grid_subset.cindex is not None:
# Only redistribute if we have one live rank.
if self.redistribute and len(vm.get_live_ranks_from_object(src_grid_subset)) > 0:
ocgis_lh(logger='grid_chunker', msg='starting redistribute', level=logging.DEBUG)
topology = src_grid_subset.abstractions_available[Topology.POLYGON]
cindex = topology.cindex
redist_dimname = self.src_grid.abstractions_available[Topology.POLYGON].element_dim.name
if src_grid_subset.is_empty:
redist_dim = None
else:
redist_dim = topology.element_dim
redistribute_by_src_idx(cindex, redist_dimname, redist_dim)
ocgis_lh(logger='grid_chunker', msg='finished redistribute', level=logging.DEBUG)
with vm.scoped_by_emptyable('src_grid_subset', src_grid_subset):
if not vm.is_null:
if not self.allow_masked:
gmask = src_grid_subset.get_mask()
if gmask is not None and gmask.any():
raise ValueError('Masked values in source grid subset.')
if self.check_contains:
src_box = box(*src_grid_subset.extent_global)
if not does_contain(src_box, dst_box):
raise ValueError('Contains check failed.')
# Try to reduce the coordinates in the case of unstructured grid data.
if hasattr(src_grid_subset, 'reduce_global') and src_grid_subset.cindex is not None:
ocgis_lh(logger='grid_chunker', msg='starting reduce_global', level=logging.DEBUG)
src_grid_subset = src_grid_subset.reduce_global()
ocgis_lh(logger='grid_chunker', msg='finished reduce_global', level=logging.DEBUG)
else:
pass
# src_grid_subset = VariableCollection(is_empty=True)
if src_grid_subset.is_empty:
src_grid_slice = None
else:
src_grid_slice = {src_grid_subset.dimensions[ii].name: src_grid_slice[ii] for ii in
range(src_grid_subset.ndim)}
if yield_dst:
yld = (src_grid_subset, src_grid_slice, dst_grid_subset, dst_slice)
else:
yld = src_grid_subset, src_grid_slice
yield yld