def step1a(input_slice):
nonlocal geo_coding
geo_coding = GeoCoding.from_dataset(input_slice)
subset = select_spatial_subset(input_slice,
xy_bbox=output_geom.xy_bbox,
xy_border=output_geom.xy_res,
ij_border=1,
geo_coding=geo_coding)
if subset is None:
monitor('no spatial overlap with input')
elif subset is not input_slice:
geo_coding = GeoCoding.from_dataset(subset)
return subset
def test_is_geo_crs_and_is_lon_normalized(self):
x = xr.DataArray(np.linspace(10.0, 20.0, 21),
dims='columns',
name='lon')
y = xr.DataArray(np.linspace(53.0, 58.0, 11), dims='rows', name='lat')
y, x = xr.broadcast(y, x)
gc = GeoCoding(x, y)
self.assertEqual(False, gc.is_geo_crs)
self.assertEqual(False, gc.is_lon_normalized)
gc = GeoCoding(x, y, is_geo_crs=True)
self.assertEqual(True, gc.is_geo_crs)
self.assertEqual(False, gc.is_lon_normalized)
gc = GeoCoding(x, y, is_lon_normalized=True)
self.assertEqual(True, gc.is_geo_crs)
self.assertEqual(True, gc.is_lon_normalized)
def _test_ij_bbox_antimeridian(self, conservative: bool):
def denorm(x):
return x if x <= 180 else x - 360
lon = xr.DataArray(np.linspace(175.0, 185.0, 21), dims='columns')
lat = xr.DataArray(np.linspace(53.0, 58.0, 11), dims='rows')
lat, lon = xr.broadcast(lat, lon)
gc = GeoCoding(x=lon,
y=lat,
x_name='lon',
y_name='lat',
is_lon_normalized=True)
ij_bbox = gc.ij_bbox_conservative if conservative else gc.ij_bbox
self.assertEqual((-1, -1, -1, -1), ij_bbox((0, -50, 30, 0)))
self.assertEqual((0, 0, 20, 10),
ij_bbox((denorm(160), 50, denorm(200), 60)))
self.assertEqual((0, 0, 20, 6),
ij_bbox((denorm(160), 50, denorm(200), 56)))
self.assertEqual((10, 0, 20, 6),
ij_bbox((denorm(180), 50, denorm(200), 56)))
self.assertEqual((10, 0, 16, 6),
ij_bbox((denorm(180), 50, denorm(183), 56)))
self.assertEqual((10, 1, 16, 6),
ij_bbox((denorm(180), 53.5, denorm(183), 56)))
self.assertEqual((8, 0, 18, 8),
ij_bbox((denorm(180), 53.5, denorm(183), 56),
ij_border=2))
self.assertEqual((12, 1, 20, 6),
ij_bbox((denorm(181), 53.5, denorm(200), 56)))
self.assertEqual((12, 1, 18, 6),
ij_bbox((denorm(181), 53.5, denorm(184), 56)))
def _select_variables(dataset,
var_names: Union[str, Sequence[str]] = None,
geo_coding: GeoCoding = None,
xy_names: Tuple[str, str] = None) -> Mapping[str, xr.DataArray]:
"""
Select variables from *dataset*.
:param dataset: Source dataset.
:param var_names: Optional variable name or sequence of variable names.
:param geo_coding: Optional dataset geo-coding.
:param xy_names: Optional tuple of the x- and y-coordinate variables in *dataset*. Ignored if *geo_coding* is given.
:return: The selected variables as a variable name to ``xr.DataArray`` mapping
"""
geo_coding = geo_coding if geo_coding is not None else GeoCoding.from_dataset(dataset, xy_names=xy_names)
src_x = geo_coding.x
x_name, y_name = geo_coding.xy_names
if var_names is None:
var_names = [var_name for var_name, var in dataset.data_vars.items()
if var_name not in (x_name, y_name) and _is_2d_var(var, src_x)]
elif isinstance(var_names, str):
var_names = (var_names,)
elif len(var_names) == 0:
raise ValueError(f'empty var_names')
src_vars = {}
for var_name in var_names:
src_var = dataset[var_name]
if not _is_2d_var(src_var, src_x):
raise ValueError(
f"cannot reproject variable {var_name!r} as its shape or dimensions "
f"do not match those of {x_name!r} and {y_name!r}")
src_vars[var_name] = src_var
return src_vars
def process(self,
dataset: xr.Dataset,
geo_coding: GeoCoding,
output_geom: ImageGeom,
output_resampling: str,
include_non_spatial_vars=False) -> xr.Dataset:
"""
Perform reprojection using tie-points / ground control points.
"""
reprojection_info = self.get_reprojection_info(dataset)
in_rectification_mode = reprojection_info.xy_gcp_step is None
if in_rectification_mode:
warn_prefix = 'unsupported argument in np-GCP rectification mode'
if reprojection_info.xy_tp_gcp_step is not None:
warnings.warn(
f'{warn_prefix}: ignoring '
f'reprojection_info.xy_tp_gcp_step = {reprojection_info.xy_tp_gcp_step!r}'
)
if output_resampling != 'Nearest':
warnings.warn(f'{warn_prefix}: ignoring '
f'dst_resampling = {output_resampling!r}')
if include_non_spatial_vars:
warnings.warn(
f'{warn_prefix}: ignoring '
f'include_non_spatial_vars = {include_non_spatial_vars!r}')
geo_coding = geo_coding.derive(
x_name=reprojection_info.xy_names[0],
y_name=reprojection_info.xy_names[1])
dataset = rectify_dataset(dataset,
compute_subset=False,
geo_coding=geo_coding,
output_geom=output_geom,
is_y_axis_inverted=True)
if dataset is not None and geo_coding.is_geo_crs and geo_coding.xy_names != (
'lon', 'lat'):
dataset = dataset.rename({
geo_coding.x_name: 'lon',
geo_coding.y_name: 'lat'
})
return dataset
else:
return reproject_xy_to_wgs84(
dataset,
src_xy_var_names=reprojection_info.xy_names,
src_xy_tp_var_names=reprojection_info.xy_tp_names,
src_xy_crs=reprojection_info.xy_crs,
src_xy_gcp_step=reprojection_info.xy_gcp_step or 1,
src_xy_tp_gcp_step=reprojection_info.xy_tp_gcp_step or 1,
dst_size=output_geom.size,
dst_region=output_geom.xy_bbox,
dst_resampling=output_resampling,
include_non_spatial_vars=include_non_spatial_vars)
def test_from_dataset_1d(self):
x = xr.DataArray(np.linspace(10.0, 20.0, 21), dims='x')
y = xr.DataArray(np.linspace(53.0, 58.0, 11), dims='y')
gc = GeoCoding.from_dataset(xr.Dataset(dict(x=x, y=y)))
self.assertIsInstance(gc.x, xr.DataArray)
self.assertIsInstance(gc.y, xr.DataArray)
self.assertEqual('x', gc.x_name)
self.assertEqual('y', gc.y_name)
self.assertEqual(False, gc.is_lon_normalized)
def test_ij_bboxes(self):
x = xr.DataArray(np.linspace(10.0, 20.0, 21), dims='x')
y = xr.DataArray(np.linspace(53.0, 58.0, 11), dims='y')
y, x = xr.broadcast(y, x)
gc = GeoCoding(x=x, y=y, x_name='x', y_name='y', is_geo_crs=True)
ij_bboxes = gc.ij_bboxes(np.array([(0.0, -50.0, 30.0, 0.0)]))
np.testing.assert_almost_equal(
ij_bboxes, np.array([(-1, -1, -1, -1)], dtype=np.int64))
ij_bboxes = gc.ij_bboxes(
np.array([(0.0, 50, 30, 60), (0.0, 50, 30, 56), (15, 50, 30, 56),
(15, 50, 18, 56), (15, 53.5, 18, 56)]))
np.testing.assert_almost_equal(
ij_bboxes,
np.array([(0, 0, 20, 10), (0, 0, 20, 6), (10, 0, 20, 6),
(10, 0, 16, 6), (10, 1, 16, 6)],
dtype=np.int64))
def select_spatial_subset(
dataset: xr.Dataset,
ij_bbox: Tuple[int, int, int, int] = None,
ij_border: int = 0,
xy_bbox: Tuple[float, float, float, float] = None,
xy_border: float = 0.,
geo_coding: GeoCoding = None,
xy_names: Tuple[str, str] = None) -> Optional[xr.Dataset]:
"""
Select a spatial subset of *dataset* for the bounding box *ij_bbox* or *xy_bbox*.
:param dataset: Source dataset.
:param ij_bbox: Bounding box (i_min, i_min, j_max, j_max) in pixel coordinates.
:param geo_coding: Optional dataset geo-coding.
:param xy_names: Optional tuple of the x- and y-coordinate variables in *dataset*. Ignored if *geo_coding* is given.
:return: Spatial dataset subset
"""
if ij_bbox is None and xy_bbox is None:
raise ValueError('One of ij_bbox and xy_bbox must be given')
if ij_bbox and xy_bbox:
raise ValueError('Only one of ij_bbox and xy_bbox can be given')
geo_coding = geo_coding if geo_coding is not None else GeoCoding.from_dataset(
dataset, xy_names=xy_names)
if xy_bbox:
ij_bbox = geo_coding.ij_bbox(xy_bbox,
ij_border=ij_border,
xy_border=xy_border)
if ij_bbox[0] == -1:
return None
width, height = geo_coding.size
i_min, j_min, i_max, j_max = ij_bbox
if i_min > 0 or j_min > 0 or i_max < width - 1 or j_max < height - 1:
x_dim, y_dim = geo_coding.dims
i_slice = slice(i_min, i_max + 1)
j_slice = slice(j_min, j_max + 1)
return dataset.isel({x_dim: i_slice, y_dim: j_slice})
return dataset
def _test_ij_bbox(self, conservative: bool):
x = xr.DataArray(np.linspace(10.0, 20.0, 21), dims='x')
y = xr.DataArray(np.linspace(53.0, 58.0, 11), dims='y')
y, x = xr.broadcast(y, x)
gc = GeoCoding(x=x, y=y, x_name='x', y_name='y', is_geo_crs=True)
ij_bbox = gc.ij_bbox_conservative if conservative else gc.ij_bbox
self.assertEqual((-1, -1, -1, -1), ij_bbox((0, -50, 30, 0)))
self.assertEqual((0, 0, 20, 10), ij_bbox((0, 50, 30, 60)))
self.assertEqual((0, 0, 20, 6), ij_bbox((0, 50, 30, 56)))
self.assertEqual((10, 0, 20, 6), ij_bbox((15, 50, 30, 56)))
self.assertEqual((10, 0, 16, 6), ij_bbox((15, 50, 18, 56)))
self.assertEqual((10, 1, 16, 6), ij_bbox((15, 53.5, 18, 56)))
self.assertEqual((8, 0, 18, 8), ij_bbox((15, 53.5, 18, 56),
ij_border=2))
def _compute_ij_images_xarray_dask(src_geo_coding: GeoCoding,
output_geom: ImageGeom,
uv_delta: float) -> da.Array:
"""Compute dask.array.Array destination image with source pixel i,j coords from xarray.DataArray x,y sources """
dst_width = output_geom.width
dst_height = output_geom.height
dst_tile_width = output_geom.tile_width
dst_tile_height = output_geom.tile_height
dst_var_shape = 2, dst_height, dst_width
dst_var_chunks = 2, dst_tile_height, dst_tile_width
dst_x_min, dst_y_min, dst_x_max, dst_y_max = output_geom.xy_bbox
dst_xy_res = output_geom.xy_res
# Compute an empirical xy_border as a function of the number of tiles, because the more tiles we have
# the smaller the destination xy-bboxes and the higher the risk to not find any source ij-bbox for
# a given xy-bbox.
# xy_border will not be larger than half of the coverage of a tile.
#
num_tiles_x = dst_width / dst_tile_width
num_tiles_y = dst_height / dst_tile_height
max_num_tiles = max(num_tiles_x, num_tiles_y)
xy_border = min(2 * max_num_tiles * dst_xy_res,
min(0.5 * (dst_x_max - dst_x_min), 0.5 * (dst_y_max - dst_y_min)))
dst_xy_bboxes = output_geom.xy_bboxes
src_ij_bboxes = src_geo_coding.ij_bboxes(dst_xy_bboxes, xy_border=xy_border, ij_border=1)
return compute_array_from_func(_compute_ij_images_xarray_dask_block,
dst_var_shape,
dst_var_chunks,
np.float64,
ctx_arg_names=[
'dtype',
'block_id',
'block_shape',
'block_slices',
],
args=(
src_geo_coding.x,
src_geo_coding.y,
src_ij_bboxes,
dst_x_min,
dst_y_min,
dst_xy_res,
uv_delta
),
name='ij_pixels',
)
def _compute_output_geom(
dataset: xr.Dataset,
geo_coding: GeoCoding = None,
xy_names: Tuple[str, str] = None,
xy_oversampling: float = 1.0,
xy_eps: float = 1e-10,
ij_denom: Union[int, Tuple[int, int]] = None) -> ImageGeom:
i_denom, j_denom = ((ij_denom, ij_denom)
if isinstance(ij_denom, int) else ij_denom) or (1, 1)
geo_coding = geo_coding if geo_coding is not None else GeoCoding.from_dataset(
dataset, xy_names=xy_names)
src_x, src_y = geo_coding.xy
dim_y, dim_x = src_x.dims
src_x_x_diff = src_x.diff(dim=dim_x)
src_x_y_diff = src_x.diff(dim=dim_y)
src_y_x_diff = src_y.diff(dim=dim_x)
src_y_y_diff = src_y.diff(dim=dim_y)
src_x_x_diff_sq = np.square(src_x_x_diff)
src_x_y_diff_sq = np.square(src_x_y_diff)
src_y_x_diff_sq = np.square(src_y_x_diff)
src_y_y_diff_sq = np.square(src_y_y_diff)
src_x_diff = np.sqrt(src_x_x_diff_sq + src_y_x_diff_sq)
src_y_diff = np.sqrt(src_x_y_diff_sq + src_y_y_diff_sq)
src_x_res = float(src_x_diff.where(src_x_diff > xy_eps).min())
src_y_res = float(src_y_diff.where(src_y_diff > xy_eps).min())
src_xy_res = min(src_x_res, src_y_res) / (math.sqrt(2.0) * xy_oversampling)
src_x_min = float(src_x.min())
src_x_max = float(src_x.max())
src_y_min = float(src_y.min())
src_y_max = float(src_y.max())
dst_width = 1 + math.floor((src_x_max - src_x_min) / src_xy_res)
dst_height = 1 + math.floor((src_y_max - src_y_min) / src_xy_res)
dst_width = i_denom * ((dst_width + i_denom - 1) // i_denom)
dst_height = j_denom * ((dst_height + j_denom - 1) // j_denom)
# Reduce height if it takes the maximum latitude over 90° (see Issue #303).
if src_y_min + dst_height * src_xy_res > 90.0:
dst_height -= 1
return ImageGeom((dst_width, dst_height),
x_min=src_x_min,
y_min=src_y_min,
xy_res=src_xy_res,
is_geo_crs=geo_coding.is_geo_crs)
def process(self,
dataset: xr.Dataset,
geo_coding: GeoCoding,
output_geom: ImageGeom,
output_resampling: str,
include_non_spatial_vars=False) -> xr.Dataset:
"""
Perform reprojection using tie-points / ground control points.
"""
reprojection_info = self.get_reprojection_info(dataset)
in_rectification_mode = reprojection_info.xy_gcp_step is None
if in_rectification_mode:
warn_prefix = 'unsupported argument in np-GCP rectification mode'
if reprojection_info.xy_tp_gcp_step is not None:
warnings.warn(
f'{warn_prefix}: ignoring '
f'reprojection_info.xy_tp_gcp_step = {reprojection_info.xy_tp_gcp_step!r}'
)
if output_resampling != 'Nearest':
warnings.warn(f'{warn_prefix}: ignoring '
f'dst_resampling = {output_resampling!r}')
if include_non_spatial_vars:
warnings.warn(
f'{warn_prefix}: ignoring '
f'include_non_spatial_vars = {include_non_spatial_vars!r}')
geo_coding = geo_coding.derive(
x_name=reprojection_info.xy_names[0],
y_name=reprojection_info.xy_names[1])
dataset = rectify_dataset(dataset,
compute_subset=False,
geo_coding=geo_coding,
output_geom=output_geom,
is_y_reversed=True)
if output_geom.is_tiled:
# The following condition may become true, if we have used rectified_dataset(input, ..., is_y_reverse=True)
# In this case y-chunksizes will also be reversed. So that the first chunk is smaller than any other.
# Zarr will reject such datasets, when written.
if dataset.chunks.get('lat')[0] < dataset.chunks.get(
'lat')[-1]:
dataset = dataset.chunk({
'lat': output_geom.tile_height,
'lon': output_geom.tile_width
})
if dataset is not None and geo_coding.is_geo_crs and geo_coding.xy_names != (
'lon', 'lat'):
dataset = dataset.rename({
geo_coding.x_name: 'lon',
geo_coding.y_name: 'lat'
})
return dataset
else:
return reproject_xy_to_wgs84(
dataset,
src_xy_var_names=reprojection_info.xy_names,
src_xy_tp_var_names=reprojection_info.xy_tp_names,
src_xy_crs=reprojection_info.xy_crs,
src_xy_gcp_step=reprojection_info.xy_gcp_step or 1,
src_xy_tp_gcp_step=reprojection_info.xy_tp_gcp_step or 1,
dst_size=output_geom.size,
dst_region=output_geom.xy_bbox,
dst_resampling=output_resampling,
include_non_spatial_vars=include_non_spatial_vars)
def rectify_dataset(dataset: xr.Dataset,
var_names: Union[str, Sequence[str]] = None,
geo_coding: GeoCoding = None,
xy_names: Tuple[str, str] = None,
output_geom: ImageGeom = None,
is_y_reversed: bool = False,
tile_size: Union[int, Tuple[int, int]] = None,
output_ij_names: Tuple[str, str] = None,
load_xy: bool = False,
load_vars: bool = False,
compute_subset: bool = True,
uv_delta: float = 1e-3) -> Optional[xr.Dataset]:
"""
Reproject *dataset* using its per-pixel x,y coordinates or the given *geo_coding*.
The function expects *dataset* to have either one- or two-dimensional coordinate variables
that provide spatial x,y coordinates for every data variable with the same spatial dimensions.
For example, a dataset may comprise variables with spatial dimensions ``var(..., y_dim, x_dim)``, then one
the function expects coordinates to be provided in two forms:
1. One-dimensional ``x_var(x_dim)`` and ``y_var(y_dim)`` (coordinate) variables.
2. Two-dimensional ``x_var(y_dim, x_dim)`` and ``y_var(y_dim, x_dim)`` (coordinate) variables.
If *output_geom* is given and defines a tile size or *tile_size* is given, and the number of tiles
is greater than one in the output's x- or y-direction, then the returned dataset will be composed of lazy,
chunked dask arrays. Otherwise the returned dataset will be composed of ordinary numpy arrays.
:param dataset: Source dataset.
:param var_names: Optional variable name or sequence of variable names.
:param geo_coding: Optional dataset geo-coding.
:param xy_names: Optional tuple of the x- and y-coordinate variables in *dataset*. Ignored if *geo_coding* is given.
:param output_geom: Optional output geometry. If not given, output geometry will be computed
to spatially fit *dataset* and to retain its spatial resolution.
:param is_y_reversed: Whether the y-axis labels in the output should be in reverse order.
:param tile_size: Optional tile size for the output.
:param output_ij_names: If given, a tuple of variable names in which to store the computed source pixel
coordinates in the returned output.
:param load_xy: Compute x,y coordinates and load into memory before the actual rectification process.
May improve runtime performance at the cost of higher memory consumption.
:param load_vars: Compute source variables and load into memory before the actual rectification process.
May improve runtime performance at the cost of higher memory consumption.
:param compute_subset: Whether to compute a spatial subset from *dataset* using *output_geom*. If set,
The function may return ``None`` in case there is no overlap.
:param uv_delta: A normalized value that is used to determine whether x,y coordinates in the output are contained
in the triangles defined by the input x,y coordinates.
The higher this value, the more inaccurate the rectification will be.
:return: a reprojected dataset, or None if the requested output does not intersect with *dataset*.
"""
src_geo_coding = geo_coding if geo_coding is not None else GeoCoding.from_dataset(dataset, xy_names=xy_names)
src_x, src_y = src_geo_coding.xy
src_attrs = dict(dataset.attrs)
if output_geom is None:
output_geom = ImageGeom.from_dataset(dataset, geo_coding=src_geo_coding)
elif compute_subset:
dataset_subset = select_spatial_subset(dataset,
xy_bbox=output_geom.xy_bbox,
ij_border=1,
xy_border=output_geom.xy_res,
geo_coding=src_geo_coding)
if dataset_subset is None:
return None
if dataset_subset is not dataset:
src_geo_coding = GeoCoding.from_dataset(dataset_subset)
src_x, src_y = src_geo_coding.x, src_geo_coding.y
dataset = dataset_subset
if tile_size is not None:
output_geom = output_geom.derive(tile_size=tile_size)
src_vars = _select_variables(dataset, var_names, geo_coding=src_geo_coding)
if load_xy:
# This is NOT faster:
src_x = src_x.compute()
src_y = src_y.compute()
src_geo_coding = src_geo_coding.derive(x=src_x, y=src_y)
if output_geom.is_tiled:
get_dst_src_ij_images = _compute_ij_images_xarray_dask
get_dst_var_image = _compute_var_image_xarray_dask
else:
get_dst_src_ij_images = _compute_ij_images_xarray_numpy
get_dst_var_image = _compute_var_image_xarray_numpy
dst_src_ij_array = get_dst_src_ij_images(src_geo_coding,
output_geom,
uv_delta)
if is_y_reversed:
# Note that the following reverse operation may change any y-axis' chunking.
# This is the case if the y-chunksize does not integer-divide y-size, e.g.
# y-chunksizes (512, 512, 512, 273) will change into (273, 512, 512, 512).
dst_src_ij_array = dst_src_ij_array[:, ::-1]
dst_dims = src_geo_coding.xy_names[::-1]
dst_ds_coords = output_geom.coord_vars(xy_names=src_geo_coding.xy_names,
is_lon_normalized=src_geo_coding.is_lon_normalized,
is_y_reversed=is_y_reversed)
dst_vars = dict()
for src_var_name, src_var in src_vars.items():
if load_vars:
# This is NOT faster:
src_var = src_var.compute()
dst_var_dims = src_var.dims[0:-2] + dst_dims
dst_var_coords = {d: src_var.coords[d] for d in dst_var_dims if d in src_var.coords}
dst_var_coords.update({d: dst_ds_coords[d] for d in dst_var_dims if d in dst_ds_coords})
dst_var_array = get_dst_var_image(src_var,
dst_src_ij_array,
fill_value=np.nan)
dst_var = xr.DataArray(dst_var_array,
dims=dst_var_dims,
coords=dst_var_coords,
attrs=src_var.attrs)
dst_vars[src_var_name] = dst_var
if output_ij_names:
output_i_name, output_j_name = output_ij_names
dst_ij_coords = {d: dst_ds_coords[d] for d in dst_dims if d in dst_ds_coords}
dst_vars[output_i_name] = xr.DataArray(dst_src_ij_array[0], dims=dst_dims, coords=dst_ij_coords)
dst_vars[output_j_name] = xr.DataArray(dst_src_ij_array[1], dims=dst_dims, coords=dst_ij_coords)
return xr.Dataset(dst_vars, coords=dst_ds_coords, attrs=src_attrs)