def fetch(self, grouped: VirtualDatasetBox,
**load_settings: Dict[str, Any]) -> xarray.Dataset:
""" Convert grouped datasets to `xarray.Dataset`. """
load_keys = self._LOAD_KEYS - {'measurements'}
merged = merge_search_terms(select_keys(self, load_keys),
select_keys(load_settings, load_keys))
product = grouped.product_definitions[self._product]
if 'measurements' in self and 'measurements' in load_settings:
for measurement in load_settings['measurements']:
self._assert(
measurement in self['measurements'],
'{} not found in {}'.format(measurement, self._product))
measurement_dicts = self.output_measurements(
grouped.product_definitions, load_settings.get('measurements'))
if grouped.load_natively:
canonical_names = [
product.canonical_measurement(measurement)
for measurement in measurement_dicts
]
dataset_geobox = geobox_union_conservative([
native_geobox(ds,
measurements=canonical_names,
basis=merged.get('like'))
for ds in grouped.box.sum().item()
])
if grouped.geopolygon is not None:
reproject_roi = compute_reproject_roi(
dataset_geobox,
GeoBox.from_geopolygon(
grouped.geopolygon,
crs=dataset_geobox.crs,
align=dataset_geobox.alignment,
resolution=dataset_geobox.resolution))
self._assert(reproject_roi.is_st,
"native load is not axis-aligned")
self._assert(numpy.isclose(reproject_roi.scale, 1.0),
"native load should not require scaling")
geobox = dataset_geobox[reproject_roi.roi_src]
else:
geobox = dataset_geobox
else:
geobox = grouped.geobox
result = Datacube.load_data(grouped.box,
geobox,
list(measurement_dicts.values()),
fuse_func=merged.get('fuse_func'),
dask_chunks=merged.get('dask_chunks'),
resampling=merged.get(
'resampling', 'nearest'))
return result
def compute_native_load_geobox(dst_geobox: GeoBox,
ds: Dataset,
band: str,
buffer: Optional[float] = None) -> GeoBox:
"""
Compute area of interest for a given Dataset given query.
Take native projection and resolution from ``ds, band`` pair and compute
region in that projection that fully encloses footprint of the
``dst_geobox`` with some padding. Construct GeoBox that encloses that
region fully with resolution/pixel alignment copied from supplied band.
:param dst_geobox:
:param ds: Sample dataset (only resolution and projection is used, not footprint)
:param band: Reference band to use (resolution of output GeoBox will match resolution of this band)
:param buffer: Buffer in units of CRS of ``ds`` (meters usually), default is 10 pixels worth
"""
native: GeoBox = native_geobox(ds, basis=band)
if buffer is None:
buffer = 10 * cast(float, max(map(abs,
native.resolution))) # type: ignore
assert native.crs is not None
return GeoBox.from_geopolygon(
dst_geobox.extent.to_crs(native.crs).buffer(buffer),
crs=native.crs,
resolution=native.resolution,
align=native.alignment,
)
def __call__(self, index, product, time, group_by) -> Tile:
# Do for a specific poly whose boundary is known
output_crs = CRS(self.storage['crs'])
filtered_items = [
'geopolygon', 'lon', 'lat', 'longitude', 'latitude', 'x', 'y'
]
filtered_dict = {
k: v
for k, v in self.input_region.items() if k in filtered_items
}
if self.feature is not None:
filtered_dict['geopolygon'] = self.feature.geopolygon
geopoly = filtered_dict['geopolygon']
else:
geopoly = query_geopolygon(**self.input_region)
dc = Datacube(index=index)
datasets = dc.find_datasets(product=product,
time=time,
group_by=group_by,
**filtered_dict)
group_by = query_group_by(group_by=group_by)
sources = dc.group_datasets(datasets, group_by)
output_resolution = [
self.storage['resolution'][dim] for dim in output_crs.dimensions
]
geopoly = geopoly.to_crs(output_crs)
geobox = GeoBox.from_geopolygon(geopoly, resolution=output_resolution)
return Tile(sources, geobox)
def __call__(self, product, time, group_by) -> Tile:
# Do for a specific poly whose boundary is known
output_crs = CRS(self.storage['crs'])
filtered_item = [
'geopolygon', 'lon', 'lat', 'longitude', 'latitude', 'x', 'y'
]
filtered_dict = {
k: v
for k, v in filter(lambda t: t[0] in filtered_item,
self.input_region.items())
}
if 'feature_id' in self.input_region:
filtered_dict['geopolygon'] = Geometry(
self.input_region['geom_feat'],
CRS(self.input_region['crs_txt']))
geopoly = filtered_dict['geopolygon']
else:
geopoly = query_geopolygon(**self.input_region)
datasets = self.dc.find_datasets(product=product,
time=time,
group_by=group_by,
**filtered_dict)
group_by = query_group_by(group_by=group_by)
sources = self.dc.group_datasets(datasets, group_by)
output_resolution = [
self.storage['resolution'][dim] for dim in output_crs.dimensions
]
geopoly = geopoly.to_crs(output_crs)
geobox = GeoBox.from_geopolygon(geopoly, resolution=output_resolution)
return Tile(sources, geobox)
def test_read_with_reproject(tmpdir):
from datacube.testutils import mk_test_image
from datacube.testutils.io import write_gtiff
from pathlib import Path
pp = Path(str(tmpdir))
xx = mk_test_image(128, 64, nodata=None)
assert (xx != -999).all()
tile = AlbersGS.tile_geobox((17, -40))[:64, :128]
mm = write_gtiff(pp / 'tst-read-with-reproject-128x64-int16.tif',
xx,
crs=str(tile.crs),
resolution=tile.resolution[::-1],
offset=tile.transform * (0, 0),
nodata=-999)
assert mm.gbox == tile
def _read(gbox,
resampling='nearest',
fallback_nodata=None,
dst_nodata=-999):
with RasterFileDataSource(mm.path, 1,
nodata=fallback_nodata).open() as rdr:
yy = np.full(gbox.shape, dst_nodata, dtype=rdr.dtype)
roi = read_time_slice(rdr, yy, gbox, resampling, dst_nodata)
return yy, roi
gbox = gbx.pad(mm.gbox, 10)
gbox = gbx.zoom_out(gbox, 0.873)
yy, roi = _read(gbox)
assert roi[0].start > 0 and roi[1].start > 0
assert (yy[0] == -999).all()
yy_expect, _ = rio_slurp(mm.path, gbox)
np.testing.assert_array_equal(yy, yy_expect)
gbox = gbx.zoom_out(mm.gbox[3:-3, 10:-10], 2.1)
yy, roi = _read(gbox)
assert roi_shape(roi) == gbox.shape
assert not (yy == -999).any()
gbox = GeoBox.from_geopolygon(mm.gbox.extent.to_crs(epsg3857).buffer(50),
resolution=mm.gbox.resolution)
assert gbox.extent.contains(mm.gbox.extent.to_crs(epsg3857))
assert gbox.crs != mm.gbox.crs
yy, roi = _read(gbox)
assert roi[0].start > 0 and roi[1].start > 0
assert (yy[0] == -999).all()
gbox = gbx.zoom_out(gbox, 4)
yy, roi = _read(gbox, resampling='average')
nvalid = (yy != -999).sum()
nempty = (yy == -999).sum()
assert nvalid > nempty
def compute_native_load_geobox(dst_geobox: GeoBox,
ds: Dataset,
band: str,
buffer: Optional[float] = None):
native = native_geobox(ds, basis=band)
if buffer is None:
buffer = 10*max(map(abs, native.resolution))
return GeoBox.from_geopolygon(dst_geobox.extent.to_crs(native.crs).buffer(buffer),
crs=native.crs,
resolution=native.resolution,
align=native.alignment)
def gbox_reproject(
geobox: GeoBox,
crs: SomeCRS,
resolution: Optional[Tuple[int, int]] = None,
pad: int = 0,
pad_wh: Union[int, Tuple[int, int]] = 16,
) -> GeoBox:
"""
Compute GeoBox in a given projection that fully encloses footprint of the source GeoBox.
:param geobox: Source GeoBox
:param crs: CRS of the output GeoBox
:param resolution: Desired output resolution (defaults to source
resolution, if source and destination projections share
common units)
:param pad: Padding in pixels of output GeoBox
:param pad_wh: Expand output GeoBox some more such that (W%pad_wh) == 0 and (H%pad_wh) == 0
"""
from datacube.utils.geometry import gbox
crs = _norm_crs_or_error(crs)
if resolution is None:
if geobox.crs.units == crs.units:
resolution = geobox.resolution
else:
raise NotImplementedError(
"Source and destination projections have different units: have to supply desired output resolution"
)
out = GeoBox.from_geopolygon(geobox.extent, resolution, crs)
if pad > 0:
out = gbox.pad(out, pad)
if pad_wh:
if isinstance(pad_wh, int):
pad_wh = max(1, pad_wh)
pad_wh = (pad_wh, pad_wh)
out = gbox.pad_wh(out, *pad_wh)
return out
def test_can_paste():
src = AlbersGS.tile_geobox((17, -40))
def check_true(dst, **kwargs):
ok, reason = can_paste(compute_reproject_roi(src, dst), **kwargs)
if not ok:
assert ok is True, reason
def check_false(dst, **kwargs):
ok, reason = can_paste(compute_reproject_roi(src, dst), **kwargs)
if ok:
assert ok is False, "Expected can_paste to return False, but got True"
check_true(gbx.pad(src, 100))
check_true(src[:10, :10])
check_true(gbx.translate_pix(src, 0.1, 0.3), ttol=0.5)
check_true(gbx.translate_pix(src, 3, -4))
check_true(gbx.flipx(src))
check_true(gbx.flipy(src))
check_true(gbx.flipx(gbx.flipy(src)))
check_true(gbx.zoom_out(src, 2))
check_true(gbx.zoom_out(src, 4))
check_true(gbx.zoom_out(src[:9, :9], 3))
# Check False code paths
dst = GeoBox.from_geopolygon(src.extent.to_crs(epsg3857).buffer(10),
resolution=src.resolution)
check_false(dst) # non ST
check_false(gbx.affine_transform_pix(src, Affine.rotation(1))) # non ST
check_false(gbx.zoom_out(src, 1.9)) # non integer scale
check_false(gbx.affine_transform_pix(src, Affine.scale(1, 2))) # sx != sy
dst = gbx.translate_pix(src, -1, -1)
dst = gbx.zoom_out(dst, 2)
check_false(dst) # src_roi doesn't align for scale
check_false(dst, stol=0.7) # src_roi/scale != dst_roi
check_false(gbx.translate_pix(src, 0.3, 0.4)) # sub-pixel translation
