def test_native_load(tmpdir):
from datacube.testutils.io import native_load, native_geobox
tmpdir = Path(str(tmpdir))
spatial = dict(resolution=(15, -15),
offset=(11230, 1381110),)
nodata = -999
aa = mk_test_image(96, 64, 'int16', nodata=nodata)
cc = mk_test_image(32, 16, 'int16', nodata=nodata)
bands = [SimpleNamespace(name=name, values=aa, nodata=nodata)
for name in ['aa', 'bb']]
bands.append(SimpleNamespace(name='cc', values=cc, nodata=nodata))
ds, gbox = gen_tiff_dataset(bands[:2],
tmpdir,
prefix='ds1-',
timestamp='2018-07-19',
**spatial)
assert set(get_raster_info(ds)) == set(ds.measurements)
xx = native_load(ds)
assert xx.geobox == gbox
np.testing.assert_array_equal(aa, xx.isel(time=0).aa.values)
np.testing.assert_array_equal(aa, xx.isel(time=0).bb.values)
ds, gbox_cc = gen_tiff_dataset(bands,
tmpdir,
prefix='ds2-',
timestamp='2018-07-19',
**spatial)
# cc is different size from aa,bb
with pytest.raises(ValueError):
xx = native_load(ds)
# cc is different size from aa,bb
with pytest.raises(ValueError):
xx = native_geobox(ds)
# aa and bb are the same
assert native_geobox(ds, ['aa', 'bb']) == gbox
xx = native_load(ds, ['aa', 'bb'])
assert xx.geobox == gbox
np.testing.assert_array_equal(aa, xx.isel(time=0).aa.values)
np.testing.assert_array_equal(aa, xx.isel(time=0).bb.values)
# cc will be reprojected
assert native_geobox(ds, basis='aa') == gbox
xx = native_load(ds, basis='aa')
assert xx.geobox == gbox
np.testing.assert_array_equal(aa, xx.isel(time=0).aa.values)
np.testing.assert_array_equal(aa, xx.isel(time=0).bb.values)
# cc is compatible with self
xx = native_load(ds, ['cc'])
assert xx.geobox == gbox_cc
np.testing.assert_array_equal(cc, xx.isel(time=0).cc.values)
def test_native_geobox_eo3(eo3_dataset_s2):
ds = eo3_dataset_s2
assert ds.crs is not None
assert 'blue' in ds.measurements
gb1 = native_geobox(ds, basis='blue')
assert gb1.width == 10980
assert gb1.height == 10980
assert gb1.crs == ds.crs
gb1_ = native_geobox(ds, ('blue', 'red', 'green'))
assert gb1_ == gb1
gb2 = native_geobox(ds, ['swir_1', 'swir_2'])
assert gb2 != gb1
assert gb2.width == 5490
assert gb2.height == 5490
assert gb2.crs == ds.crs
with pytest.raises(ValueError):
native_geobox(ds)
with pytest.raises(ValueError):
native_geobox(ds, ['no_such_band'])
ds.metadata_doc['measurements']['red_edge_1']['grid'] = 'no-such-grid'
with pytest.raises(ValueError):
native_geobox(ds, ['red_edge_1'])
def compute_native_load_geobox(dst_geobox: GeoBox,
ds: Dataset,
band: str,
buffer: Optional[float] = None) -> GeoBox:
"""
Compute area of interest for a given Dataset given query.
Take native projection and resolution from ``ds, band`` pair and compute
region in that projection that fully encloses footprint of the
``dst_geobox`` with some padding. Construct GeoBox that encloses that
region fully with resolution/pixel alignment copied from supplied band.
:param dst_geobox:
:param ds: Sample dataset (only resolution and projection is used, not footprint)
:param band: Reference band to use (resolution of output GeoBox will match resolution of this band)
:param buffer: Buffer in units of CRS of ``ds`` (meters usually), default is 10 pixels worth
"""
native: GeoBox = native_geobox(ds, basis=band)
if buffer is None:
buffer = 10 * cast(float, max(map(abs,
native.resolution))) # type: ignore
assert native.crs is not None
return GeoBox.from_geopolygon(
dst_geobox.extent.to_crs(native.crs).buffer(buffer),
crs=native.crs,
resolution=native.resolution,
align=native.alignment,
)
def fetch(self, grouped: VirtualDatasetBox,
**load_settings: Dict[str, Any]) -> xarray.Dataset:
""" Convert grouped datasets to `xarray.Dataset`. """
load_keys = self._LOAD_KEYS - {'measurements'}
merged = merge_search_terms(select_keys(self, load_keys),
select_keys(load_settings, load_keys))
product = grouped.product_definitions[self._product]
if 'measurements' in self and 'measurements' in load_settings:
for measurement in load_settings['measurements']:
self._assert(
measurement in self['measurements'],
'{} not found in {}'.format(measurement, self._product))
measurement_dicts = self.output_measurements(
grouped.product_definitions, load_settings.get('measurements'))
if grouped.load_natively:
canonical_names = [
product.canonical_measurement(measurement)
for measurement in measurement_dicts
]
dataset_geobox = geobox_union_conservative([
native_geobox(ds,
measurements=canonical_names,
basis=merged.get('like'))
for ds in grouped.box.sum().item()
])
if grouped.geopolygon is not None:
reproject_roi = compute_reproject_roi(
dataset_geobox,
GeoBox.from_geopolygon(
grouped.geopolygon,
crs=dataset_geobox.crs,
align=dataset_geobox.alignment,
resolution=dataset_geobox.resolution))
self._assert(reproject_roi.is_st,
"native load is not axis-aligned")
self._assert(numpy.isclose(reproject_roi.scale, 1.0),
"native load should not require scaling")
geobox = dataset_geobox[reproject_roi.roi_src]
else:
geobox = dataset_geobox
else:
geobox = grouped.geobox
result = Datacube.load_data(grouped.box,
geobox,
list(measurement_dicts.values()),
fuse_func=merged.get('fuse_func'),
dask_chunks=merged.get('dask_chunks'),
resampling=merged.get(
'resampling', 'nearest'))
return result
def test_native_geobox_ingested():
from datacube.testutils.io import native_geobox
from datacube.testutils.geom import AlbersGS
gbox = AlbersGS.tile_geobox((15, -40))
ds = mk_sample_dataset([dict(name='a')],
geobox=gbox,
product_opts=dict(with_grid_spec=True))
assert native_geobox(ds) == gbox
# check that dataset covering several tiles is detected as invalid
ds = mk_sample_dataset([dict(name='a')],
geobox=gbox.buffered(10, 10),
product_opts=dict(with_grid_spec=True))
with pytest.raises(ValueError):
native_geobox(ds)
def compute_native_load_geobox(dst_geobox: GeoBox,
ds: Dataset,
band: str,
buffer: Optional[float] = None):
native = native_geobox(ds, basis=band)
if buffer is None:
buffer = 10*max(map(abs, native.resolution))
return GeoBox.from_geopolygon(dst_geobox.extent.to_crs(native.crs).buffer(buffer),
crs=native.crs,
resolution=native.resolution,
align=native.alignment)
def get_shape_and_transform(dataset, measurements):
"""
Get shape and transform for the given set of measurements. It try
each measurement until it succeeds.
"""
for m in measurements:
try:
geo = native_geobox(dataset, [m])
except Exception:
_LOG.warn("Failed to compute shape and transform %s", m)
continue
return geo.shape, geo.transform
# All the bands failed use default shape [1,1]
_LOG.warn("All measurements failed to compute shape and transform %s",
measurements)
return [1, 1], dataset.transform
def _native_resolution(self, task: AlchemistTask) -> float:
geobox = native_geobox(task.dataset,
basis=list(task.dataset.measurements.keys())[0])
return geobox.affine[0]