def test_band_info():
bands = [
dict(name=n, dtype='uint8', units='K', nodata=33, path=n + '.tiff')
for n in 'a b c'.split(' ')
]
ds = mk_sample_dataset(bands,
uri='file:///tmp/datataset.yml',
format='GeoTIFF')
binfo = BandInfo(ds, 'b')
assert binfo.name == 'b'
assert binfo.band is None
assert binfo.layer is None
assert binfo.dtype == 'uint8'
assert binfo.transform is None
assert binfo.crs is None
assert binfo.units == 'K'
assert binfo.nodata == 33
assert binfo.center_time == ds.center_time
assert binfo.uri == 'file:///tmp/b.tiff'
assert binfo.format == ds.format
assert binfo.driver_data is None
assert binfo.uri_scheme == 'file'
with pytest.raises(ValueError):
BandInfo(ds, 'no_such_band')
ds.uris = []
with pytest.raises(ValueError):
BandInfo(ds, 'a')
ds.uris = None
with pytest.raises(ValueError):
BandInfo(ds, 'a')
def _fuse_measurement(dest,
datasets,
geobox,
measurement,
skip_broken_datasets=False,
progress_cbk=None,
extra_dim_index=None):
srcs = []
for ds in datasets:
src = None
with ignore_exceptions_if(skip_broken_datasets):
src = new_datasource(
BandInfo(ds, measurement.name,
extra_dim_index=extra_dim_index))
if src is None:
if not skip_broken_datasets:
raise ValueError(f"Failed to load dataset: {ds.id}")
else:
srcs.append(src)
reproject_and_fuse(srcs,
dest,
geobox,
dest.dtype.type(measurement.nodata),
resampling=measurement.get('resampling_method',
'nearest'),
fuse_func=measurement.get('fuser', None),
skip_broken_datasets=skip_broken_datasets,
progress_cbk=progress_cbk,
extra_dim_index=extra_dim_index)
def test_new_datasource_fallback():
bands = [dict(name='green', path='')]
dataset = mk_sample_dataset(bands, 'file:///foo', format='GeoTiff')
assert dataset.uri_scheme == 'file'
rdr = new_datasource(BandInfo(dataset, 'green'))
assert rdr is not None
assert isinstance(rdr, RasterDatasetDataSource)
# check that None format works
band = BandInfo(mk_sample_dataset(bands, 'file:///file', format=None),
'green')
rdr = new_datasource(band)
assert rdr is not None
assert isinstance(rdr, RasterDatasetDataSource)
def _fuse_measurement(dest, datasets, geobox, measurement,
skip_broken_datasets=False,
progress_cbk=None):
reproject_and_fuse([new_datasource(BandInfo(dataset, measurement.name)) for dataset in datasets],
dest,
geobox,
dest.dtype.type(measurement.nodata),
resampling=measurement.get('resampling_method', 'nearest'),
fuse_func=measurement.get('fuser', None),
skip_broken_datasets=skip_broken_datasets,
progress_cbk=progress_cbk)
def test_multiband_support_in_datasetsource(example_gdal_path):
defn = {
"id": '12345678123456781234567812345678',
"format": {
"name": "GeoTiff"
},
"image": {
"bands": {
'green': {
'type': 'reflective',
'cell_size': 25.0,
'path': example_gdal_path,
'label': 'Coastal Aerosol',
'number': '1',
},
}
}
}
# Without new band attribute, default to band number 1
d = Dataset(_EXAMPLE_DATASET_TYPE, defn, uris=['file:///tmp'])
ds = RasterDatasetDataSource(BandInfo(d, 'green'))
bandnum = ds.get_bandnumber(None)
assert bandnum == 1
with ds.open() as foo:
data = foo.read()
assert isinstance(data, np.ndarray)
#############
# With new 'image.bands.[band].band' attribute
band_num = 3
defn['image']['bands']['green']['band'] = band_num
d = Dataset(_EXAMPLE_DATASET_TYPE, defn, uris=['file:///tmp'])
ds = RasterDatasetDataSource(BandInfo(d, 'green'))
assert ds.get_bandnumber(None) == band_num
def fuse_measurement(dest: np.ndarray,
datasets: List[Dataset],
geobox: GeoBox,
measurement: Measurement,
mk_new: Callable[[BandInfo], DataSource],
skip_broken_datasets: bool = False):
reproject_and_fuse([mk_new(BandInfo(dataset, measurement.name)) for dataset in datasets],
dest,
geobox,
dest.dtype.type(measurement.nodata),
resampling=measurement.get('resampling_method', 'nearest'),
fuse_func=measurement.get('fuser', None),
skip_broken_datasets=skip_broken_datasets)
def test_band_info():
bands = [
dict(name=n, dtype='uint8', units='K', nodata=33, path=n + '.tiff')
for n in 'a b c'.split(' ')
]
ds = mk_sample_dataset(bands,
uri='file:///tmp/datataset.yml',
format='GeoTIFF')
binfo = BandInfo(ds, 'b')
assert binfo.name == 'b'
assert binfo.band is None
assert binfo.layer is None
assert binfo.dtype == 'uint8'
assert binfo.transform is None
assert binfo.crs is None
assert binfo.units == 'K'
assert binfo.nodata == 33
assert binfo.uri == 'file:///tmp/b.tiff'
assert binfo.format == ds.format
assert binfo.driver_data is None
assert binfo.uri_scheme == 'file'
with pytest.raises(ValueError):
BandInfo(ds, 'no_such_band')
# Check case where dataset is missing band that is present in the product
del ds.metadata_doc['image']['bands']['c']
with pytest.raises(ValueError):
BandInfo(ds, 'c')
ds.uris = []
with pytest.raises(ValueError):
BandInfo(ds, 'a')
ds.uris = None
with pytest.raises(ValueError):
BandInfo(ds, 'a')
def test_hdf5_lock_release_on_failure():
from datacube.storage._rio import RasterDatasetDataSource, _HDF5_LOCK
from datacube.storage import BandInfo
band = dict(name='xx', layer='xx', dtype='uint8', units='K', nodata=33)
ds = mk_sample_dataset(
[band],
uri='file:///tmp/this_probably_doesnot_exist_37237827513/xx.nc',
format=NetCDF)
src = RasterDatasetDataSource(BandInfo(ds, 'xx'))
with pytest.raises(OSError):
with src.open():
assert False and "Did not expect to get here"
assert not _HDF5_LOCK._is_owned()
def test_new_datasource_s3():
pytest.importorskip('datacube.drivers.s3.storage.s3aio.s3lio')
from datacube.drivers.s3 import driver as s3_driver
from datacube.drivers.s3.datasource import S3DataSource
bands = [dict(name='green',
path='')]
dataset = mk_sample_dataset(bands, s3_driver.PROTOCOL + ':///foo', format=s3_driver.FORMAT)
s3_dataset_fake = S3_dataset(macro_shape=(10, 12), numpy_type='float32')
dataset.s3_metadata = {'green': {'s3_dataset': s3_dataset_fake}}
assert dataset.format == s3_driver.FORMAT
assert dataset.uri_scheme == s3_driver.PROTOCOL
rdr = s3_driver.reader_driver_init().new_datasource(BandInfo(dataset, 'green'))
assert rdr is not None
assert isinstance(rdr, S3DataSource)
def mk_band(
name: str,
base_uri: str,
path: str = '',
format: str = GeoTIFF, # pylint: disable=redefined-builtin
**extras) -> BandInfo:
"""
**extras**:
layer, band, nodata, dtype, units, aliases
"""
band_opts = {
k: extras.pop(k)
for k in 'path layer band nodata dtype units aliases'.split()
if k in extras
}
band = dict(name=name, path=path, **band_opts)
ds = mk_sample_dataset([band], base_uri, format=format, **extras)
return BandInfo(ds, name)
def get_measurements(dataset, band_grids=None):
"""
Extract and return measurement paths in a dictionary:
Returns
{
'measurements':
{
'coastal_aerosol': {
'path': path_name1,
'band': band_number,
'layer': null or 'layer_name'
'grid': 'ir'
},
...
}
}
"""
grids_map = ({
m: grid
for grid in band_grids for m in band_grids[grid] if grid != "default"
} if band_grids else dict())
measurements = dict()
for m in dataset.measurements:
if m not in dataset.type.measurements:
_LOG.warn("Measurement not in product definition (skipping): %s",
m)
continue
band_info = BandInfo(dataset, m)
measurements[m] = {"path": dataset.measurements[m]["path"]}
if band_info.band and band_info.band != 1:
measurements[m]["band"] = band_info.band
if band_info.layer is not None:
measurements[m]["layer"] = band_info.layer
if grids_map.get(m):
measurements[m]["grid"] = grids_map[m]
return {"measurements": measurements}
def ds(uri):
d = Dataset(_EXAMPLE_DATASET_TYPE, defn, uris=[uri])
return RasterDatasetDataSource(BandInfo(d, 'green'))