def check_legacy_open(index):
from datacube.api.core import Datacube
dc = Datacube(index=index)
data_array = dc.load(product='ls5_nbar_albers',
measurements=['blue'],
time='1992-03-23T23:14:25.500000',
use_threads=True)
assert data_array['blue'].shape[0] == 1
assert (data_array.blue != -999).any()
# force fusing load by duplicating dataset
dss = dc.find_datasets(product='ls5_nbar_albers',
time='1992-03-23T23:14:25.500000')
assert len(dss) == 1
dss = dss*2
sources = dc.group_datasets(dss, query_group_by('time'))
gbox = data_array.geobox
mm = [dss[0].type.measurements['blue']]
xx = dc.load_data(sources, gbox, mm)
assert (xx == data_array).all()
with rasterio.Env():
xx_lazy = dc.load_data(sources, gbox, mm, dask_chunks={'time': 1})
assert xx_lazy['blue'].data.dask
assert xx_lazy.blue[0, :, :].equals(xx.blue[0, :, :])
def check_load_via_dss(index):
dc = Datacube(index=index)
dss = dc.find_datasets(product='ls5_nbar_albers')
assert len(dss) > 0
xx1 = dc.load(product='ls5_nbar_albers', measurements=['blue'])
xx2 = dc.load(datasets=dss, measurements=['blue'])
assert xx1.blue.shape
assert (xx1.blue != -999).any()
assert (xx1.blue == xx2.blue).all()
xx2 = dc.load(datasets=iter(dss), measurements=['blue'])
assert xx1.blue.shape
assert (xx1.blue != -999).any()
assert (xx1.blue == xx2.blue).all()
with pytest.raises(ValueError):
dc.load(measurements=['blue'])
def test_end_to_end(clirunner, index, testdata_dir, ingest_configs,
datacube_env_name):
"""
Loads two dataset configurations, then ingests a sample Landsat 5 scene
One dataset configuration specifies Australian Albers Equal Area Projection,
the other is simply latitude/longitude.
The input dataset should be recorded in the index, and two sets of storage units
should be created on disk and recorded in the index.
"""
lbg_nbar = testdata_dir / 'lbg' / LBG_NBAR
lbg_pq = testdata_dir / 'lbg' / LBG_PQ
ls5_nbar_albers_ingest_config = testdata_dir / ingest_configs[
'ls5_nbar_albers']
ls5_pq_albers_ingest_config = testdata_dir / ingest_configs['ls5_pq_albers']
# Add the LS5 Dataset Types
clirunner(['-v', 'product', 'add', str(LS5_DATASET_TYPES)])
# Index the Datasets
# - do test run first to increase test coverage
clirunner(
['-v', 'dataset', 'add', '--dry-run',
str(lbg_nbar),
str(lbg_pq)])
# - do actual indexing
clirunner(['-v', 'dataset', 'add', str(lbg_nbar), str(lbg_pq)])
# - this will be no-op but with ignore lineage
clirunner([
'-v', 'dataset', 'add', '--confirm-ignore-lineage',
str(lbg_nbar),
str(lbg_pq)
])
# Test no-op update
for policy in ['archive', 'forget', 'keep']:
clirunner([
'-v', 'dataset', 'update', '--dry-run', '--location-policy',
policy,
str(lbg_nbar),
str(lbg_pq)
])
# Test no changes needed update
clirunner([
'-v', 'dataset', 'update', '--location-policy', policy,
str(lbg_nbar),
str(lbg_pq)
])
# TODO: test location update
# 1. Make a copy of a file
# 2. Call dataset update with archive/forget
# 3. Check location
# Ingest NBAR
clirunner(['-v', 'ingest', '-c', str(ls5_nbar_albers_ingest_config)])
# Ingest PQ
clirunner(['-v', 'ingest', '-c', str(ls5_pq_albers_ingest_config)])
dc = Datacube(index=index)
assert isinstance(str(dc), str)
assert isinstance(repr(dc), str)
with pytest.raises(ValueError):
dc.find_datasets(time='2019') # no product supplied, raises exception
check_open_with_dc(index)
check_open_with_grid_workflow(index)
check_load_via_dss(index)