def test_native_load(tmpdir):
from datacube.testutils.io import native_load, native_geobox
tmpdir = Path(str(tmpdir))
spatial = dict(resolution=(15, -15),
offset=(11230, 1381110),)
nodata = -999
aa = mk_test_image(96, 64, 'int16', nodata=nodata)
cc = mk_test_image(32, 16, 'int16', nodata=nodata)
bands = [SimpleNamespace(name=name, values=aa, nodata=nodata)
for name in ['aa', 'bb']]
bands.append(SimpleNamespace(name='cc', values=cc, nodata=nodata))
ds, gbox = gen_tiff_dataset(bands[:2],
tmpdir,
prefix='ds1-',
timestamp='2018-07-19',
**spatial)
assert set(get_raster_info(ds)) == set(ds.measurements)
xx = native_load(ds)
assert xx.geobox == gbox
np.testing.assert_array_equal(aa, xx.isel(time=0).aa.values)
np.testing.assert_array_equal(aa, xx.isel(time=0).bb.values)
ds, gbox_cc = gen_tiff_dataset(bands,
tmpdir,
prefix='ds2-',
timestamp='2018-07-19',
**spatial)
# cc is different size from aa,bb
with pytest.raises(ValueError):
xx = native_load(ds)
# cc is different size from aa,bb
with pytest.raises(ValueError):
xx = native_geobox(ds)
# aa and bb are the same
assert native_geobox(ds, ['aa', 'bb']) == gbox
xx = native_load(ds, ['aa', 'bb'])
assert xx.geobox == gbox
np.testing.assert_array_equal(aa, xx.isel(time=0).aa.values)
np.testing.assert_array_equal(aa, xx.isel(time=0).bb.values)
# cc will be reprojected
assert native_geobox(ds, basis='aa') == gbox
xx = native_load(ds, basis='aa')
assert xx.geobox == gbox
np.testing.assert_array_equal(aa, xx.isel(time=0).aa.values)
np.testing.assert_array_equal(aa, xx.isel(time=0).bb.values)
# cc is compatible with self
xx = native_load(ds, ['cc'])
assert xx.geobox == gbox_cc
np.testing.assert_array_equal(cc, xx.isel(time=0).cc.values)
def test_read_with_reproject(tmpdir):
from datacube.testutils import mk_test_image
from datacube.testutils.io import write_gtiff
from pathlib import Path
pp = Path(str(tmpdir))
xx = mk_test_image(128, 64, nodata=None)
assert (xx != -999).all()
tile = AlbersGS.tile_geobox((17, -40))[:64, :128]
mm = write_gtiff(pp / 'tst-read-with-reproject-128x64-int16.tif',
xx,
crs=str(tile.crs),
resolution=tile.resolution[::-1],
offset=tile.transform * (0, 0),
nodata=-999)
assert mm.gbox == tile
def _read(gbox,
resampling='nearest',
fallback_nodata=None,
dst_nodata=-999):
with RasterFileDataSource(mm.path, 1,
nodata=fallback_nodata).open() as rdr:
yy = np.full(gbox.shape, dst_nodata, dtype=rdr.dtype)
roi = read_time_slice(rdr, yy, gbox, resampling, dst_nodata)
return yy, roi
gbox = gbx.pad(mm.gbox, 10)
gbox = gbx.zoom_out(gbox, 0.873)
yy, roi = _read(gbox)
assert roi[0].start > 0 and roi[1].start > 0
assert (yy[0] == -999).all()
yy_expect, _ = rio_slurp(mm.path, gbox)
np.testing.assert_array_equal(yy, yy_expect)
gbox = gbx.zoom_out(mm.gbox[3:-3, 10:-10], 2.1)
yy, roi = _read(gbox)
assert roi_shape(roi) == gbox.shape
assert not (yy == -999).any()
gbox = GeoBox.from_geopolygon(mm.gbox.extent.to_crs(epsg3857).buffer(50),
resolution=mm.gbox.resolution)
assert gbox.extent.contains(mm.gbox.extent.to_crs(epsg3857))
assert gbox.crs != mm.gbox.crs
yy, roi = _read(gbox)
assert roi[0].start > 0 and roi[1].start > 0
assert (yy[0] == -999).all()
gbox = gbx.zoom_out(gbox, 4)
yy, roi = _read(gbox, resampling='average')
nvalid = (yy != -999).sum()
nempty = (yy == -999).sum()
assert nvalid > nempty
def test_load_data(tmpdir):
tmpdir = Path(str(tmpdir))
group_by = query_group_by('time')
spatial = dict(resolution=(15, -15),
offset=(11230, 1381110),)
nodata = -999
aa = mk_test_image(96, 64, 'int16', nodata=nodata)
ds, gbox = gen_tiff_dataset([SimpleNamespace(name='aa', values=aa, nodata=nodata)],
tmpdir,
prefix='ds1-',
timestamp='2018-07-19',
**spatial)
assert ds.time is not None
ds2, _ = gen_tiff_dataset([SimpleNamespace(name='aa', values=aa, nodata=nodata)],
tmpdir,
prefix='ds2-',
timestamp='2018-07-19',
**spatial)
assert ds.time is not None
assert ds.time == ds2.time
sources = Datacube.group_datasets([ds], 'time')
sources2 = Datacube.group_datasets([ds, ds2], group_by)
mm = ['aa']
mm = [ds.type.measurements[k] for k in mm]
ds_data = Datacube.load_data(sources, gbox, mm)
assert ds_data.aa.nodata == nodata
np.testing.assert_array_equal(aa, ds_data.aa.values[0])
custom_fuser_call_count = 0
def custom_fuser(dest, delta):
nonlocal custom_fuser_call_count
custom_fuser_call_count += 1
dest[:] += delta
progress_call_data = []
def progress_cbk(n, nt):
progress_call_data.append((n, nt))
ds_data = Datacube.load_data(sources2, gbox, mm, fuse_func=custom_fuser,
progress_cbk=progress_cbk)
assert ds_data.aa.nodata == nodata
assert custom_fuser_call_count > 0
np.testing.assert_array_equal(nodata + aa + aa, ds_data.aa.values[0])
assert progress_call_data == [(1, 2), (2, 2)]
def test_rio_slurp(tmpdir):
w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7
pp = Path(str(tmpdir))
aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw)
assert aa.shape == (h, w)
assert aa.dtype.name == dtype
assert aa[10, 30] == (30 << 8) | 10
assert aa[10, 11] == nodata
aa0 = aa.copy()
mm0 = write_gtiff(pp / "rio-slurp-aa.tif", aa, nodata=-999, overwrite=True)
mm00 = write_gtiff(pp / "rio-slurp-aa-missing-nodata.tif",
aa,
nodata=None,
overwrite=True)
aa, mm = rio_slurp(mm0.path)
np.testing.assert_array_equal(aa, aa0)
assert mm.gbox == mm0.gbox
assert aa.shape == mm.gbox.shape
aa, mm = rio_slurp(mm0.path, aa0.shape)
np.testing.assert_array_equal(aa, aa0)
assert aa.shape == mm.gbox.shape
assert mm.gbox is mm.src_gbox
aa, mm = rio_slurp(mm0.path, (3, 7))
assert aa.shape == (3, 7)
assert aa.shape == mm.gbox.shape
assert mm.gbox != mm.src_gbox
assert mm.src_gbox == mm0.gbox
assert mm.gbox.extent == mm0.gbox.extent
aa, mm = rio_slurp(mm0.path, aa0.shape)
np.testing.assert_array_equal(aa, aa0)
assert aa.shape == mm.gbox.shape
aa, mm = rio_slurp(mm0.path, mm0.gbox, resampling='nearest')
np.testing.assert_array_equal(aa, aa0)
aa, mm = rio_slurp(mm0.path, gbox=mm0.gbox, dtype='float32')
assert aa.dtype == 'float32'
np.testing.assert_array_equal(aa, aa0.astype('float32'))
aa, mm = rio_slurp(mm0.path, mm0.gbox, dst_nodata=-33)
np.testing.assert_array_equal(aa == -33, aa0 == -999)
aa, mm = rio_slurp(mm00.path, mm00.gbox, dst_nodata=None)
np.testing.assert_array_equal(aa, aa0)
def test_testutils_testimage():
from datacube.testutils import mk_test_image, split_test_image
for dtype in ('uint16', 'uint32', 'int32', 'float32'):
aa = mk_test_image(128, 64, dtype=dtype, nodata=None)
assert aa.shape == (64, 128)
assert aa.dtype == dtype
xx, yy = split_test_image(aa)
assert (xx[:, 33] == 33).all()
assert (xx[:, 127] == 127).all()
assert (yy[23, :] == 23).all()
assert (yy[63, :] == 63).all()
def gen_test_data(prefix, dask=False):
w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7
aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw)
ds, gbox = gen_tiff_dataset(
SimpleNamespace(name='aa', values=aa, nodata=nodata), prefix)
extras = {}
if dask:
extras.update(dask_chunks={'time': 1})
xx = native_load(ds, **extras)
return xx.aa.isel(time=0), ds
def gen_test_data(prefix, dask=False, shape=None):
w, h, dtype, nodata, ndw = 96, 64, "int16", -999, 7
if shape is not None:
h, w = shape
aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw)
ds, gbox = gen_tiff_dataset(
SimpleNamespace(name="aa", values=aa, nodata=nodata), prefix)
extras = {}
if dask:
extras.update(dask_chunks={"time": 1})
xx = native_load(ds, **extras)
return xx.aa.isel(time=0), ds
def test_rio_slurp_with_gbox(tmpdir):
w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7
pp = Path(str(tmpdir))
aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw)
assert aa.dtype.name == dtype
assert aa[10, 30] == (30 << 8) | 10
assert aa[10, 11] == nodata
aa = np.stack([aa, aa[::-1, ::-1]])
assert aa.shape == (2, h, w)
aa0 = aa.copy()
mm = write_gtiff(pp / "rio-slurp-aa.tif", aa, nodata=-999, overwrite=True)
assert mm.count == 2
aa, mm = rio_slurp(mm.path, mm.gbox)
assert aa.shape == aa0.shape
np.testing.assert_array_equal(aa, aa0)
def test_testutils_gtif(tmpdir):
from datacube.testutils import mk_test_image
from datacube.testutils.io import write_gtiff, rio_slurp
w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7
aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw)
bb = mk_test_image(w, h, dtype, nodata=None)
assert aa.shape == (h, w)
assert aa.dtype.name == dtype
assert aa[10, 30] == (30 << 8) | 10
assert aa[10, 11] == nodata
assert bb[10, 11] == (11 << 8) | 10
aa5 = np.stack((aa, ) * 5)
fname = pathlib.Path(str(tmpdir / "aa.tiff"))
fname5 = pathlib.Path(str(tmpdir / "aa5.tiff"))
aa_meta = write_gtiff(fname,
aa,
nodata=nodata,
blocksize=128,
resolution=(100, -100),
offset=(12300, 11100),
overwrite=True)
aa5_meta = write_gtiff(str(fname5),
aa5,
nodata=nodata,
resolution=(100, -100),
offset=(12300, 11100),
overwrite=True)
assert fname.exists()
assert fname5.exists()
assert aa_meta.gbox.shape == (h, w)
assert aa_meta.path is fname
aa_, aa_meta_ = rio_slurp(fname)
aa5_, aa5_meta_ = rio_slurp(fname5)
assert aa_meta_.path is fname
(sx, _, tx, _, sy, ty, *_) = aa5_meta_.transform
assert (tx, ty) == (12300, 11100)
assert (sx, sy) == (100, -100)
np.testing.assert_array_equal(aa, aa_)
np.testing.assert_array_equal(aa5, aa5_)
assert aa_meta_.transform == aa_meta.transform
assert aa5_meta_.transform == aa5_meta.transform
# check that overwrite is off by default
with pytest.raises(IOError):
write_gtiff(fname, aa, nodata=nodata, blocksize=128)
# check that overwrite re-writes file
write_gtiff(fname,
bb[:32, :32],
gbox=aa_meta.gbox[:32, :32],
overwrite=True)
bb_, mm = rio_slurp(fname, (32, 32))
np.testing.assert_array_equal(bb[:32, :32], bb_)
assert mm.gbox == aa_meta.gbox[:32, :32]
with pytest.raises(ValueError):
write_gtiff(fname, np.zeros((3, 4, 5, 6)))
def test_read_paste(tmpdir):
from datacube.testutils import mk_test_image
from datacube.testutils.io import write_gtiff
from pathlib import Path
pp = Path(str(tmpdir))
xx = mk_test_image(128, 64, nodata=None)
assert (xx != -999).all()
mm = write_gtiff(pp / 'tst-read-paste-128x64-int16.tif', xx, nodata=None)
def _read(gbox,
resampling='nearest',
fallback_nodata=-999,
dst_nodata=-999,
check_paste=False):
with RasterFileDataSource(mm.path, 1,
nodata=fallback_nodata).open() as rdr:
if check_paste:
# check that we are using paste
paste_ok, reason = can_paste(
compute_reproject_roi(rdr_geobox(rdr), gbox))
assert paste_ok is True, reason
yy = np.full(gbox.shape, dst_nodata, dtype=rdr.dtype)
roi = read_time_slice(rdr, yy, gbox, resampling, dst_nodata)
return yy, roi
# read native whole
yy, roi = _read(mm.gbox)
np.testing.assert_array_equal(xx, yy)
assert roi == np.s_[0:64, 0:128]
# read native whole, no nodata case
yy, roi = _read(mm.gbox, fallback_nodata=None)
np.testing.assert_array_equal(xx, yy)
assert roi == np.s_[0:64, 0:128]
# read native whole, ignoring small sub-pixel translation
yy, roi = _read(gbx.translate_pix(mm.gbox, 0.3, -0.4), fallback_nodata=-33)
np.testing.assert_array_equal(xx, yy)
assert roi == np.s_[0:64, 0:128]
# no overlap between src and dst
yy, roi = _read(gbx.translate_pix(mm.gbox, 10000, -10000))
assert roi_is_empty(roi)
# read with Y flipped
yy, roi = _read(gbx.flipy(mm.gbox))
np.testing.assert_array_equal(xx[::-1, :], yy)
assert roi == np.s_[0:64, 0:128]
# read with X flipped
yy, roi = _read(gbx.flipx(mm.gbox))
np.testing.assert_array_equal(xx[:, ::-1], yy)
assert roi == np.s_[0:64, 0:128]
# read with X and Y flipped
yy, roi = _read(gbx.flipy(gbx.flipx(mm.gbox)))
assert roi == np.s_[0:64, 0:128]
np.testing.assert_array_equal(xx[::-1, ::-1], yy[roi])
# dst is fully inside src
sroi = np.s_[10:19, 31:47]
yy, roi = _read(mm.gbox[sroi])
np.testing.assert_array_equal(xx[sroi], yy[roi])
# partial overlap
yy, roi = _read(gbx.translate_pix(mm.gbox, -3, -10))
assert roi == np.s_[10:64, 3:128]
np.testing.assert_array_equal(xx[:-10, :-3], yy[roi])
assert (yy[:10, :] == -999).all()
assert (yy[:, :3] == -999).all()
# scaling paste
yy, roi = _read(gbx.zoom_out(mm.gbox, 2), check_paste=True)
assert roi == np.s_[0:32, 0:64]
np.testing.assert_array_equal(xx[1::2, 1::2], yy)
def test_load_data_cbk(tmpdir):
from datacube.api import TerminateCurrentLoad
tmpdir = Path(str(tmpdir))
spatial = dict(
resolution=(15, -15),
offset=(11230, 1381110),
)
nodata = -999
aa = mk_test_image(96, 64, 'int16', nodata=nodata)
bands = [
SimpleNamespace(name=name, values=aa, nodata=nodata)
for name in ['aa', 'bb']
]
ds, gbox = gen_tiff_dataset(bands,
tmpdir,
prefix='ds1-',
timestamp='2018-07-19',
**spatial)
assert ds.time is not None
ds2, _ = gen_tiff_dataset(bands,
tmpdir,
prefix='ds2-',
timestamp='2018-07-19',
**spatial)
assert ds.time is not None
assert ds.time == ds2.time
sources = Datacube.group_datasets([ds, ds2], 'time')
progress_call_data = []
def progress_cbk(n, nt):
progress_call_data.append((n, nt))
ds_data = Datacube.load_data(sources,
gbox,
ds.type.measurements,
progress_cbk=progress_cbk)
assert progress_call_data == [(1, 4), (2, 4), (3, 4), (4, 4)]
np.testing.assert_array_equal(aa, ds_data.aa.values[0])
np.testing.assert_array_equal(aa, ds_data.bb.values[0])
def progress_cbk_fail_early(n, nt):
progress_call_data.append((n, nt))
raise TerminateCurrentLoad()
def progress_cbk_fail_early2(n, nt):
progress_call_data.append((n, nt))
if n > 1:
raise KeyboardInterrupt()
progress_call_data = []
ds_data = Datacube.load_data(sources,
gbox,
ds.type.measurements,
progress_cbk=progress_cbk_fail_early)
assert progress_call_data == [(1, 4)]
assert ds_data.dc_partial_load is True
np.testing.assert_array_equal(aa, ds_data.aa.values[0])
np.testing.assert_array_equal(nodata, ds_data.bb.values[0])
progress_call_data = []
ds_data = Datacube.load_data(sources,
gbox,
ds.type.measurements,
progress_cbk=progress_cbk_fail_early2)
assert ds_data.dc_partial_load is True
assert progress_call_data == [(1, 4), (2, 4)]
