def test_new_xr_load(data_folder):
base = "file://" + str(data_folder) + "/metadata.yml"
rdr = mk_rio_driver()
assert rdr is not None
_bands = []
def band_info_collector(bands, ctx):
for b in bands:
_bands.append(b)
tee_new_load_context(rdr, band_info_collector)
band_a = dict(name='a', path='test.tif')
band_b = dict(name='b', band=2, path='test.tif')
ds = mk_sample_dataset([band_a, band_b], base)
sources = Datacube.group_datasets([ds], 'time')
im, meta = rio_slurp(str(data_folder) + '/test.tif')
measurements = [ds.type.measurements[n] for n in ('a', 'b')]
xx, _ = xr_load(sources, meta.gbox, measurements, rdr)
assert len(_bands) == 2
assert im[0].shape == xx.a.isel(time=0).shape
assert im[1].shape == xx.b.isel(time=0).shape
np.testing.assert_array_equal(im[0], xx.a.values[0])
np.testing.assert_array_equal(im[1], xx.b.values[0])
def test_cog_file(tmpdir):
pp = Path(str(tmpdir))
xx, ds = gen_test_data(pp)
# write to file
ff = write_cog(xx, pp / "cog.tif")
assert isinstance(ff, Path)
assert ff == pp / "cog.tif"
assert ff.exists()
yy = rio_slurp_xarray(pp / "cog.tif")
np.testing.assert_array_equal(yy.values, xx.values)
assert yy.geobox == xx.geobox
assert yy.nodata == xx.nodata
_write_cog(np.stack([xx.values, xx.values]),
xx.geobox,
pp / "cog-2-bands.tif",
overview_levels=[])
yy, mm = rio_slurp(pp / "cog-2-bands.tif")
assert mm.gbox == xx.geobox
assert yy.shape == (2, *xx.shape)
np.testing.assert_array_equal(yy[0], xx.values)
np.testing.assert_array_equal(yy[1], xx.values)
with pytest.raises(ValueError, match="Need 2d or 3d ndarray on input"):
_write_cog(xx.values.ravel(), xx.geobox, pp / "wontwrite.tif")
def test_read_with_reproject(tmpdir):
from datacube.testutils import mk_test_image
from datacube.testutils.io import write_gtiff
from pathlib import Path
pp = Path(str(tmpdir))
xx = mk_test_image(128, 64, nodata=None)
assert (xx != -999).all()
tile = AlbersGS.tile_geobox((17, -40))[:64, :128]
mm = write_gtiff(pp / 'tst-read-with-reproject-128x64-int16.tif',
xx,
crs=str(tile.crs),
resolution=tile.resolution[::-1],
offset=tile.transform * (0, 0),
nodata=-999)
assert mm.gbox == tile
def _read(gbox,
resampling='nearest',
fallback_nodata=None,
dst_nodata=-999):
with RasterFileDataSource(mm.path, 1,
nodata=fallback_nodata).open() as rdr:
yy = np.full(gbox.shape, dst_nodata, dtype=rdr.dtype)
roi = read_time_slice(rdr, yy, gbox, resampling, dst_nodata)
return yy, roi
gbox = gbx.pad(mm.gbox, 10)
gbox = gbx.zoom_out(gbox, 0.873)
yy, roi = _read(gbox)
assert roi[0].start > 0 and roi[1].start > 0
assert (yy[0] == -999).all()
yy_expect, _ = rio_slurp(mm.path, gbox)
np.testing.assert_array_equal(yy, yy_expect)
gbox = gbx.zoom_out(mm.gbox[3:-3, 10:-10], 2.1)
yy, roi = _read(gbox)
assert roi_shape(roi) == gbox.shape
assert not (yy == -999).any()
gbox = GeoBox.from_geopolygon(mm.gbox.extent.to_crs(epsg3857).buffer(50),
resolution=mm.gbox.resolution)
assert gbox.extent.contains(mm.gbox.extent.to_crs(epsg3857))
assert gbox.crs != mm.gbox.crs
yy, roi = _read(gbox)
assert roi[0].start > 0 and roi[1].start > 0
assert (yy[0] == -999).all()
gbox = gbx.zoom_out(gbox, 4)
yy, roi = _read(gbox, resampling='average')
nvalid = (yy != -999).sum()
nempty = (yy == -999).sum()
assert nvalid > nempty
def test_rio_slurp(tmpdir):
w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7
pp = Path(str(tmpdir))
aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw)
assert aa.shape == (h, w)
assert aa.dtype.name == dtype
assert aa[10, 30] == (30 << 8) | 10
assert aa[10, 11] == nodata
aa0 = aa.copy()
mm0 = write_gtiff(pp / "rio-slurp-aa.tif", aa, nodata=-999, overwrite=True)
mm00 = write_gtiff(pp / "rio-slurp-aa-missing-nodata.tif",
aa,
nodata=None,
overwrite=True)
aa, mm = rio_slurp(mm0.path)
np.testing.assert_array_equal(aa, aa0)
assert mm.gbox == mm0.gbox
assert aa.shape == mm.gbox.shape
aa, mm = rio_slurp(mm0.path, aa0.shape)
np.testing.assert_array_equal(aa, aa0)
assert aa.shape == mm.gbox.shape
assert mm.gbox is mm.src_gbox
aa, mm = rio_slurp(mm0.path, (3, 7))
assert aa.shape == (3, 7)
assert aa.shape == mm.gbox.shape
assert mm.gbox != mm.src_gbox
assert mm.src_gbox == mm0.gbox
assert mm.gbox.extent == mm0.gbox.extent
aa, mm = rio_slurp(mm0.path, aa0.shape)
np.testing.assert_array_equal(aa, aa0)
assert aa.shape == mm.gbox.shape
aa, mm = rio_slurp(mm0.path, mm0.gbox, resampling='nearest')
np.testing.assert_array_equal(aa, aa0)
aa, mm = rio_slurp(mm0.path, gbox=mm0.gbox, dtype='float32')
assert aa.dtype == 'float32'
np.testing.assert_array_equal(aa, aa0.astype('float32'))
aa, mm = rio_slurp(mm0.path, mm0.gbox, dst_nodata=-33)
np.testing.assert_array_equal(aa == -33, aa0 == -999)
aa, mm = rio_slurp(mm00.path, mm00.gbox, dst_nodata=None)
np.testing.assert_array_equal(aa, aa0)
def test_missing_file_handling():
with pytest.raises(IOError):
rio_slurp('no-such-file.tiff')
# by default should catch any exception
with ignore_exceptions_if(True):
rio_slurp('no-such-file.tiff')
# this is equivalent to previous default behaviour, note that missing http
# resources are not OSError
with ignore_exceptions_if(True, (OSError, )):
rio_slurp('no-such-file.tiff')
# check that only requested exceptions are caught
with pytest.raises(IOError):
with ignore_exceptions_if(True, (ValueError, ArithmeticError)):
rio_slurp('no-such-file.tiff')
def test_cog_file(tmpdir, opts):
pp = Path(str(tmpdir))
xx, ds = gen_test_data(pp)
# write to file
ff = write_cog(xx, pp / "cog.tif", **opts)
assert isinstance(ff, Path)
assert ff == pp / "cog.tif"
assert ff.exists()
yy = rio_slurp_xarray(pp / "cog.tif")
np.testing.assert_array_equal(yy.values, xx.values)
assert yy.geobox == xx.geobox
assert yy.nodata == xx.nodata
_write_cog(np.stack([xx.values, xx.values]),
xx.geobox,
pp / "cog-2-bands.tif",
overview_levels=[],
**opts)
yy, mm = rio_slurp(pp / "cog-2-bands.tif")
assert mm.gbox == xx.geobox
assert yy.shape == (2, *xx.shape)
np.testing.assert_array_equal(yy[0], xx.values)
np.testing.assert_array_equal(yy[1], xx.values)
with pytest.raises(ValueError, match="Need 2d or 3d ndarray on input"):
_write_cog(xx.values.ravel(), xx.geobox, pp / "wontwrite.tif")
# sizes that are not multiples of 16
# also check that supplying `nodata=` doesn't break things
xx_odd = xx[:23, :63]
ff = write_cog(xx_odd,
pp / "cog_odd.tif",
nodata=xx_odd.attrs["nodata"],
**opts)
assert isinstance(ff, Path)
assert ff == pp / "cog_odd.tif"
assert ff.exists()
yy = rio_slurp_xarray(pp / "cog_odd.tif")
np.testing.assert_array_equal(yy.values, xx_odd.values)
assert yy.geobox == xx_odd.geobox
assert yy.nodata == xx_odd.nodata
with pytest.warns(UserWarning):
write_cog(xx, pp / "cog_badblocksize.tif", blocksize=50)
def test_rio_slurp_with_gbox(tmpdir):
w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7
pp = Path(str(tmpdir))
aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw)
assert aa.dtype.name == dtype
assert aa[10, 30] == (30 << 8) | 10
assert aa[10, 11] == nodata
aa = np.stack([aa, aa[::-1, ::-1]])
assert aa.shape == (2, h, w)
aa0 = aa.copy()
mm = write_gtiff(pp / "rio-slurp-aa.tif", aa, nodata=-999, overwrite=True)
assert mm.count == 2
aa, mm = rio_slurp(mm.path, mm.gbox)
assert aa.shape == aa0.shape
np.testing.assert_array_equal(aa, aa0)
def test_testutils_gtif(tmpdir):
from datacube.testutils import mk_test_image
from datacube.testutils.io import write_gtiff, rio_slurp
w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7
aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw)
bb = mk_test_image(w, h, dtype, nodata=None)
assert aa.shape == (h, w)
assert aa.dtype.name == dtype
assert aa[10, 30] == (30 << 8) | 10
assert aa[10, 11] == nodata
assert bb[10, 11] == (11 << 8) | 10
aa5 = np.stack((aa, ) * 5)
fname = pathlib.Path(str(tmpdir / "aa.tiff"))
fname5 = pathlib.Path(str(tmpdir / "aa5.tiff"))
aa_meta = write_gtiff(fname,
aa,
nodata=nodata,
blocksize=128,
resolution=(100, -100),
offset=(12300, 11100),
overwrite=True)
aa5_meta = write_gtiff(str(fname5),
aa5,
nodata=nodata,
resolution=(100, -100),
offset=(12300, 11100),
overwrite=True)
assert fname.exists()
assert fname5.exists()
assert aa_meta.gbox.shape == (h, w)
assert aa_meta.path is fname
aa_, aa_meta_ = rio_slurp(fname)
aa5_, aa5_meta_ = rio_slurp(fname5)
assert aa_meta_.path is fname
(sx, _, tx, _, sy, ty, *_) = aa5_meta_.transform
assert (tx, ty) == (12300, 11100)
assert (sx, sy) == (100, -100)
np.testing.assert_array_equal(aa, aa_)
np.testing.assert_array_equal(aa5, aa5_)
assert aa_meta_.transform == aa_meta.transform
assert aa5_meta_.transform == aa5_meta.transform
# check that overwrite is off by default
with pytest.raises(IOError):
write_gtiff(fname, aa, nodata=nodata, blocksize=128)
# check that overwrite re-writes file
write_gtiff(fname,
bb[:32, :32],
gbox=aa_meta.gbox[:32, :32],
overwrite=True)
bb_, mm = rio_slurp(fname, (32, 32))
np.testing.assert_array_equal(bb[:32, :32], bb_)
assert mm.gbox == aa_meta.gbox[:32, :32]
with pytest.raises(ValueError):
write_gtiff(fname, np.zeros((3, 4, 5, 6)))
