def test_new_xr_load(data_folder): base = "file://" + str(data_folder) + "/metadata.yml" rdr = mk_rio_driver() assert rdr is not None _bands = [] def band_info_collector(bands, ctx): for b in bands: _bands.append(b) tee_new_load_context(rdr, band_info_collector) band_a = dict(name='a', path='test.tif') band_b = dict(name='b', band=2, path='test.tif') ds = mk_sample_dataset([band_a, band_b], base) sources = Datacube.group_datasets([ds], 'time') im, meta = rio_slurp(str(data_folder) + '/test.tif') measurements = [ds.type.measurements[n] for n in ('a', 'b')] xx, _ = xr_load(sources, meta.gbox, measurements, rdr) assert len(_bands) == 2 assert im[0].shape == xx.a.isel(time=0).shape assert im[1].shape == xx.b.isel(time=0).shape np.testing.assert_array_equal(im[0], xx.a.values[0]) np.testing.assert_array_equal(im[1], xx.b.values[0])
def test_cog_file(tmpdir): pp = Path(str(tmpdir)) xx, ds = gen_test_data(pp) # write to file ff = write_cog(xx, pp / "cog.tif") assert isinstance(ff, Path) assert ff == pp / "cog.tif" assert ff.exists() yy = rio_slurp_xarray(pp / "cog.tif") np.testing.assert_array_equal(yy.values, xx.values) assert yy.geobox == xx.geobox assert yy.nodata == xx.nodata _write_cog(np.stack([xx.values, xx.values]), xx.geobox, pp / "cog-2-bands.tif", overview_levels=[]) yy, mm = rio_slurp(pp / "cog-2-bands.tif") assert mm.gbox == xx.geobox assert yy.shape == (2, *xx.shape) np.testing.assert_array_equal(yy[0], xx.values) np.testing.assert_array_equal(yy[1], xx.values) with pytest.raises(ValueError, match="Need 2d or 3d ndarray on input"): _write_cog(xx.values.ravel(), xx.geobox, pp / "wontwrite.tif")
def test_read_with_reproject(tmpdir): from datacube.testutils import mk_test_image from datacube.testutils.io import write_gtiff from pathlib import Path pp = Path(str(tmpdir)) xx = mk_test_image(128, 64, nodata=None) assert (xx != -999).all() tile = AlbersGS.tile_geobox((17, -40))[:64, :128] mm = write_gtiff(pp / 'tst-read-with-reproject-128x64-int16.tif', xx, crs=str(tile.crs), resolution=tile.resolution[::-1], offset=tile.transform * (0, 0), nodata=-999) assert mm.gbox == tile def _read(gbox, resampling='nearest', fallback_nodata=None, dst_nodata=-999): with RasterFileDataSource(mm.path, 1, nodata=fallback_nodata).open() as rdr: yy = np.full(gbox.shape, dst_nodata, dtype=rdr.dtype) roi = read_time_slice(rdr, yy, gbox, resampling, dst_nodata) return yy, roi gbox = gbx.pad(mm.gbox, 10) gbox = gbx.zoom_out(gbox, 0.873) yy, roi = _read(gbox) assert roi[0].start > 0 and roi[1].start > 0 assert (yy[0] == -999).all() yy_expect, _ = rio_slurp(mm.path, gbox) np.testing.assert_array_equal(yy, yy_expect) gbox = gbx.zoom_out(mm.gbox[3:-3, 10:-10], 2.1) yy, roi = _read(gbox) assert roi_shape(roi) == gbox.shape assert not (yy == -999).any() gbox = GeoBox.from_geopolygon(mm.gbox.extent.to_crs(epsg3857).buffer(50), resolution=mm.gbox.resolution) assert gbox.extent.contains(mm.gbox.extent.to_crs(epsg3857)) assert gbox.crs != mm.gbox.crs yy, roi = _read(gbox) assert roi[0].start > 0 and roi[1].start > 0 assert (yy[0] == -999).all() gbox = gbx.zoom_out(gbox, 4) yy, roi = _read(gbox, resampling='average') nvalid = (yy != -999).sum() nempty = (yy == -999).sum() assert nvalid > nempty
def test_rio_slurp(tmpdir): w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7 pp = Path(str(tmpdir)) aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw) assert aa.shape == (h, w) assert aa.dtype.name == dtype assert aa[10, 30] == (30 << 8) | 10 assert aa[10, 11] == nodata aa0 = aa.copy() mm0 = write_gtiff(pp / "rio-slurp-aa.tif", aa, nodata=-999, overwrite=True) mm00 = write_gtiff(pp / "rio-slurp-aa-missing-nodata.tif", aa, nodata=None, overwrite=True) aa, mm = rio_slurp(mm0.path) np.testing.assert_array_equal(aa, aa0) assert mm.gbox == mm0.gbox assert aa.shape == mm.gbox.shape aa, mm = rio_slurp(mm0.path, aa0.shape) np.testing.assert_array_equal(aa, aa0) assert aa.shape == mm.gbox.shape assert mm.gbox is mm.src_gbox aa, mm = rio_slurp(mm0.path, (3, 7)) assert aa.shape == (3, 7) assert aa.shape == mm.gbox.shape assert mm.gbox != mm.src_gbox assert mm.src_gbox == mm0.gbox assert mm.gbox.extent == mm0.gbox.extent aa, mm = rio_slurp(mm0.path, aa0.shape) np.testing.assert_array_equal(aa, aa0) assert aa.shape == mm.gbox.shape aa, mm = rio_slurp(mm0.path, mm0.gbox, resampling='nearest') np.testing.assert_array_equal(aa, aa0) aa, mm = rio_slurp(mm0.path, gbox=mm0.gbox, dtype='float32') assert aa.dtype == 'float32' np.testing.assert_array_equal(aa, aa0.astype('float32')) aa, mm = rio_slurp(mm0.path, mm0.gbox, dst_nodata=-33) np.testing.assert_array_equal(aa == -33, aa0 == -999) aa, mm = rio_slurp(mm00.path, mm00.gbox, dst_nodata=None) np.testing.assert_array_equal(aa, aa0)
def test_missing_file_handling(): with pytest.raises(IOError): rio_slurp('no-such-file.tiff') # by default should catch any exception with ignore_exceptions_if(True): rio_slurp('no-such-file.tiff') # this is equivalent to previous default behaviour, note that missing http # resources are not OSError with ignore_exceptions_if(True, (OSError, )): rio_slurp('no-such-file.tiff') # check that only requested exceptions are caught with pytest.raises(IOError): with ignore_exceptions_if(True, (ValueError, ArithmeticError)): rio_slurp('no-such-file.tiff')
def test_cog_file(tmpdir, opts): pp = Path(str(tmpdir)) xx, ds = gen_test_data(pp) # write to file ff = write_cog(xx, pp / "cog.tif", **opts) assert isinstance(ff, Path) assert ff == pp / "cog.tif" assert ff.exists() yy = rio_slurp_xarray(pp / "cog.tif") np.testing.assert_array_equal(yy.values, xx.values) assert yy.geobox == xx.geobox assert yy.nodata == xx.nodata _write_cog(np.stack([xx.values, xx.values]), xx.geobox, pp / "cog-2-bands.tif", overview_levels=[], **opts) yy, mm = rio_slurp(pp / "cog-2-bands.tif") assert mm.gbox == xx.geobox assert yy.shape == (2, *xx.shape) np.testing.assert_array_equal(yy[0], xx.values) np.testing.assert_array_equal(yy[1], xx.values) with pytest.raises(ValueError, match="Need 2d or 3d ndarray on input"): _write_cog(xx.values.ravel(), xx.geobox, pp / "wontwrite.tif") # sizes that are not multiples of 16 # also check that supplying `nodata=` doesn't break things xx_odd = xx[:23, :63] ff = write_cog(xx_odd, pp / "cog_odd.tif", nodata=xx_odd.attrs["nodata"], **opts) assert isinstance(ff, Path) assert ff == pp / "cog_odd.tif" assert ff.exists() yy = rio_slurp_xarray(pp / "cog_odd.tif") np.testing.assert_array_equal(yy.values, xx_odd.values) assert yy.geobox == xx_odd.geobox assert yy.nodata == xx_odd.nodata with pytest.warns(UserWarning): write_cog(xx, pp / "cog_badblocksize.tif", blocksize=50)
def test_rio_slurp_with_gbox(tmpdir): w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7 pp = Path(str(tmpdir)) aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw) assert aa.dtype.name == dtype assert aa[10, 30] == (30 << 8) | 10 assert aa[10, 11] == nodata aa = np.stack([aa, aa[::-1, ::-1]]) assert aa.shape == (2, h, w) aa0 = aa.copy() mm = write_gtiff(pp / "rio-slurp-aa.tif", aa, nodata=-999, overwrite=True) assert mm.count == 2 aa, mm = rio_slurp(mm.path, mm.gbox) assert aa.shape == aa0.shape np.testing.assert_array_equal(aa, aa0)
def test_testutils_gtif(tmpdir): from datacube.testutils import mk_test_image from datacube.testutils.io import write_gtiff, rio_slurp w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7 aa = mk_test_image(w, h, dtype, nodata, nodata_width=ndw) bb = mk_test_image(w, h, dtype, nodata=None) assert aa.shape == (h, w) assert aa.dtype.name == dtype assert aa[10, 30] == (30 << 8) | 10 assert aa[10, 11] == nodata assert bb[10, 11] == (11 << 8) | 10 aa5 = np.stack((aa, ) * 5) fname = pathlib.Path(str(tmpdir / "aa.tiff")) fname5 = pathlib.Path(str(tmpdir / "aa5.tiff")) aa_meta = write_gtiff(fname, aa, nodata=nodata, blocksize=128, resolution=(100, -100), offset=(12300, 11100), overwrite=True) aa5_meta = write_gtiff(str(fname5), aa5, nodata=nodata, resolution=(100, -100), offset=(12300, 11100), overwrite=True) assert fname.exists() assert fname5.exists() assert aa_meta.gbox.shape == (h, w) assert aa_meta.path is fname aa_, aa_meta_ = rio_slurp(fname) aa5_, aa5_meta_ = rio_slurp(fname5) assert aa_meta_.path is fname (sx, _, tx, _, sy, ty, *_) = aa5_meta_.transform assert (tx, ty) == (12300, 11100) assert (sx, sy) == (100, -100) np.testing.assert_array_equal(aa, aa_) np.testing.assert_array_equal(aa5, aa5_) assert aa_meta_.transform == aa_meta.transform assert aa5_meta_.transform == aa5_meta.transform # check that overwrite is off by default with pytest.raises(IOError): write_gtiff(fname, aa, nodata=nodata, blocksize=128) # check that overwrite re-writes file write_gtiff(fname, bb[:32, :32], gbox=aa_meta.gbox[:32, :32], overwrite=True) bb_, mm = rio_slurp(fname, (32, 32)) np.testing.assert_array_equal(bb[:32, :32], bb_) assert mm.gbox == aa_meta.gbox[:32, :32] with pytest.raises(ValueError): write_gtiff(fname, np.zeros((3, 4, 5, 6)))