def reproject_and_fuse(datasources: List[DataSource],
destination: np.ndarray,
dst_gbox: GeoBox,
dst_nodata: Optional[Union[int, float]],
resampling: str = 'nearest',
fuse_func: Optional[FuserFunction] = None,
skip_broken_datasets: bool = False,
progress_cbk: Optional[ProgressFunction] = None):
"""
Reproject and fuse `sources` into a 2D numpy array `destination`.
:param datasources: Data sources to open and read from
:param destination: ndarray of appropriate size to read data into
:param dst_gbox: GeoBox defining destination region
:param skip_broken_datasets: Carry on in the face of adversity and failing reads.
:param progress_cbk: If supplied will be called with 2 integers `Items processed, Total Items`
after reading each file.
"""
# pylint: disable=too-many-locals
from ._read import read_time_slice
assert len(destination.shape) == 2
def copyto_fuser(dest: np.ndarray, src: np.ndarray) -> None:
_default_fuser(dest, src, dst_nodata)
fuse_func = fuse_func or copyto_fuser
destination.fill(dst_nodata)
if len(datasources) == 0:
return destination
elif len(datasources) == 1:
with ignore_exceptions_if(skip_broken_datasets):
with datasources[0].open() as rdr:
read_time_slice(rdr, destination, dst_gbox, resampling,
dst_nodata)
if progress_cbk:
progress_cbk(1, 1)
return destination
else:
# Multiple sources, we need to fuse them together into a single array
buffer_ = np.full(destination.shape,
dst_nodata,
dtype=destination.dtype)
for n_so_far, source in enumerate(datasources, 1):
with ignore_exceptions_if(skip_broken_datasets):
with source.open() as rdr:
roi = read_time_slice(rdr, buffer_, dst_gbox, resampling,
dst_nodata)
if not roi_is_empty(roi):
fuse_func(destination[roi], buffer_[roi])
buffer_[roi] = dst_nodata # clean up for next read
if progress_cbk:
progress_cbk(n_so_far, len(datasources))
return destination
def test_compute_reproject_roi_issue647():
""" In some scenarios non-overlapping geoboxes will result in non-empty
`roi_dst` even though `roi_src` is empty.
Test this case separately.
"""
from datacube.utils.geometry import CRS
src = GeoBox(10980, 10980, Affine(10, 0, 300000, 0, -10, 5900020),
CRS('epsg:32756'))
dst = GeoBox(976, 976, Affine(10, 0, 1730240, 0, -10, -4170240),
CRS('EPSG:3577'))
assert src.extent.overlaps(dst.extent.to_crs(src.crs)) is False
rr = compute_reproject_roi(src, dst)
assert roi_is_empty(rr.roi_src)
assert roi_is_empty(rr.roi_dst)
def reproject_and_fuse(datasources: List[DataSource],
destination: np.ndarray,
dst_gbox: GeoBox,
dst_nodata: Optional[Union[int, float]],
resampling: str = 'nearest',
fuse_func: Optional[FuserFunction] = None,
skip_broken_datasets: bool = False):
"""
Reproject and fuse `sources` into a 2D numpy array `destination`.
:param datasources: Data sources to open and read from
:param destination: ndarray of appropriate size to read data into
:param dst_gbox: GeoBox defining destination region
:param skip_broken_datasets: Carry on in the face of adversity and failing reads.
"""
# pylint: disable=too-many-locals
assert len(destination.shape) == 2
def copyto_fuser(dest: np.ndarray, src: np.ndarray) -> None:
where_nodata = (
dest == dst_nodata) if not np.isnan(dst_nodata) else np.isnan(dest)
np.copyto(dest, src, where=where_nodata)
fuse_func = fuse_func or copyto_fuser
destination.fill(dst_nodata)
if len(datasources) == 0:
return destination
elif len(datasources) == 1:
with ignore_exceptions_if(skip_broken_datasets):
with datasources[0].open() as rdr:
read_time_slice(rdr, destination, dst_gbox, resampling,
dst_nodata)
return destination
else:
# Multiple sources, we need to fuse them together into a single array
buffer_ = np.full(destination.shape,
dst_nodata,
dtype=destination.dtype)
for source in datasources:
with ignore_exceptions_if(skip_broken_datasets):
with source.open() as rdr:
roi = read_time_slice(rdr, buffer_, dst_gbox, resampling,
dst_nodata)
if not roi_is_empty(roi):
fuse_func(destination[roi], buffer_[roi])
buffer_[roi] = dst_nodata # clean up for next read
return destination
def test_read_paste(tmpdir):
from datacube.testutils import mk_test_image
from datacube.testutils.io import write_gtiff
from pathlib import Path
pp = Path(str(tmpdir))
xx = mk_test_image(128, 64, nodata=None)
assert (xx != -999).all()
mm = write_gtiff(pp / 'tst-read-paste-128x64-int16.tif', xx, nodata=None)
def _read(gbox,
resampling='nearest',
fallback_nodata=-999,
dst_nodata=-999,
check_paste=False):
with RasterFileDataSource(mm.path, 1,
nodata=fallback_nodata).open() as rdr:
if check_paste:
# check that we are using paste
paste_ok, reason = can_paste(
compute_reproject_roi(rdr_geobox(rdr), gbox))
assert paste_ok is True, reason
yy = np.full(gbox.shape, dst_nodata, dtype=rdr.dtype)
roi = read_time_slice(rdr, yy, gbox, resampling, dst_nodata)
return yy, roi
# read native whole
yy, roi = _read(mm.gbox)
np.testing.assert_array_equal(xx, yy)
assert roi == np.s_[0:64, 0:128]
# read native whole, no nodata case
yy, roi = _read(mm.gbox, fallback_nodata=None)
np.testing.assert_array_equal(xx, yy)
assert roi == np.s_[0:64, 0:128]
# read native whole, ignoring small sub-pixel translation
yy, roi = _read(gbx.translate_pix(mm.gbox, 0.3, -0.4), fallback_nodata=-33)
np.testing.assert_array_equal(xx, yy)
assert roi == np.s_[0:64, 0:128]
# no overlap between src and dst
yy, roi = _read(gbx.translate_pix(mm.gbox, 10000, -10000))
assert roi_is_empty(roi)
# read with Y flipped
yy, roi = _read(gbx.flipy(mm.gbox))
np.testing.assert_array_equal(xx[::-1, :], yy)
assert roi == np.s_[0:64, 0:128]
# read with X flipped
yy, roi = _read(gbx.flipx(mm.gbox))
np.testing.assert_array_equal(xx[:, ::-1], yy)
assert roi == np.s_[0:64, 0:128]
# read with X and Y flipped
yy, roi = _read(gbx.flipy(gbx.flipx(mm.gbox)))
assert roi == np.s_[0:64, 0:128]
np.testing.assert_array_equal(xx[::-1, ::-1], yy[roi])
# dst is fully inside src
sroi = np.s_[10:19, 31:47]
yy, roi = _read(mm.gbox[sroi])
np.testing.assert_array_equal(xx[sroi], yy[roi])
# partial overlap
yy, roi = _read(gbx.translate_pix(mm.gbox, -3, -10))
assert roi == np.s_[10:64, 3:128]
np.testing.assert_array_equal(xx[:-10, :-3], yy[roi])
assert (yy[:10, :] == -999).all()
assert (yy[:, :3] == -999).all()
# scaling paste
yy, roi = _read(gbx.zoom_out(mm.gbox, 2), check_paste=True)
assert roi == np.s_[0:32, 0:64]
np.testing.assert_array_equal(xx[1::2, 1::2], yy)
def test_roi_tools():
from datacube.utils.geometry import (
roi_is_empty,
roi_is_full,
roi_shape,
roi_normalise,
roi_boundary,
roi_from_points,
roi_center,
roi_pad,
roi_intersect,
scaled_down_roi,
scaled_up_roi,
scaled_down_shape,
)
from numpy import s_
assert roi_shape(s_[2:4, 3:4]) == (2, 1)
assert roi_shape(s_[:4, :7]) == (4, 7)
assert roi_is_empty(s_[:4, :5]) is False
assert roi_is_empty(s_[1:1, :10]) is True
assert roi_is_empty(s_[7:3, :10]) is True
assert roi_is_empty(s_[:3]) is False
assert roi_is_empty(s_[4:4]) is True
assert roi_is_full(s_[:3], 3) is True
assert roi_is_full(s_[:3, 0:4], (3, 4)) is True
assert roi_is_full(s_[:, 0:4], (33, 4)) is True
assert roi_is_full(s_[1:3, 0:4], (3, 4)) is False
assert roi_is_full(s_[1:3, 0:4], (2, 4)) is False
assert roi_is_full(s_[0:4, 0:4], (3, 4)) is False
roi = s_[0:8, 0:4]
roi_ = scaled_down_roi(roi, 2)
assert roi_shape(roi_) == (4, 2)
assert scaled_down_roi(scaled_up_roi(roi, 3), 3) == roi
assert scaled_down_shape(roi_shape(roi),
2) == roi_shape(scaled_down_roi(roi, 2))
assert roi_shape(scaled_up_roi(roi, 10000, (40, 50))) == (40, 50)
assert roi_normalise(s_[3:4], 40) == s_[3:4]
assert roi_normalise(s_[:4], (40, )) == s_[0:4]
assert roi_normalise(s_[:], (40, )) == s_[0:40]
assert roi_normalise(s_[:-1], (3, )) == s_[0:2]
assert roi_normalise(s_[-2:-1, :], (10, 20)) == s_[8:9, 0:20]
assert roi_normalise(s_[-2:-1, :, 3:4], (10, 20, 100)) == s_[8:9, 0:20,
3:4]
assert roi_center(s_[0:3]) == 1.5
assert roi_center(s_[0:2, 0:6]) == (1, 3)
roi = s_[0:2, 4:13]
xy = roi_boundary(roi)
assert xy.shape == (4, 2)
assert roi_from_points(xy, (2, 13)) == roi
assert roi_intersect(roi, roi) == roi
assert roi_intersect(s_[0:3], s_[1:7]) == s_[1:3]
assert roi_intersect(s_[0:3], (s_[1:7], )) == s_[1:3]
assert roi_intersect((s_[0:3], ), s_[1:7]) == (s_[1:3], )
assert roi_intersect(s_[4:7, 5:6], s_[0:1, 7:8]) == s_[4:4, 6:6]
assert roi_pad(s_[0:4], 1, 4) == s_[0:4]
assert roi_pad(s_[0:4, 1:5], 1, (4, 6)) == s_[0:4, 0:6]
assert roi_pad(s_[2:3, 1:5], 10, (7, 9)) == s_[0:7, 0:9]
