def reproject_and_fuse(datasources: List[DataSource],
destination: np.ndarray,
dst_gbox: GeoBox,
dst_nodata: Optional[Union[int, float]],
resampling: str = 'nearest',
fuse_func: Optional[FuserFunction] = None,
skip_broken_datasets: bool = False,
progress_cbk: Optional[ProgressFunction] = None):
"""
Reproject and fuse `sources` into a 2D numpy array `destination`.
:param datasources: Data sources to open and read from
:param destination: ndarray of appropriate size to read data into
:param dst_gbox: GeoBox defining destination region
:param skip_broken_datasets: Carry on in the face of adversity and failing reads.
:param progress_cbk: If supplied will be called with 2 integers `Items processed, Total Items`
after reading each file.
"""
# pylint: disable=too-many-locals
from ._read import read_time_slice
assert len(destination.shape) == 2
def copyto_fuser(dest: np.ndarray, src: np.ndarray) -> None:
_default_fuser(dest, src, dst_nodata)
fuse_func = fuse_func or copyto_fuser
destination.fill(dst_nodata)
if len(datasources) == 0:
return destination
elif len(datasources) == 1:
with ignore_exceptions_if(skip_broken_datasets):
with datasources[0].open() as rdr:
read_time_slice(rdr, destination, dst_gbox, resampling,
dst_nodata)
if progress_cbk:
progress_cbk(1, 1)
return destination
else:
# Multiple sources, we need to fuse them together into a single array
buffer_ = np.full(destination.shape,
dst_nodata,
dtype=destination.dtype)
for n_so_far, source in enumerate(datasources, 1):
with ignore_exceptions_if(skip_broken_datasets):
with source.open() as rdr:
roi = read_time_slice(rdr, buffer_, dst_gbox, resampling,
dst_nodata)
if not roi_is_empty(roi):
fuse_func(destination[roi], buffer_[roi])
buffer_[roi] = dst_nodata # clean up for next read
if progress_cbk:
progress_cbk(n_so_far, len(datasources))
return destination
def reproject_and_fuse(datasources: List[DataSource],
destination: np.ndarray,
dst_gbox: GeoBox,
dst_nodata: Optional[Union[int, float]],
resampling: str = 'nearest',
fuse_func: Optional[FuserFunction] = None,
skip_broken_datasets: bool = False):
"""
Reproject and fuse `sources` into a 2D numpy array `destination`.
:param datasources: Data sources to open and read from
:param destination: ndarray of appropriate size to read data into
:param dst_gbox: GeoBox defining destination region
:param skip_broken_datasets: Carry on in the face of adversity and failing reads.
"""
# pylint: disable=too-many-locals
assert len(destination.shape) == 2
def copyto_fuser(dest: np.ndarray, src: np.ndarray) -> None:
where_nodata = (
dest == dst_nodata) if not np.isnan(dst_nodata) else np.isnan(dest)
np.copyto(dest, src, where=where_nodata)
fuse_func = fuse_func or copyto_fuser
destination.fill(dst_nodata)
if len(datasources) == 0:
return destination
elif len(datasources) == 1:
with ignore_exceptions_if(skip_broken_datasets):
with datasources[0].open() as rdr:
read_time_slice(rdr, destination, dst_gbox, resampling,
dst_nodata)
return destination
else:
# Multiple sources, we need to fuse them together into a single array
buffer_ = np.full(destination.shape,
dst_nodata,
dtype=destination.dtype)
for source in datasources:
with ignore_exceptions_if(skip_broken_datasets):
with source.open() as rdr:
roi = read_time_slice(rdr, buffer_, dst_gbox, resampling,
dst_nodata)
if not roi_is_empty(roi):
fuse_func(destination[roi], buffer_[roi])
buffer_[roi] = dst_nodata # clean up for next read
return destination
def reproject_and_fuse(datasources,
destination,
dst_transform,
dst_projection,
dst_nodata,
resampling='nearest',
fuse_func=None,
skip_broken_datasets=False):
"""
Reproject and fuse `sources` into a 2D numpy array `destination`.
:param List[DataSource] datasources: Data sources to open and read from
:param numpy.ndarray destination: ndarray of appropriate size to read data into
:type resampling: str
:type fuse_func: callable or None
:param bool skip_broken_datasets: Carry on in the face of adversity and failing reads.
"""
assert len(destination.shape) == 2
resampling = _rasterio_resampling_method(resampling)
def copyto_fuser(dest, src):
"""
:type dest: numpy.ndarray
:type src: numpy.ndarray
"""
where_nodata = (dest == dst_nodata
) if not numpy.isnan(dst_nodata) else numpy.isnan(dest)
numpy.copyto(dest, src, where=where_nodata)
fuse_func = fuse_func or copyto_fuser
destination.fill(dst_nodata)
if len(datasources) == 0:
return destination
elif len(datasources) == 1:
with ignore_exceptions_if(skip_broken_datasets):
read_from_source(datasources[0], destination, dst_transform,
dst_nodata, dst_projection, resampling)
return destination
else:
# Multiple sources, we need to fuse them together into a single array
buffer_ = numpy.empty(destination.shape, dtype=destination.dtype)
for source in datasources:
with ignore_exceptions_if(skip_broken_datasets):
read_from_source(source, buffer_, dst_transform, dst_nodata,
dst_projection, resampling)
fuse_func(destination, buffer_)
return destination
def _fuse_measurement(dest,
datasets,
geobox,
measurement,
skip_broken_datasets=False,
progress_cbk=None,
extra_dim_index=None):
srcs = []
for ds in datasets:
src = None
with ignore_exceptions_if(skip_broken_datasets):
src = new_datasource(
BandInfo(ds, measurement.name,
extra_dim_index=extra_dim_index))
if src is None:
if not skip_broken_datasets:
raise ValueError(f"Failed to load dataset: {ds.id}")
else:
srcs.append(src)
reproject_and_fuse(srcs,
dest,
geobox,
dest.dtype.type(measurement.nodata),
resampling=measurement.get('resampling_method',
'nearest'),
fuse_func=measurement.get('fuser', None),
skip_broken_datasets=skip_broken_datasets,
progress_cbk=progress_cbk,
extra_dim_index=extra_dim_index)
def test_missing_file_handling():
with pytest.raises(IOError):
rio_slurp('no-such-file.tiff')
# by default should catch any exception
with ignore_exceptions_if(True):
rio_slurp('no-such-file.tiff')
# this is equivalent to previous default behaviour, note that missing http
# resources are not OSError
with ignore_exceptions_if(True, (OSError, )):
rio_slurp('no-such-file.tiff')
# check that only requested exceptions are caught
with pytest.raises(IOError):
with ignore_exceptions_if(True, (ValueError, ArithmeticError)):
rio_slurp('no-such-file.tiff')
