def reproject_and_fuse(datasources: List[DataSource], destination: np.ndarray, dst_gbox: GeoBox, dst_nodata: Optional[Union[int, float]], resampling: str = 'nearest', fuse_func: Optional[FuserFunction] = None, skip_broken_datasets: bool = False, progress_cbk: Optional[ProgressFunction] = None): """ Reproject and fuse `sources` into a 2D numpy array `destination`. :param datasources: Data sources to open and read from :param destination: ndarray of appropriate size to read data into :param dst_gbox: GeoBox defining destination region :param skip_broken_datasets: Carry on in the face of adversity and failing reads. :param progress_cbk: If supplied will be called with 2 integers `Items processed, Total Items` after reading each file. """ # pylint: disable=too-many-locals from ._read import read_time_slice assert len(destination.shape) == 2 def copyto_fuser(dest: np.ndarray, src: np.ndarray) -> None: _default_fuser(dest, src, dst_nodata) fuse_func = fuse_func or copyto_fuser destination.fill(dst_nodata) if len(datasources) == 0: return destination elif len(datasources) == 1: with ignore_exceptions_if(skip_broken_datasets): with datasources[0].open() as rdr: read_time_slice(rdr, destination, dst_gbox, resampling, dst_nodata) if progress_cbk: progress_cbk(1, 1) return destination else: # Multiple sources, we need to fuse them together into a single array buffer_ = np.full(destination.shape, dst_nodata, dtype=destination.dtype) for n_so_far, source in enumerate(datasources, 1): with ignore_exceptions_if(skip_broken_datasets): with source.open() as rdr: roi = read_time_slice(rdr, buffer_, dst_gbox, resampling, dst_nodata) if not roi_is_empty(roi): fuse_func(destination[roi], buffer_[roi]) buffer_[roi] = dst_nodata # clean up for next read if progress_cbk: progress_cbk(n_so_far, len(datasources)) return destination
def reproject_and_fuse(datasources: List[DataSource], destination: np.ndarray, dst_gbox: GeoBox, dst_nodata: Optional[Union[int, float]], resampling: str = 'nearest', fuse_func: Optional[FuserFunction] = None, skip_broken_datasets: bool = False): """ Reproject and fuse `sources` into a 2D numpy array `destination`. :param datasources: Data sources to open and read from :param destination: ndarray of appropriate size to read data into :param dst_gbox: GeoBox defining destination region :param skip_broken_datasets: Carry on in the face of adversity and failing reads. """ # pylint: disable=too-many-locals assert len(destination.shape) == 2 def copyto_fuser(dest: np.ndarray, src: np.ndarray) -> None: where_nodata = ( dest == dst_nodata) if not np.isnan(dst_nodata) else np.isnan(dest) np.copyto(dest, src, where=where_nodata) fuse_func = fuse_func or copyto_fuser destination.fill(dst_nodata) if len(datasources) == 0: return destination elif len(datasources) == 1: with ignore_exceptions_if(skip_broken_datasets): with datasources[0].open() as rdr: read_time_slice(rdr, destination, dst_gbox, resampling, dst_nodata) return destination else: # Multiple sources, we need to fuse them together into a single array buffer_ = np.full(destination.shape, dst_nodata, dtype=destination.dtype) for source in datasources: with ignore_exceptions_if(skip_broken_datasets): with source.open() as rdr: roi = read_time_slice(rdr, buffer_, dst_gbox, resampling, dst_nodata) if not roi_is_empty(roi): fuse_func(destination[roi], buffer_[roi]) buffer_[roi] = dst_nodata # clean up for next read return destination
def reproject_and_fuse(datasources, destination, dst_transform, dst_projection, dst_nodata, resampling='nearest', fuse_func=None, skip_broken_datasets=False): """ Reproject and fuse `sources` into a 2D numpy array `destination`. :param List[DataSource] datasources: Data sources to open and read from :param numpy.ndarray destination: ndarray of appropriate size to read data into :type resampling: str :type fuse_func: callable or None :param bool skip_broken_datasets: Carry on in the face of adversity and failing reads. """ assert len(destination.shape) == 2 resampling = _rasterio_resampling_method(resampling) def copyto_fuser(dest, src): """ :type dest: numpy.ndarray :type src: numpy.ndarray """ where_nodata = (dest == dst_nodata ) if not numpy.isnan(dst_nodata) else numpy.isnan(dest) numpy.copyto(dest, src, where=where_nodata) fuse_func = fuse_func or copyto_fuser destination.fill(dst_nodata) if len(datasources) == 0: return destination elif len(datasources) == 1: with ignore_exceptions_if(skip_broken_datasets): read_from_source(datasources[0], destination, dst_transform, dst_nodata, dst_projection, resampling) return destination else: # Multiple sources, we need to fuse them together into a single array buffer_ = numpy.empty(destination.shape, dtype=destination.dtype) for source in datasources: with ignore_exceptions_if(skip_broken_datasets): read_from_source(source, buffer_, dst_transform, dst_nodata, dst_projection, resampling) fuse_func(destination, buffer_) return destination
def _fuse_measurement(dest, datasets, geobox, measurement, skip_broken_datasets=False, progress_cbk=None, extra_dim_index=None): srcs = [] for ds in datasets: src = None with ignore_exceptions_if(skip_broken_datasets): src = new_datasource( BandInfo(ds, measurement.name, extra_dim_index=extra_dim_index)) if src is None: if not skip_broken_datasets: raise ValueError(f"Failed to load dataset: {ds.id}") else: srcs.append(src) reproject_and_fuse(srcs, dest, geobox, dest.dtype.type(measurement.nodata), resampling=measurement.get('resampling_method', 'nearest'), fuse_func=measurement.get('fuser', None), skip_broken_datasets=skip_broken_datasets, progress_cbk=progress_cbk, extra_dim_index=extra_dim_index)
def test_missing_file_handling(): with pytest.raises(IOError): rio_slurp('no-such-file.tiff') # by default should catch any exception with ignore_exceptions_if(True): rio_slurp('no-such-file.tiff') # this is equivalent to previous default behaviour, note that missing http # resources are not OSError with ignore_exceptions_if(True, (OSError, )): rio_slurp('no-such-file.tiff') # check that only requested exceptions are caught with pytest.raises(IOError): with ignore_exceptions_if(True, (ValueError, ArithmeticError)): rio_slurp('no-such-file.tiff')