def test_read_raster_window(dummy1_tif, minmax_zoom):
"""Read array with read_raster_window."""
zoom = 8
# without reproject
config = MapcheteConfig(minmax_zoom.path)
rasterfile = config.params_at_zoom(zoom)["input"]["file1"]
dummy1_bbox = rasterfile.bbox()
pixelbuffer = 5
tile_pyramid = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
tiles = list(tile_pyramid.tiles_from_geom(dummy1_bbox, zoom))
# add edge tile
tiles.append(tile_pyramid.tile(8, 0, 0))
for tile in tiles:
width, height = tile.shape
for band in read_raster_window(dummy1_tif, tile):
assert isinstance(band, ma.MaskedArray)
assert band.shape == (width, height)
for index in range(1, 4):
band = read_raster_window(dummy1_tif, tile, index)
assert isinstance(band, ma.MaskedArray)
assert band.shape == (width, height)
for index in [None, [1, 2, 3]]:
band = read_raster_window(dummy1_tif, tile, index)
assert isinstance(band, ma.MaskedArray)
assert band.ndim == 3
assert band.shape == (3, width, height)
def test_read_raster_window_reproject(dummy1_3857_tif, minmax_zoom):
"""Read array with read_raster_window."""
zoom = 8
# with reproject
config_raw = minmax_zoom.dict
config_raw["input"].update(file1=dummy1_3857_tif)
config = MapcheteConfig(config_raw)
rasterfile = config.params_at_zoom(zoom)["input"]["file1"]
dummy1_bbox = rasterfile.bbox()
pixelbuffer = 5
tile_pyramid = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
tiles = list(tile_pyramid.tiles_from_geom(dummy1_bbox, zoom))
# target window out of CRS bounds
band = read_raster_window(dummy1_3857_tif, tile_pyramid.tile(12, 0, 0))
assert isinstance(band, ma.MaskedArray)
assert band.mask.all()
# not intersecting tile
tiles.append(tile_pyramid.tile(zoom, 1, 1)) # out of CRS bounds
tiles.append(tile_pyramid.tile(zoom, 16, 1)) # out of file bbox
for tile in tiles:
for band in read_raster_window(dummy1_3857_tif, tile):
assert isinstance(band, ma.MaskedArray)
assert band.shape == tile.shape
bands = read_raster_window(dummy1_3857_tif, tile, [1])
assert isinstance(bands, ma.MaskedArray)
assert bands.shape == tile.shape
# errors
with pytest.raises(IOError):
read_raster_window("nonexisting_path", tile)
def test_read_raster_window():
"""Read array with read_raster_window."""
dummy1 = os.path.join(TESTDATA_DIR, "dummy1.tif")
zoom = 8
config = MapcheteConfig(
os.path.join(SCRIPTDIR, "testdata/minmax_zoom.mapchete"))
rasterfile = config.at_zoom(7)["input"]["file1"]
dummy1_bbox = rasterfile.bbox()
pixelbuffer = 5
tile_pyramid = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
tiles = tile_pyramid.tiles_from_geom(dummy1_bbox, zoom)
width = height = tile_pyramid.tile_size + 2 * pixelbuffer
for tile in tiles:
for band in raster.read_raster_window(dummy1, tile):
assert isinstance(band, ma.MaskedArray)
assert band.shape == (width, height)
for index in range(4):
band = raster.read_raster_window(dummy1, tile, index).next()
assert isinstance(band, ma.MaskedArray)
assert band.shape == (width, height)
for resampling in [
"nearest", "bilinear", "cubic", "cubic_spline", "lanczos",
"average", "mode"
]:
raster.read_raster_window(dummy1, tile, resampling=resampling)
def test_read_raster_window_input_list(cleantopo_br):
process_zoom = 5
conf = dict(**cleantopo_br.dict)
conf["output"].update(metatiling=1)
with mapchete.open(conf) as mp:
mp.batch_process(process_zoom)
tiles = [(tile, mp.config.output.get_path(tile))
for tile in mp.config.output_pyramid.tiles_from_bounds(
mp.config.bounds, process_zoom)
if path_exists(mp.config.output.get_path(tile))]
upper_tile = next(mp.get_process_tiles(process_zoom - 1))
assert len(tiles) > 1
resampled = resample_from_array(in_raster=create_mosaic([
(tile, read_raster_window(path, tile)) for tile, path in tiles
]),
out_tile=upper_tile)
resampled2 = read_raster_window([p for _, p in tiles],
upper_tile,
src_nodata=0,
dst_nodata=0)
assert resampled.dtype == resampled2.dtype
assert resampled.shape == resampled2.shape
assert np.array_equal(resampled.mask, resampled2.mask)
# TODO slight rounding errors occur
assert np.allclose(resampled, resampled2, rtol=0.01)
def _read_as_tiledir(data_type=None,
out_tile=None,
td_crs=None,
tiles_paths=None,
profile=None,
validity_check=False,
indexes=None,
resampling=None,
dst_nodata=None,
gdal_opts=None,
**kwargs):
logger.debug("reading data from CRS %s to CRS %s", td_crs, out_tile.tp.crs)
if data_type == "vector":
if tiles_paths:
return read_vector_window([path for _, path in tiles_paths],
out_tile,
validity_check=validity_check)
else:
return []
elif data_type == "raster":
if tiles_paths:
return read_raster_window([path for _, path in tiles_paths],
out_tile,
indexes=indexes,
resampling=resampling,
src_nodata=profile["nodata"],
dst_nodata=dst_nodata,
gdal_opts=gdal_opts)
else:
bands = len(indexes) if indexes else profile["count"]
return ma.masked_array(data=np.full(
(bands, out_tile.height, out_tile.width),
profile["nodata"],
dtype=profile["dtype"]),
mask=True)
def test_read_raster_window_mask(s2_band):
"""No resampling artefacts on mask edges."""
tile = BufferedTilePyramid("geodetic").tile(zoom=13, row=1918, col=8905)
data = read_raster_window(
s2_band, tile, resampling="cubic", src_nodata=0, dst_nodata=0)
assert data.any()
assert not np.where(data == 1, True, False).any()
def read(
self, validity_check=False, indexes=None, resampling="nearest",
dst_nodata=None, gdal_opts=None
):
"""
Read reprojected & resampled input data.
Parameters
----------
validity_check : bool
vector file: also run checks if reprojected geometry is valid,
otherwise throw RuntimeError (default: True)
indexes : list or int
raster file: a list of band numbers; None will read all.
resampling : string
raster file: one of "nearest", "average", "bilinear" or "lanczos"
dst_nodata : int or float, optional
raster file: if not set, the nodata value from the source dataset
will be used
gdal_opts : dict
raster file: GDAL options passed on to rasterio.Env()
Returns
-------
data : list for vector files or numpy array for raster files
"""
if self._file_type == "vector":
if self.is_empty():
return []
return list(chain.from_iterable([
read_vector_window(
_path, self.tile, validity_check=validity_check)
for _, _path in self._tiles_paths
]))
else:
if self.is_empty():
count = (len(indexes) if indexes else self._profile["count"], )
return ma.masked_array(
data=np.full(
count + self.tile.shape, self._profile["nodata"],
dtype=self._profile["dtype"]),
mask=True
)
tiles = [
(_tile, read_raster_window(
_path, _tile, indexes=indexes, resampling=resampling,
src_nodata=self._profile["nodata"], dst_nodata=dst_nodata,
gdal_opts=gdal_opts))
for _tile, _path in self._tiles_paths
]
return resample_from_array(
in_raster=create_mosaic(
tiles=tiles, nodata=self._profile["nodata"]),
out_tile=self.tile,
resampling=resampling,
nodataval=self._profile["nodata"])
def _bands_from_cache(self, indexes=None):
"""Cache reprojected source data for multiple usage."""
band_indexes = self._get_band_indexes(indexes)
for band_index in band_indexes:
if band_index not in self._np_band_cache:
band = read_raster_window(
self.raster_file.path,
self.tile,
indexes=band_index,
resampling=self.resampling
).next()
self._np_band_cache[band_index] = band
yield self._np_band_cache[band_index]
def read(self, indexes=None):
"""
Read reprojected & resampled input data.
Returns
-------
data : array
"""
return read_raster_window(self.raster_file.path,
self.tile,
indexes=self._get_band_indexes(indexes),
resampling=self.resampling,
gdal_opts=self.gdal_opts)
def read(self, output_tile, **kwargs):
"""
Read existing process output.
Parameters
----------
output_tile : ``BufferedTile``
must be member of output ``TilePyramid``
Returns
-------
NumPy array
"""
return read_raster_window(self.rio_file, output_tile)
def test_read_raster_window_resampling(cleantopo_br_tif):
"""Assert various resampling options work."""
tp = BufferedTilePyramid("geodetic")
with rasterio.open(cleantopo_br_tif, "r") as src:
tiles = tp.tiles_from_bounds(src.bounds, 4)
for tile in tiles:
outputs = [
read_raster_window(cleantopo_br_tif, tile, resampling=resampling)
for resampling in [
"nearest", "bilinear", "cubic", "cubic_spline", "lanczos",
"average", "mode"
]
]
# resampling test:
assert any([
not np.array_equal(w, v) for v, w in zip(outputs[:-1], outputs[1:])
])
def test_s3_read_raster_window(s2_band_remote):
tile = BufferedTilePyramid("geodetic").tile(10, 276, 1071)
assert read_raster_window(s2_band_remote, tile).any()
def test_read_raster_window_partly_overlapping(cleantopo_br_tif):
"""Read array with read_raster_window where window is bigger than file."""
tile = BufferedTilePyramid("geodetic").tile(4, 15, 31)
data = read_raster_window(cleantopo_br_tif, tile)
assert isinstance(data, ma.MaskedArray)
assert data.mask.any()
def test_read_raster_window_retry(invalid_tif):
tile = BufferedTilePyramid("geodetic").tile(zoom=13, row=1918, col=8905)
with pytest.raises(MapcheteIOError):
read_raster_window(invalid_tif, tile)
with pytest.raises(FileNotFoundError):
read_raster_window("not_existing.tif", tile)
def read(self,
indexes=None,
resampling=None,
mask_nodata=True,
mask_clouds=False,
mask_white_areas=False,
return_empty=False):
"""
Read reprojected & resampled input data.
Parameters
----------
indexes : integer or list
band number or list of band numbers
resampling : str
resampling method
mask_nodata : bool
mask out nodata (values in all bands equal 0) areas (default: True)
mask_clouds : bool
mask out clouds (default: False)
mask_white_areas : bool
mask out white (values over 4096) areas; might just work on RGB
bands! (default: False)
return_empty : bool
returns empty array if True or raise MapcheteEmptyInputTile
exception if False (default: False)
Returns
-------
data : NumPy array or raise MapcheteEmptyInputTile exception
Band data
"""
resampling = resampling if resampling is not None else self.resampling
band_indexes = self._get_band_indexes(indexes)
# return immediately if tile does not intersect with input data
if self.is_empty():
if return_empty:
return self._empty(len(band_indexes))
else:
raise MapcheteEmptyInputTile
# iterate through affected granules
granules = [
granule for granule in self.s2metadata["granules"]
if granule["footprint"].intersects(self.tile.bbox)
]
# read bands from granules
bands = []
for band_index in band_indexes:
band = ma.masked_array(ma.zeros(self.tile.shape, dtype=self.dtype),
mask=True)
for granule in granules:
new_data = read_raster_window(granule["band_path"][band_index],
self.tile,
indexes=[1],
resampling=self.resampling,
src_nodata=0,
dst_nodata=0)
if new_data.mask.all():
continue
band = ma.masked_array(data=np.where(new_data == 0, band.data,
new_data),
mask=np.where(new_data == 0, True,
False))
bands.append(band)
# get combined mask
mask = self._mask(bands, mask_nodata, mask_white_areas, mask_clouds)
# skip if emtpy
if mask.all():
if return_empty:
return self._empty(len(band_indexes))
else:
raise MapcheteEmptyInputTile("all values masked")
else:
nd_mask = np.stack([mask for _ in band_indexes])
return ma.masked_array(data=np.where(nd_mask, 0, np.stack(bands)),
mask=nd_mask)
