def test_create_mosaic():
"""Create mosaic from tiles."""
tp = BufferedTilePyramid("geodetic")
# quick return mosaic if there is just one tile
tile = tp.tile(3, 3, 3)
data = np.ones(tile.shape)
mosaic = create_mosaic([(tile, data)])
assert isinstance(mosaic, ReferencedRaster)
assert np.array_equal(data, mosaic.data)
assert tile.affine == mosaic.affine
# multiple tiles
for pixelbuffer in [0, 10]:
tp = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
tiles = [(tp.tile(5, row, col), np.ones(tp.tile(5, row, col).shape))
for row, col in product(range(4), range(4))]
# 4x4 top left tiles from zoom 5 equal top left tile from zoom 3
# also use tile generator
mosaic = create_mosaic((t for t in tiles))
assert isinstance(mosaic, ReferencedRaster)
assert np.all(np.where(mosaic.data == 1, True, False))
mosaic_bbox = box(
mosaic.affine[2],
mosaic.affine[5] + mosaic.data.shape[1] * mosaic.affine[4],
mosaic.affine[2] + mosaic.data.shape[2] * mosaic.affine[0],
mosaic.affine[5])
control_bbox = box(*unary_union([t.bbox for t, _ in tiles]).bounds)
assert mosaic_bbox.equals(control_bbox)
def test_create_mosaic_errors():
"""Check error handling of create_mosaic()."""
tp_geo = BufferedTilePyramid("geodetic")
tp_mer = BufferedTilePyramid("mercator")
geo_tile = tp_geo.tile(1, 0, 0)
geo_tile.data = np.ndarray(geo_tile.shape)
mer_tile = tp_mer.tile(1, 1, 0)
mer_tile.data = np.ndarray(mer_tile.shape)
# CRS error
with pytest.raises(ValueError):
raster.create_mosaic([geo_tile, mer_tile])
# zoom error
with pytest.raises(ValueError):
diff_zoom = tp_geo.tile(2, 1, 0)
diff_zoom.data = np.ndarray(diff_zoom.shape)
raster.create_mosaic([geo_tile, diff_zoom])
# tile data error
with pytest.raises(TypeError):
geo_tile.data = None
# for one tile
raster.create_mosaic([geo_tile])
with pytest.raises(TypeError):
geo_tile.data = None
# for multiple tiles
raster.create_mosaic([geo_tile, geo_tile])
# tile data type error
with pytest.raises(TypeError):
geo_tile.data = np.ndarray(geo_tile.shape)
diff_type = tp_geo.tile(1, 1, 0)
diff_type.data = np.ndarray(diff_zoom.shape).astype("int")
raster.create_mosaic([geo_tile, diff_type])
def test_processing():
"""Test correct processing (read and write) outputs."""
for cleantopo_process in [
"testdata/cleantopo_tl.mapchete", "testdata/cleantopo_br.mapchete"
]:
mp = mapchete.open(os.path.join(SCRIPTDIR, cleantopo_process))
for zoom in range(6):
tiles = []
for tile in mp.get_process_tiles(zoom):
output = mp.execute(tile)
tiles.append(output)
assert isinstance(output, BufferedTile)
assert isinstance(output.data, ma.MaskedArray)
assert output.data.shape == output.shape
assert not ma.all(output.data.mask)
mp.write(output)
mosaic, mosaic_affine = create_mosaic(tiles)
try:
temp_vrt = os.path.join(OUT_DIR, str(zoom) + ".vrt")
gdalbuildvrt = "gdalbuildvrt %s %s/%s/*/*.tif > /dev/null" % (
temp_vrt, OUT_DIR, zoom)
os.system(gdalbuildvrt)
with rasterio.open(temp_vrt, "r") as testfile:
for file_item, mosaic_item in zip(
testfile.meta["transform"], mosaic_affine):
assert file_item == mosaic_item
band = testfile.read(1, masked=True)
assert band.shape == mosaic.shape
assert ma.allclose(band, mosaic)
assert ma.allclose(band.mask, mosaic.mask)
finally:
shutil.rmtree(OUT_DIR, ignore_errors=True)
def test_read_raster_window_input_list(cleantopo_br):
process_zoom = 5
conf = dict(**cleantopo_br.dict)
conf["output"].update(metatiling=1)
with mapchete.open(conf) as mp:
mp.batch_process(process_zoom)
tiles = [(tile, mp.config.output.get_path(tile))
for tile in mp.config.output_pyramid.tiles_from_bounds(
mp.config.bounds, process_zoom)
if path_exists(mp.config.output.get_path(tile))]
upper_tile = next(mp.get_process_tiles(process_zoom - 1))
assert len(tiles) > 1
resampled = resample_from_array(in_raster=create_mosaic([
(tile, read_raster_window(path, tile)) for tile, path in tiles
]),
out_tile=upper_tile)
resampled2 = read_raster_window([p for _, p in tiles],
upper_tile,
src_nodata=0,
dst_nodata=0)
assert resampled.dtype == resampled2.dtype
assert resampled.shape == resampled2.shape
assert np.array_equal(resampled.mask, resampled2.mask)
# TODO slight rounding errors occur
assert np.allclose(resampled, resampled2, rtol=0.01)
def read(self, **kwargs):
"""
Read existing output data from a previous run.
Returns
-------
process output : NumPy array (raster) or feature iterator (vector)
"""
if self.tile.pixelbuffer > self.config.output.pixelbuffer:
output_tiles = list(
self.config.output_pyramid.tiles_from_bounds(
self.tile.bounds, self.tile.zoom))
else:
output_tiles = self.config.output_pyramid.intersecting(self.tile)
if self.config.output.METADATA["data_type"] == "raster":
return raster.extract_from_array(in_raster=raster.create_mosaic([
(output_tile, self.config.output.read(output_tile))
for output_tile in output_tiles
]),
out_tile=self.tile)
elif self.config.output.METADATA["data_type"] == "vector":
return list(
chain.from_iterable([
self.config.output.read(output_tile)
for output_tile in output_tiles
]))
def _interpolate_from_baselevel(self, tile=None, baselevel=None):
starttime = time.time()
# resample from parent tile
if baselevel == "higher":
parent_tile = tile.get_parent()
process_data = raster.resample_from_array(
in_raster=self.get_raw_output(parent_tile,
_baselevel_readonly=True),
in_affine=parent_tile.affine,
out_tile=tile,
resampling=self.config.baselevels["higher"],
nodataval=self.config.output.nodata)
# resample from children tiles
elif baselevel == "lower":
mosaic, mosaic_affine = raster.create_mosaic([
(child_tile,
self.get_raw_output(child_tile, _baselevel_readonly=True))
for child_tile in self.config.baselevels["tile_pyramid"].tile(
*tile.id).get_children()
])
process_data = raster.resample_from_array(
in_raster=mosaic,
in_affine=mosaic_affine,
out_tile=tile,
resampling=self.config.baselevels["lower"],
nodataval=self.config.output.nodata)
elapsed = "%ss" % (round((time.time() - starttime), 3))
logger.debug((tile.id, "generated from baselevel", elapsed))
return process_data
def _interpolate_from_baselevel(self, process_tile, baselevel):
try:
starttime = time.time()
# resample from parent tile
if baselevel == "higher":
parent_tile = self.get_raw_output(process_tile.get_parent(),
_baselevel_readonly=True)
process_data = raster.resample_from_array(
parent_tile.data,
parent_tile.affine,
process_tile,
self.config.baselevels["higher"],
nodataval=self.config.output.nodata)
# resample from children tiles
elif baselevel == "lower":
mosaic, mosaic_affine = raster.create_mosaic([
self.get_raw_output(child_tile, _baselevel_readonly=True)
for child_tile in process_tile.get_children()
])
process_data = raster.resample_from_array(
mosaic,
mosaic_affine,
process_tile,
self.config.baselevels["lower"],
nodataval=self.config.output.nodata)
elapsed = "%ss" % (round((time.time() - starttime), 3))
LOGGER.debug(
(process_tile.id, "generated from baselevel", elapsed))
except Exception as e:
elapsed = "%ss" % (round((time.time() - starttime), 3))
LOGGER.error((process_tile.id, "baselevel error", e, elapsed))
raise
return process_data
def test_processing(mp_tmpdir, cleantopo_br, cleantopo_tl):
"""Test correct processing (read and write) outputs."""
for cleantopo_process in [cleantopo_br.path, cleantopo_tl.path]:
with mapchete.open(cleantopo_process) as mp:
for zoom in range(6):
tiles = []
for tile in mp.get_process_tiles(zoom):
output = mp.execute(tile)
tiles.append((tile, output))
assert isinstance(output, ma.MaskedArray)
assert output.shape == output.shape
assert not ma.all(output.mask)
mp.write(tile, output)
mosaic = create_mosaic(tiles)
try:
temp_vrt = os.path.join(mp_tmpdir, str(zoom)+".vrt")
gdalbuildvrt = "gdalbuildvrt %s %s/%s/*/*.tif > /dev/null" % (
temp_vrt, mp.config.output.path, zoom)
os.system(gdalbuildvrt)
with rasterio.open(temp_vrt, "r") as testfile:
for file_item, mosaic_item in zip(
testfile.meta["transform"], mosaic.affine
):
assert file_item == mosaic_item
band = testfile.read(1, masked=True)
assert band.shape == mosaic.data.shape
assert ma.allclose(band, mosaic.data)
assert ma.allclose(band.mask, mosaic.data.mask)
finally:
shutil.rmtree(mp_tmpdir, ignore_errors=True)
def test_old_style_process_class(mp_tmpdir, cleantopo_tl, old_style_process_py):
"""Test correct processing using MapcheteProcess class."""
config = cleantopo_tl.dict
config.update(process_file=old_style_process_py)
with mapchete.open(config) as mp:
for zoom in range(6):
tiles = []
for tile in mp.get_process_tiles(zoom):
output = mp.execute(tile)
tiles.append((tile, output))
assert isinstance(output, ma.MaskedArray)
assert output.shape == output.shape
assert not ma.all(output.mask)
mp.write(tile, output)
mosaic, mosaic_affine = create_mosaic(tiles)
try:
temp_vrt = os.path.join(mp_tmpdir, str(zoom)+".vrt")
gdalbuildvrt = "gdalbuildvrt %s %s/%s/*/*.tif > /dev/null" % (
temp_vrt, mp_tmpdir, zoom)
os.system(gdalbuildvrt)
with rasterio.open(temp_vrt, "r") as testfile:
for file_item, mosaic_item in zip(
testfile.meta["transform"], mosaic_affine
):
assert file_item == mosaic_item
band = testfile.read(1, masked=True)
assert band.shape == mosaic.shape
assert ma.allclose(band, mosaic)
assert ma.allclose(band.mask, mosaic.mask)
finally:
shutil.rmtree(mp_tmpdir, ignore_errors=True)
def read(
self, validity_check=False, indexes=None, resampling="nearest",
dst_nodata=None, gdal_opts=None
):
"""
Read reprojected & resampled input data.
Parameters
----------
validity_check : bool
vector file: also run checks if reprojected geometry is valid,
otherwise throw RuntimeError (default: True)
indexes : list or int
raster file: a list of band numbers; None will read all.
resampling : string
raster file: one of "nearest", "average", "bilinear" or "lanczos"
dst_nodata : int or float, optional
raster file: if not set, the nodata value from the source dataset
will be used
gdal_opts : dict
raster file: GDAL options passed on to rasterio.Env()
Returns
-------
data : list for vector files or numpy array for raster files
"""
if self._file_type == "vector":
if self.is_empty():
return []
return list(chain.from_iterable([
read_vector_window(
_path, self.tile, validity_check=validity_check)
for _, _path in self._tiles_paths
]))
else:
if self.is_empty():
count = (len(indexes) if indexes else self._profile["count"], )
return ma.masked_array(
data=np.full(
count + self.tile.shape, self._profile["nodata"],
dtype=self._profile["dtype"]),
mask=True
)
tiles = [
(_tile, read_raster_window(
_path, _tile, indexes=indexes, resampling=resampling,
src_nodata=self._profile["nodata"], dst_nodata=dst_nodata,
gdal_opts=gdal_opts))
for _tile, _path in self._tiles_paths
]
return resample_from_array(
in_raster=create_mosaic(
tiles=tiles, nodata=self._profile["nodata"]),
out_tile=self.tile,
resampling=resampling,
nodataval=self._profile["nodata"])
def _read_existing_output(self, tile, output_tiles):
if self.config.output.METADATA["data_type"] == "raster":
mosaic, affine = raster.create_mosaic([
(output_tile, self.read(output_tile))
for output_tile in output_tiles
])
return raster.extract_from_array(mosaic, affine, tile)
elif self.config.output.METADATA["data_type"] == "vector":
return list(
chain.from_iterable(
[self.read(output_tile) for output_tile in output_tiles]))
def _interpolate_from_baselevel(self, baselevel=None):
# This is a special tile derived from a pyramid which has the pixelbuffer setting
# from the output pyramid but metatiling from the process pyramid. This is due to
# performance reasons as for the usual case overview tiles do not need the
# process pyramid pixelbuffers.
tile = self.config_baselevels["tile_pyramid"].tile(*self.tile.id)
# get output_tiles that intersect with process tile
output_tiles = (list(
self.output_reader.pyramid.tiles_from_bounds(
tile.bounds, tile.zoom)) if tile.pixelbuffer >
self.output_reader.pyramid.pixelbuffer else
self.output_reader.pyramid.intersecting(tile))
with Timer() as t:
# resample from parent tile
if baselevel == "higher":
parent_tile = self.tile.get_parent()
process_data = raster.resample_from_array(
self.output_reader.read(parent_tile),
in_affine=parent_tile.affine,
out_tile=self.tile,
resampling=self.config_baselevels["higher"],
nodata=self.output_reader.output_params["nodata"])
# resample from children tiles
elif baselevel == "lower":
if self.output_reader.pyramid.pixelbuffer:
lower_tiles = set([
y for y in chain(*[
self.output_reader.pyramid.tiles_from_bounds(
x.bounds, x.zoom + 1) for x in output_tiles
])
])
else:
lower_tiles = [
y for y in chain(
*[x.get_children() for x in output_tiles])
]
mosaic = raster.create_mosaic(
[(lower_tile, self.output_reader.read(lower_tile))
for lower_tile in lower_tiles],
nodata=self.output_reader.output_params["nodata"])
process_data = raster.resample_from_array(
in_raster=mosaic.data,
in_affine=mosaic.affine,
out_tile=self.tile,
resampling=self.config_baselevels["lower"],
nodata=self.output_reader.output_params["nodata"])
logger.debug((self.tile.id, "generated from baselevel", str(t)))
return process_data
def test_create_mosaic_antimeridian():
"""Create mosaic using tiles on opposing antimeridian sides."""
zoom = 5
row = 0
pixelbuffer = 5
tp = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
west = tp.tile(zoom, row, 0)
east = tp.tile(zoom, row, tp.matrix_width(zoom) - 1)
for tile in [west, east]:
tile.data = np.ones(tile.shape)
mosaic = raster.create_mosaic([west, east])
assert isinstance(mosaic, raster.ReferencedRaster)
# Huge array gets initialized because the two tiles are on opposing sides
# of the projection area. The below test should pass if the tiles are
# stitched together next to each other.
assert mosaic.data.shape == (1, west.height,
west.width * 2 - 2 * pixelbuffer)
def extract_subset(self, input_data_tiles=None, out_tile=None):
"""
Extract subset from multiple tiles.
input_data_tiles : list of (``Tile``, process data) tuples
out_tile : ``Tile``
Returns
-------
NumPy array or list of features.
"""
if self.METADATA["data_type"] == "raster":
mosaic = create_mosaic(input_data_tiles)
return extract_from_array(in_raster=prepare_array(
mosaic.data,
nodata=self.output_params["nodata"],
dtype=self.output_params["dtype"]),
in_affine=mosaic.affine,
out_tile=out_tile)
elif self.METADATA["data_type"] == "vector":
return [
feature for feature in list(
chain.from_iterable(
[features for _, features in input_data_tiles]))
if shape(feature["geometry"]).intersects(out_tile.bbox)
]
def test_create_mosaic_errors():
"""Check error handling of create_mosaic()."""
tp_geo = BufferedTilePyramid("geodetic")
tp_mer = BufferedTilePyramid("mercator")
geo_tile = tp_geo.tile(1, 0, 0)
geo_tile_data = np.ndarray(geo_tile.shape)
mer_tile = tp_mer.tile(1, 1, 0)
mer_tile_data = np.ndarray(mer_tile.shape)
# tiles error
with pytest.raises(TypeError):
create_mosaic("invalid tiles")
with pytest.raises(TypeError):
create_mosaic(["invalid tiles"])
# CRS error
with pytest.raises(ValueError):
create_mosaic([(geo_tile, geo_tile_data), (mer_tile, mer_tile_data)])
# zoom error
with pytest.raises(ValueError):
diff_zoom = tp_geo.tile(2, 1, 0)
diff_zoom_data = np.ndarray(diff_zoom.shape)
create_mosaic([(geo_tile, geo_tile_data), (diff_zoom, diff_zoom_data)])
# tile data error
with pytest.raises(TypeError):
# for one tile
create_mosaic([(geo_tile, None)])
with pytest.raises(TypeError):
# for multiple tiles
create_mosaic([(geo_tile, None), (geo_tile, None)])
# tile data type error
with pytest.raises(TypeError):
diff_type = tp_geo.tile(1, 1, 0)
diff_type_data = np.ndarray(diff_zoom.shape).astype("int")
create_mosaic([(geo_tile, geo_tile_data), (diff_type, diff_type_data)])
def test_create_mosaic_antimeridian():
"""Create mosaic using tiles on opposing antimeridian sides."""
zoom = 5
row = 0
pixelbuffer = 5
tp = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
west = tp.tile(zoom, row, 0)
east = tp.tile(zoom, row, tp.matrix_width(zoom) - 1)
mosaic = create_mosaic([(west, np.ones(west.shape).astype("uint8")),
(east, np.ones(east.shape).astype("uint8") * 2)])
assert isinstance(mosaic, ReferencedRaster)
# Huge array gets initialized because the two tiles are on opposing sides of the
# projection area. The below test should pass if the tiles are stitched together next
# to each other.
assert mosaic.data.shape == (1, west.height,
west.width * 2 - 2 * pixelbuffer)
assert mosaic.data[0][0][0] == 2
assert mosaic.data[0][0][-1] == 1
# If tiles from opposing sides from Antimeridian are mosaicked it will happen that the
# output mosaic exceeds the CRS bounds (obviously). In such a case the mosaicking
# function shall make sure that the larger part of the output mosaic shall be inside
# the CRS bounds.
# (1) mosaic crosses Antimeridian in the West, larger part is on Western hemisphere:
tiles_ids = [
# Western hemisphere tiles
(zoom, row, 0),
(zoom, row, 1),
# Eastern hemisphere tile
(zoom, row, tp.matrix_width(zoom) - 1),
]
tiles = [(tp.tile(*tile_id), np.ones(tp.tile(*tile_id).shape))
for tile_id in tiles_ids]
mosaic = create_mosaic(tiles)
control_bounds = Bounds(
# Eastern tile has to be shifted
-(360 - tp.tile(*tiles_ids[2]).left),
tp.tile(*tiles_ids[2]).bottom,
tp.tile(*tiles_ids[1]).right,
tp.tile(*tiles_ids[1]).top,
)
assert mosaic.bounds == control_bounds
# (2) mosaic crosses Antimeridian in the West, larger part is on Eastern hemisphere:
tiles_ids = [
# Western hemisphere tile
(zoom, row, 0),
# Eastern hemisphere tiles
(zoom, row, tp.matrix_width(zoom) - 1),
(zoom, row, tp.matrix_width(zoom) - 2),
]
tiles = [(tp.tile(*tile_id), np.ones(tp.tile(*tile_id).shape))
for tile_id in tiles_ids]
mosaic = create_mosaic(tiles)
control_bounds = Bounds(
tp.tile(*tiles_ids[2]).left,
tp.tile(*tiles_ids[2]).bottom,
# Western tile has to be shifted
360 + tp.tile(*tiles_ids[0]).right,
tp.tile(*tiles_ids[0]).top,
)
assert mosaic.bounds == control_bounds