def test_input_data(mp_tmpdir, cleantopo_br):
"""Check GeoTIFF proces output as input data."""
with mapchete.open(cleantopo_br.path) as mp:
tp = BufferedTilePyramid("geodetic")
# TODO tile with existing but empty data
tile = tp.tile(5, 5, 5)
output_params = dict(
grid="geodetic",
format="GeoTIFF",
path=mp_tmpdir,
pixelbuffer=0,
metatiling=1,
bands=2,
dtype="int16",
delimiters=dict(bounds=Bounds(-180.0, -90.0, 180.0, 90.0),
effective_bounds=Bounds(-180.439453125, -90.0,
180.439453125, 90.0),
zoom=[5],
process_bounds=Bounds(-180.0, -90.0, 180.0, 90.0)))
output = gtiff.OutputDataWriter(output_params)
with output.open(tile, mp) as input_tile:
for data in [
input_tile.read(),
input_tile.read(1),
input_tile.read([1]),
# TODO assert valid indexes are passed input_tile.read([1, 2])
]:
assert isinstance(data, ma.masked_array)
assert input_tile.is_empty()
# open without resampling
with output.open(tile, mp) as input_tile:
pass
def prepare(self, process_area=None, **kwargs):
bounds = snap_bounds(
bounds=Bounds(*process_area.intersection(
box(*self.output_params["delimiters"]
["effective_bounds"])).bounds),
pyramid=self.pyramid,
zoom=self.zoom) if process_area else self.output_params[
"delimiters"]["effective_bounds"]
height = math.ceil((bounds.top - bounds.bottom) /
self.pyramid.pixel_x_size(self.zoom))
width = math.ceil((bounds.right - bounds.left) /
self.pyramid.pixel_x_size(self.zoom))
logger.debug("output raster bounds: %s", bounds)
logger.debug("output raster shape: %s, %s", height, width)
self._profile = dict(
GTIFF_DEFAULT_PROFILE,
driver="GTiff",
transform=Affine(self.pyramid.pixel_x_size(self.zoom), 0,
bounds.left, 0,
-self.pyramid.pixel_y_size(self.zoom),
bounds.top),
height=height,
width=width,
count=self.output_params["bands"],
crs=self.pyramid.crs,
**{
k: self.output_params.get(k, GTIFF_DEFAULT_PROFILE[k])
for k in GTIFF_DEFAULT_PROFILE.keys()
},
bigtiff=self.output_params.get("bigtiff", "NO"))
logger.debug("single GTiff profile: %s", self._profile)
self.in_memory = (self.in_memory if self.in_memory is False else
height * width < IN_MEMORY_THRESHOLD)
# set up rasterio
if path_exists(self.path):
if self.output_params["mode"] != "overwrite":
raise MapcheteConfigError(
"single GTiff file already exists, use overwrite mode to replace"
)
else:
logger.debug("remove existing file: %s", self.path)
os.remove(self.path)
# create output directory if necessary
makedirs(os.path.dirname(self.path))
logger.debug("open output file: %s", self.path)
self._ctx = ExitStack()
# (1) use memfile if output is remote or COG
if self.cog or path_is_remote(self.path):
if self.in_memory:
self._memfile = self._ctx.enter_context(MemoryFile())
self.dst = self._ctx.enter_context(
self._memfile.open(**self._profile))
else:
# in case output raster is too big, use tempfile on disk
self._tempfile = self._ctx.enter_context(NamedTemporaryFile())
self.dst = self._ctx.enter_context(
rasterio.open(self._tempfile.name, "w+", **self._profile))
else:
self.dst = self._ctx.enter_context(
rasterio.open(self.path, "w+", **self._profile))
def prepare(self, process_area=None, **kwargs):
bounds = snap_bounds(
bounds=Bounds(*process_area.intersection(
box(*self.output_params["delimiters"]
["effective_bounds"])).bounds),
pyramid=self.pyramid,
zoom=self.zoom) if process_area else self.output_params[
"delimiters"]["effective_bounds"]
height = math.ceil((bounds.top - bounds.bottom) /
self.pyramid.pixel_x_size(self.zoom))
width = math.ceil((bounds.right - bounds.left) /
self.pyramid.pixel_x_size(self.zoom))
logger.debug("output raster bounds: %s", bounds)
logger.debug("output raster shape: %s, %s", height, width)
self._profile = dict(
GTIFF_DEFAULT_PROFILE,
driver="GTiff",
transform=Affine(self.pyramid.pixel_x_size(self.zoom), 0,
bounds.left, 0,
-self.pyramid.pixel_y_size(self.zoom),
bounds.top),
height=height,
width=width,
count=self.output_params["bands"],
crs=self.pyramid.crs,
**{
k: self.output_params.get(k, GTIFF_DEFAULT_PROFILE[k])
for k in GTIFF_DEFAULT_PROFILE.keys()
})
logger.debug("single GTiff profile: %s", self._profile)
if height * width > 20000 * 20000:
raise ValueError("output GeoTIFF too big")
# set up rasterio
if path_exists(self.path):
if self.output_params["mode"] != "overwrite":
raise MapcheteConfigError(
"single GTiff file already exists, use overwrite mode to replace"
)
else:
logger.debug("remove existing file: %s", self.path)
os.remove(self.path)
logger.debug("open output file: %s", self.path)
self.rio_file = rasterio.open(self.path, "w+", **self._profile)
def test_create_mosaic_antimeridian():
"""Create mosaic using tiles on opposing antimeridian sides."""
zoom = 5
row = 0
pixelbuffer = 5
tp = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
west = tp.tile(zoom, row, 0)
east = tp.tile(zoom, row, tp.matrix_width(zoom) - 1)
mosaic = create_mosaic([(west, np.ones(west.shape).astype("uint8")),
(east, np.ones(east.shape).astype("uint8") * 2)])
assert isinstance(mosaic, ReferencedRaster)
# Huge array gets initialized because the two tiles are on opposing sides of the
# projection area. The below test should pass if the tiles are stitched together next
# to each other.
assert mosaic.data.shape == (1, west.height,
west.width * 2 - 2 * pixelbuffer)
assert mosaic.data[0][0][0] == 2
assert mosaic.data[0][0][-1] == 1
# If tiles from opposing sides from Antimeridian are mosaicked it will happen that the
# output mosaic exceeds the CRS bounds (obviously). In such a case the mosaicking
# function shall make sure that the larger part of the output mosaic shall be inside
# the CRS bounds.
# (1) mosaic crosses Antimeridian in the West, larger part is on Western hemisphere:
tiles_ids = [
# Western hemisphere tiles
(zoom, row, 0),
(zoom, row, 1),
# Eastern hemisphere tile
(zoom, row, tp.matrix_width(zoom) - 1),
]
tiles = [(tp.tile(*tile_id), np.ones(tp.tile(*tile_id).shape))
for tile_id in tiles_ids]
mosaic = create_mosaic(tiles)
control_bounds = Bounds(
# Eastern tile has to be shifted
-(360 - tp.tile(*tiles_ids[2]).left),
tp.tile(*tiles_ids[2]).bottom,
tp.tile(*tiles_ids[1]).right,
tp.tile(*tiles_ids[1]).top,
)
assert mosaic.bounds == control_bounds
# (2) mosaic crosses Antimeridian in the West, larger part is on Eastern hemisphere:
tiles_ids = [
# Western hemisphere tile
(zoom, row, 0),
# Eastern hemisphere tiles
(zoom, row, tp.matrix_width(zoom) - 1),
(zoom, row, tp.matrix_width(zoom) - 2),
]
tiles = [(tp.tile(*tile_id), np.ones(tp.tile(*tile_id).shape))
for tile_id in tiles_ids]
mosaic = create_mosaic(tiles)
control_bounds = Bounds(
tp.tile(*tiles_ids[2]).left,
tp.tile(*tiles_ids[2]).bottom,
# Western tile has to be shifted
360 + tp.tile(*tiles_ids[0]).right,
tp.tile(*tiles_ids[0]).top,
)
assert mosaic.bounds == control_bounds
def create_mosaic(tiles, nodata=0):
"""
Create a mosaic from tiles.
Tiles must be connected (also possible over Antimeridian), otherwise strange things
can happen!
Parameters
----------
tiles : iterable
an iterable containing tuples of a BufferedTile and an array
nodata : integer or float
raster nodata value to initialize the mosaic with (default: 0)
Returns
-------
mosaic : ReferencedRaster
"""
if isinstance(tiles, GeneratorType):
tiles = list(tiles)
elif not isinstance(tiles, list):
raise TypeError("tiles must be either a list or generator")
if not all([isinstance(pair, tuple) for pair in tiles]):
raise TypeError("tiles items must be tuples")
if not all([
all([isinstance(tile, BufferedTile),
isinstance(data, np.ndarray)]) for tile, data in tiles
]):
raise TypeError("tuples must be pairs of BufferedTile and array")
if len(tiles) == 0:
raise ValueError("tiles list is empty")
logger.debug("create mosaic from %s tile(s)", len(tiles))
# quick return if there is just one tile
if len(tiles) == 1:
tile, data = tiles[0]
return ReferencedRaster(data=data,
affine=tile.affine,
bounds=tile.bounds,
crs=tile.crs)
# assert all tiles have same properties
pyramid, resolution, dtype = _get_tiles_properties(tiles)
# just handle antimeridian on global pyramid types
shift = _shift_required(tiles)
logger.debug("shift: %s" % shift)
# determine mosaic shape and reference
m_left, m_bottom, m_right, m_top = None, None, None, None
for tile, data in tiles:
num_bands = data.shape[0] if data.ndim > 2 else 1
left, bottom, right, top = tile.bounds
if shift:
# shift by half of the grid width
left += pyramid.x_size / 2
right += pyramid.x_size / 2
# if tile is now shifted outside pyramid bounds, move within
if right > pyramid.right:
right -= pyramid.x_size
left -= pyramid.x_size
m_left = min([left, m_left]) if m_left is not None else left
m_bottom = min([bottom, m_bottom]) if m_bottom is not None else bottom
m_right = max([right, m_right]) if m_right is not None else right
m_top = max([top, m_top]) if m_top is not None else top
height = int(round((m_top - m_bottom) / resolution))
width = int(round((m_right - m_left) / resolution))
# initialize empty mosaic
mosaic = ma.MaskedArray(data=np.full((num_bands, height, width),
dtype=dtype,
fill_value=nodata),
mask=np.ones((num_bands, height, width)))
# create Affine
affine = Affine(resolution, 0, m_left, 0, -resolution, m_top)
# fill mosaic array with tile data
for tile, data in tiles:
data = prepare_array(data, nodata=nodata, dtype=dtype)
t_left, t_bottom, t_right, t_top = tile.bounds
if shift:
t_left += pyramid.x_size / 2
t_right += pyramid.x_size / 2
# if tile is now shifted outside pyramid bounds, move within
if t_right > pyramid.right:
t_right -= pyramid.x_size
t_left -= pyramid.x_size
minrow, maxrow, mincol, maxcol = bounds_to_ranges(
out_bounds=(t_left, t_bottom, t_right, t_top),
in_affine=affine,
in_shape=(height, width))
existing_data = mosaic[:, minrow:maxrow, mincol:maxcol]
existing_mask = mosaic.mask[:, minrow:maxrow, mincol:maxcol]
mosaic[:, minrow:maxrow,
mincol:maxcol] = np.where(data.mask, existing_data, data)
mosaic.mask[:, minrow:maxrow,
mincol:maxcol] = np.where(data.mask, existing_mask,
data.mask)
if shift:
# shift back output mosaic
m_left -= pyramid.x_size / 2
m_right -= pyramid.x_size / 2
# if mosaic crosses Antimeridian, make sure the mosaic output bounds are based on the
# hemisphere of the Antimeridian with the larger mosaic intersection
if m_left < pyramid.left or m_right > pyramid.right:
# mosaic crosses Antimeridian
logger.debug("mosaic crosses Antimeridian")
left_distance = abs(pyramid.left - m_left)
right_distance = abs(pyramid.left - m_right)
# per default, the mosaic is placed on the right side of the Antimeridian, so we
# only need to move the bounds in case the larger part of the mosaic is on the
# left side
if left_distance > right_distance:
m_left += pyramid.x_size
m_right += pyramid.x_size
logger.debug(Bounds(m_left, m_bottom, m_right, m_top))
return ReferencedRaster(data=mosaic,
affine=Affine(resolution, 0, m_left, 0,
-resolution, m_top),
bounds=Bounds(m_left, m_bottom, m_right, m_top),
crs=tile.crs)
def test_output_data(mp_tmpdir):
"""Check GeoTIFF as output data."""
output_params = dict(
grid="geodetic",
format="GeoTIFF",
path=mp_tmpdir,
pixelbuffer=0,
metatiling=1,
bands=1,
dtype="int16",
delimiters=dict(bounds=Bounds(-180.0, -90.0, 180.0, 90.0),
effective_bounds=Bounds(-180.439453125, -90.0,
180.439453125, 90.0),
zoom=[5],
process_bounds=Bounds(-180.0, -90.0, 180.0, 90.0)))
output = gtiff.OutputDataWriter(output_params)
assert output.path == mp_tmpdir
assert output.file_extension == ".tif"
tp = BufferedTilePyramid("geodetic")
tile = tp.tile(5, 5, 5)
# get_path
assert output.get_path(tile) == os.path.join(
*[mp_tmpdir, "5", "5", "5" + ".tif"])
# prepare_path
try:
temp_dir = os.path.join(*[mp_tmpdir, "5", "5"])
output.prepare_path(tile)
assert os.path.isdir(temp_dir)
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
# profile
assert isinstance(output.profile(tile), dict)
# write
try:
data = np.ones((1, ) + tile.shape) * 128
output.write(tile, data)
# tiles_exist
assert output.tiles_exist(tile)
# read
data = output.read(tile)
assert isinstance(data, np.ndarray)
assert not data[0].mask.any()
finally:
shutil.rmtree(mp_tmpdir, ignore_errors=True)
# read empty
try:
data = output.read(tile)
assert isinstance(data, np.ndarray)
assert data[0].mask.all()
finally:
shutil.rmtree(mp_tmpdir, ignore_errors=True)
# empty
try:
empty = output.empty(tile)
assert isinstance(empty, ma.MaskedArray)
assert not empty.any()
finally:
shutil.rmtree(mp_tmpdir, ignore_errors=True)
# deflate with predictor
try:
# with pytest.deprecated_call():
output_params.update(compress="deflate", predictor=2)
output = gtiff.OutputDataWriter(output_params)
assert output.profile(tile)["compress"] == "deflate"
assert output.profile(tile)["predictor"] == 2
finally:
shutil.rmtree(mp_tmpdir, ignore_errors=True)
# using deprecated "compression" property
try:
with pytest.deprecated_call():
output_params.update(compression="deflate", predictor=2)
output = gtiff.OutputDataWriter(output_params)
assert output.profile(tile)["compress"] == "deflate"
assert output.profile(tile)["predictor"] == 2
finally:
shutil.rmtree(mp_tmpdir, ignore_errors=True)