def test_cog_translate_web():
"""
Test Web-Optimized COG.
- Test COG size is a multiple of 256 (mercator tile size)
- Test COG bounds are aligned with mercator grid at max zoom
"""
runner = CliRunner()
with runner.isolated_filesystem():
web_profile = cog_profiles.get("raw")
web_profile.update({"blockxsize": 256, "blockysize": 256})
config = dict(GDAL_TIFF_OVR_BLOCKSIZE="128")
cog_translate(
raster_path_web,
"cogeo.tif",
web_profile,
quiet=True,
web_optimized=True,
config=config,
)
with rasterio.open(raster_path_web) as src_dst:
with rasterio.open("cogeo.tif") as out_dst:
blocks = list(set(out_dst.block_shapes))
assert len(blocks) == 1
ts = blocks[0][0]
assert not out_dst.width % ts
assert not out_dst.height % ts
max_zoom = get_max_zoom(out_dst)
bounds = list(
transform_bounds(
*[src_dst.crs, "epsg:4326"] + list(src_dst.bounds),
densify_pts=21
)
)
leftTile = mercantile.tile(bounds[0], bounds[3], max_zoom)
tbounds = mercantile.xy_bounds(leftTile)
west, north = tbounds.left, tbounds.top
assert out_dst.transform.xoff == west
assert out_dst.transform.yoff == north
rightTile = mercantile.tile(bounds[2], bounds[1], max_zoom)
tbounds = mercantile.xy_bounds(rightTile)
east, south = tbounds.right, tbounds.bottom
lrx = round(
out_dst.transform.xoff + out_dst.transform.a * out_dst.width, 6
)
lry = round(
out_dst.transform.yoff + out_dst.transform.e * out_dst.height, 6
)
assert lrx == round(east, 6)
assert lry == round(south, 6)
def test_cog_translate_webZooms():
"""
Test Web-Optimized COG.
- Test COG size is a multiple of 256 (mercator tile size)
- Test COG bounds are aligned with mercator grid at max zoom
- Test high resolution internal tiles are equal to mercator tile using
cogdumper and rio-tiler
- Test overview internal tiles are equal to mercator tile using
cogdumper and rio-tiler
"""
runner = CliRunner()
with runner.isolated_filesystem():
web_profile = cog_profiles.get("raw")
web_profile.update({"blockxsize": 256, "blockysize": 256})
config = dict(GDAL_TIFF_OVR_BLOCKSIZE="128")
cog_translate(
raster_path_north,
"cogeo.tif",
web_profile,
quiet=True,
web_optimized=True,
config=config,
)
with rasterio.open("cogeo.tif") as out_dst:
assert get_max_zoom(out_dst) == 8
cog_translate(
raster_path_north,
"cogeo.tif",
web_profile,
quiet=True,
web_optimized=True,
latitude_adjustment=False,
config=config,
)
with rasterio.open("cogeo.tif") as out_dst:
assert get_max_zoom(out_dst) == 10
def cog_translate(
source,
dst_path,
dst_kwargs,
indexes=None,
nodata=None,
dtype=None,
add_mask=None,
overview_level=None,
overview_resampling="nearest",
web_optimized=False,
latitude_adjustment=True,
resampling="nearest",
in_memory=None,
config=None,
allow_intermediate_compression=False,
forward_band_tags=False,
quiet=False,
):
"""
Create Cloud Optimized Geotiff.
Parameters
----------
source : str, PathLike object or rasterio.io.DatasetReader
A dataset path, URL or rasterio.io.DatasetReader object.
Will be opened in "r" mode.
dst_path : str or Path-like object
An output dataset path or or PathLike object.
Will be opened in "w" mode.
dst_kwargs: dict
Output dataset creation options.
indexes : tuple or int, optional
Raster band indexes to copy.
nodata, int, optional
Overwrite nodata masking values for input dataset.
dtype: str, optional
Overwrite output data type. Default will be the input data type.
add_mask, bool, optional
Force output dataset creation with a mask.
overview_level : int, optional (default: 6)
COGEO overview (decimation) level
overview_resampling : str, optional (default: "nearest")
Resampling algorithm for overviews
web_optimized: bool, option (default: False)
Create web-optimized cogeo.
latitude_adjustment: bool, option (default: True)
Use mercator meters for zoom calculation or ensure max zoom equality.
resampling : str, optional (default: "nearest")
Resampling algorithm.
in_memory: bool, optional
Force processing raster in memory (default: process in memory if small)
config : dict
Rasterio Env options.
allow_intermediate_compression: bool, optional (default: False)
Allow intermediate file compression to reduce memory/disk footprint.
Note: This could reduce the speed of the process.
Ref: https://github.com/cogeotiff/rio-cogeo/issues/103
forward_band_tags: bool, optional
Forward band tags to output bands.
Ref: https://github.com/cogeotiff/rio-cogeo/issues/19
quiet: bool, optional (default: False)
Mask processing steps.
"""
if isinstance(indexes, int):
indexes = (indexes, )
config = config or {}
with rasterio.Env(**config):
with ExitStack() as ctx:
if isinstance(source, (DatasetReader, DatasetWriter, WarpedVRT)):
src_dst = source
else:
src_dst = ctx.enter_context(rasterio.open(source))
meta = src_dst.meta
indexes = indexes if indexes else src_dst.indexes
nodata = nodata if nodata is not None else src_dst.nodata
dtype = dtype if dtype else src_dst.dtypes[0]
alpha = has_alpha_band(src_dst)
mask = has_mask_band(src_dst)
if not add_mask and (
(nodata is not None or alpha)
and dst_kwargs.get("compress") in ["JPEG", "jpeg"]):
warnings.warn(
"Using lossy compression with Nodata or Alpha band "
"can results in unwanted artefacts.",
LossyCompression,
)
tilesize = min(int(dst_kwargs["blockxsize"]),
int(dst_kwargs["blockysize"]))
if src_dst.width < tilesize or src_dst.height < tilesize:
tilesize = 2**int(
math.log(min(src_dst.width, src_dst.height), 2))
if tilesize < 64:
warnings.warn(
"Raster has dimension < 64px. Output COG cannot be tiled"
" and overviews cannot be added.",
IncompatibleBlockRasterSize,
)
dst_kwargs.pop("blockxsize", None)
dst_kwargs.pop("blockysize", None)
dst_kwargs.pop("tiled")
overview_level = 0
else:
warnings.warn(
"Block Size are bigger than raster sizes. "
"Setting blocksize to {}".format(tilesize),
IncompatibleBlockRasterSize,
)
dst_kwargs["blockxsize"] = tilesize
dst_kwargs["blockysize"] = tilesize
vrt_params = dict(add_alpha=True, dtype=dtype)
if nodata is not None:
vrt_params.update(
dict(nodata=nodata, add_alpha=False, src_nodata=nodata))
if alpha:
vrt_params.update(dict(add_alpha=False))
if web_optimized:
bounds = list(
transform_bounds(*[src_dst.crs, "epsg:4326"] +
list(src_dst.bounds),
densify_pts=21))
center = [(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2]
lat = 0 if latitude_adjustment else center[1]
max_zoom = get_max_zoom(src_dst, lat=lat, tilesize=tilesize)
extrema = tile_extrema(bounds, max_zoom)
w, n = mercantile.xy(*mercantile.ul(
extrema["x"]["min"], extrema["y"]["min"], max_zoom))
vrt_res = _meters_per_pixel(max_zoom, 0, tilesize=tilesize)
vrt_transform = Affine(vrt_res, 0, w, 0, -vrt_res, n)
vrt_width = (extrema["x"]["max"] -
extrema["x"]["min"]) * tilesize
vrt_height = (extrema["y"]["max"] -
extrema["y"]["min"]) * tilesize
vrt_params.update(
dict(
crs="epsg:3857",
transform=vrt_transform,
width=vrt_width,
height=vrt_height,
resampling=ResamplingEnums[resampling],
))
with WarpedVRT(src_dst, **vrt_params) as vrt_dst:
meta = vrt_dst.meta
meta["count"] = len(indexes)
if add_mask:
meta.pop("nodata", None)
meta.pop("alpha", None)
if (dst_kwargs.get("photometric", "").upper() == "YCBCR"
and meta["count"] == 1):
warnings.warn(
"PHOTOMETRIC=YCBCR not supported on a 1-band raster"
" and has been set to 'MINISBLACK'")
dst_kwargs["photometric"] = "MINISBLACK"
meta.update(**dst_kwargs)
if not allow_intermediate_compression:
meta.pop("compress", None)
meta.pop("photometric", None)
if in_memory is None:
in_memory = vrt_dst.width * vrt_dst.height < IN_MEMORY_THRESHOLD
if in_memory:
tmpfile = ctx.enter_context(MemoryFile())
tmp_dst = ctx.enter_context(tmpfile.open(**meta))
else:
tmpfile = ctx.enter_context(TemporaryRasterFile(dst_path))
tmp_dst = ctx.enter_context(
rasterio.open(tmpfile.name, "w", **meta))
# Transfer color interpolation
if len(indexes) == 1 and (vrt_dst.colorinterp[indexes[0] - 1]
is not ColorInterp.palette):
tmp_dst.colorinterp = [ColorInterp.gray]
else:
tmp_dst.colorinterp = [
vrt_dst.colorinterp[b - 1] for b in indexes
]
if tmp_dst.colorinterp[0] is ColorInterp.palette:
try:
tmp_dst.write_colormap(1, vrt_dst.colormap(1))
except ValueError:
warnings.warn(
"Dataset has `Palette` color interpretation"
" but is missing colormap information")
wind = list(tmp_dst.block_windows(1))
if not quiet:
click.echo("Reading input: {}".format(source), err=True)
fout = os.devnull if quiet else sys.stderr
with click.progressbar(wind,
length=len(wind),
file=fout,
show_percent=True) as windows:
for ij, w in windows:
matrix = vrt_dst.read(window=w, indexes=indexes)
tmp_dst.write(matrix, window=w)
if add_mask or mask:
# Cast mask to uint8 to fix rasterio 1.1.2 error (ref #115)
mask_value = vrt_dst.dataset_mask(
window=w).astype("uint8")
tmp_dst.write_mask(mask_value, window=w)
if overview_level is None:
overview_level = get_maximum_overview_level(
vrt_dst, tilesize)
if not quiet and overview_level:
click.echo("Adding overviews...", err=True)
overviews = [2**j for j in range(1, overview_level + 1)]
tmp_dst.build_overviews(overviews,
ResamplingEnums[overview_resampling])
if not quiet:
click.echo("Updating dataset tags...", err=True)
for i, b in enumerate(indexes):
tmp_dst.set_band_description(i + 1,
src_dst.descriptions[b - 1])
if forward_band_tags:
tmp_dst.update_tags(i + 1, **src_dst.tags(b))
tags = src_dst.tags()
tags.update(
dict(
OVR_RESAMPLING_ALG=ResamplingEnums[overview_resampling]
.name.upper()))
tmp_dst.update_tags(**tags)
tmp_dst._set_all_scales(
[vrt_dst.scales[b - 1] for b in indexes])
tmp_dst._set_all_offsets(
[vrt_dst.offsets[b - 1] for b in indexes])
if not quiet:
click.echo("Writing output to: {}".format(dst_path),
err=True)
copy(tmp_dst, dst_path, copy_src_overviews=True, **dst_kwargs)
def test_cog_translate_Internal():
"""
Test Web-Optimized COG.
- Test COG size is a multiple of 256 (mercator tile size)
- Test COG bounds are aligned with mercator grid at max zoom
- Test high resolution internal tiles are equal to mercator tile using
cogdumper and rio-tiler
- Test overview internal tiles are equal to mercator tile using
cogdumper and rio-tiler
"""
from cogdumper.cog_tiles import COGTiff
from cogdumper.filedumper import Reader as FileReader
runner = CliRunner()
with runner.isolated_filesystem():
web_profile = cog_profiles.get("raw")
web_profile.update({"blockxsize": 256, "blockysize": 256})
config = dict(GDAL_TIFF_OVR_BLOCKSIZE="128")
cog_translate(
raster_path_web,
"cogeo.tif",
web_profile,
quiet=True,
web_optimized=True,
config=config,
)
with rasterio.open(raster_path_web) as src_dst:
with rasterio.open("cogeo.tif") as out_dst:
blocks = list(set(out_dst.block_shapes))
assert len(blocks) == 1
ts = blocks[0][0]
assert not out_dst.width % ts
assert not out_dst.height % ts
max_zoom = get_max_zoom(out_dst)
bounds = list(
transform_bounds(
*[src_dst.crs, "epsg:4326"] + list(src_dst.bounds),
densify_pts=21
)
)
leftTile = mercantile.tile(bounds[0], bounds[3], max_zoom)
tbounds = mercantile.xy_bounds(leftTile)
west, north = tbounds.left, tbounds.top
assert out_dst.transform.xoff == west
assert out_dst.transform.yoff == north
rightTile = mercantile.tile(bounds[2], bounds[1], max_zoom)
tbounds = mercantile.xy_bounds(rightTile)
east, south = tbounds.right, tbounds.bottom
lrx = round(
out_dst.transform.xoff + out_dst.transform.a * out_dst.width, 6
)
lry = round(
out_dst.transform.yoff + out_dst.transform.e * out_dst.height, 6
)
assert lrx == round(east, 6)
assert lry == round(south, 6)
with open("cogeo.tif", "rb") as out_body:
reader = FileReader(out_body)
cog = COGTiff(reader.read)
# High resolution
# Top Left tile
mime_type, tile = cog.get_tile(0, 0, 0)
tile_length = 256 * 256 * 3
t = struct.unpack_from("{}b".format(tile_length), tile)
arr = numpy.array(t).reshape(256, 256, 3).astype(numpy.uint8)
arr = numpy.transpose(arr, [2, 0, 1])
tbounds = mercantile.xy_bounds(leftTile)
data, mask = tile_read(
"cogeo.tif", tbounds, 256, resampling_method="nearest"
)
numpy.testing.assert_array_equal(data, arr)
# Bottom right tile
mime_type, tile = cog.get_tile(4, 3, 0)
tile_length = 256 * 256 * 3
t = struct.unpack_from("{}b".format(tile_length), tile)
arr = numpy.array(t).reshape(256, 256, 3).astype(numpy.uint8)
arr = numpy.transpose(arr, [2, 0, 1])
tbounds = mercantile.xy_bounds(rightTile)
data, mask = tile_read(
"cogeo.tif", tbounds, 256, resampling_method="nearest"
)
numpy.testing.assert_array_equal(data, arr)
# Low resolution (overview 1)
# Top Left tile
# NOTE: overview internal tile size is 128px
# We need to stack two internal tiles to compare with
# the 256px mercator tile fetched by rio-tiler
# ref: https://github.com/cogeotiff/rio-cogeo/issues/60
mime_type, tile = cog.get_tile(1, 0, 1)
tile_length = 128 * 128 * 3
t = struct.unpack_from("{}b".format(tile_length), tile)
arr1 = numpy.array(t).reshape(128, 128, 3).astype(numpy.uint8)
arr1 = numpy.transpose(arr1, [2, 0, 1])
mime_type, tile = cog.get_tile(2, 0, 1)
tile_length = 128 * 128 * 3
t = struct.unpack_from("{}b".format(tile_length), tile)
arr2 = numpy.array(t).reshape(128, 128, 3).astype(numpy.uint8)
arr2 = numpy.transpose(arr2, [2, 0, 1])
arr = numpy.dstack((arr1, arr2))
lowTile = mercantile.Tile(118594, 60034, 17)
tbounds = mercantile.xy_bounds(lowTile)
data, mask = tile_read(
"cogeo.tif", tbounds, 256, resampling_method="nearest"
)
data = data[:, 128:, :]
numpy.testing.assert_array_equal(data, arr)
def cog_translate(
src_path,
dst_path,
dst_kwargs,
indexes=None,
nodata=None,
add_mask=None,
overview_level=None,
overview_resampling="nearest",
web_optimized=False,
latitude_adjustment=True,
resampling="nearest",
in_memory=None,
config=None,
quiet=False,
):
"""
Create Cloud Optimized Geotiff.
Parameters
----------
src_path : str or PathLike object
A dataset path or URL. Will be opened in "r" mode.
dst_path : str or Path-like object
An output dataset path or or PathLike object.
Will be opened in "w" mode.
dst_kwargs: dict
Output dataset creation options.
indexes : tuple, int, optional
Raster band indexes to copy.
nodata, int, optional
Overwrite nodata masking values for input dataset.
add_mask, bool, optional
Force output dataset creation with a mask.
overview_level : int, optional (default: 6)
COGEO overview (decimation) level
overview_resampling : str, optional (default: "nearest")
Resampling algorithm for overviews
web_optimized: bool, option (default: False)
Create web-optimized cogeo.
latitude_adjustment: bool, option (default: True)
Use mercator meters for zoom calculation or ensure max zoom equality.
resampling : str, optional (default: "nearest")
Resampling algorithm.
in_memory: bool, optional
Force processing raster in memory (default: process in memory if small)
config : dict
Rasterio Env options.
quiet: bool, optional (default: False)
Mask processing steps.
"""
config = config or {}
with rasterio.Env(**config):
with rasterio.open(src_path) as src_dst:
meta = src_dst.meta
indexes = indexes if indexes else src_dst.indexes
nodata = nodata if nodata is not None else src_dst.nodata
alpha = has_alpha_band(src_dst)
mask = has_mask_band(src_dst)
if not add_mask and (
(nodata is not None or alpha)
and dst_kwargs.get("compress") in ["JPEG", "jpeg"]):
warnings.warn(
"Using lossy compression with Nodata or Alpha band "
"can results in unwanted artefacts.",
LossyCompression,
)
tilesize = min(int(dst_kwargs["blockxsize"]),
int(dst_kwargs["blockysize"]))
if src_dst.width < tilesize or src_dst.height < tilesize:
tilesize = 2**int(
math.log(min(src_dst.width, src_dst.height), 2))
if tilesize < 64:
warnings.warn(
"Raster has dimension < 64px. Output COG cannot be tiled"
" and overviews cannot be added.",
IncompatibleBlockRasterSize,
)
dst_kwargs.pop("blockxsize", None)
dst_kwargs.pop("blockysize", None)
dst_kwargs.pop("tiled")
overview_level = 0
else:
warnings.warn(
"Block Size are bigger than raster sizes. "
"Setting blocksize to {}".format(tilesize),
IncompatibleBlockRasterSize,
)
dst_kwargs["blockxsize"] = tilesize
dst_kwargs["blockysize"] = tilesize
vrt_params = dict(add_alpha=True)
if nodata is not None:
vrt_params.update(
dict(nodata=nodata, add_alpha=False, src_nodata=nodata))
if alpha:
vrt_params.update(dict(add_alpha=False))
if web_optimized:
bounds = list(
transform_bounds(*[src_dst.crs, "epsg:4326"] +
list(src_dst.bounds),
densify_pts=21))
center = [(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2]
lat = 0 if latitude_adjustment else center[1]
max_zoom = get_max_zoom(src_dst, lat=lat, tilesize=tilesize)
extrema = tile_extrema(bounds, max_zoom)
w, n = mercantile.xy(*mercantile.ul(
extrema["x"]["min"], extrema["y"]["min"], max_zoom))
vrt_res = _meters_per_pixel(max_zoom, 0, tilesize=tilesize)
vrt_transform = Affine(vrt_res, 0, w, 0, -vrt_res, n)
vrt_width = (extrema["x"]["max"] -
extrema["x"]["min"]) * tilesize
vrt_height = (extrema["y"]["max"] -
extrema["y"]["min"]) * tilesize
vrt_params.update(
dict(
crs="epsg:3857",
transform=vrt_transform,
width=vrt_width,
height=vrt_height,
resampling=ResamplingEnums[resampling],
))
with WarpedVRT(src_dst, **vrt_params) as vrt_dst:
meta = vrt_dst.meta
meta["count"] = len(indexes)
if add_mask:
meta.pop("nodata", None)
meta.pop("alpha", None)
meta.update(**dst_kwargs)
meta.pop("compress", None)
meta.pop("photometric", None)
if in_memory is None:
in_memory = vrt_dst.width * vrt_dst.height < IN_MEMORY_THRESHOLD
with ExitStack() as ctx:
if in_memory:
tmpfile = ctx.enter_context(MemoryFile())
tmp_dst = ctx.enter_context(tmpfile.open(**meta))
else:
tmpfile = ctx.enter_context(
TemporaryRasterFile(dst_path))
tmp_dst = ctx.enter_context(
rasterio.open(tmpfile.name, "w", **meta))
wind = list(tmp_dst.block_windows(1))
if not quiet:
click.echo("Reading input: {}".format(src_path),
err=True)
fout = os.devnull if quiet else sys.stderr
with click.progressbar(wind,
length=len(wind),
file=fout,
show_percent=True) as windows:
for ij, w in windows:
matrix = vrt_dst.read(window=w, indexes=indexes)
tmp_dst.write(matrix, window=w)
if add_mask or mask:
mask_value = vrt_dst.dataset_mask(window=w)
tmp_dst.write_mask(mask_value, window=w)
if overview_level is None:
overview_level = get_maximum_overview_level(
vrt_dst, tilesize)
if not quiet and overview_level:
click.echo("Adding overviews...", err=True)
overviews = [2**j for j in range(1, overview_level + 1)]
tmp_dst.build_overviews(
overviews, ResamplingEnums[overview_resampling])
if not quiet:
click.echo("Updating dataset tags...", err=True)
for i, b in enumerate(indexes):
tmp_dst.set_band_description(
i + 1, src_dst.descriptions[b - 1])
tags = src_dst.tags()
tags.update(
dict(OVR_RESAMPLING_ALG=ResamplingEnums[
overview_resampling].name.upper()))
tmp_dst.update_tags(**tags)
if not quiet:
click.echo("Writing output to: {}".format(dst_path),
err=True)
copy(tmp_dst,
dst_path,
copy_src_overviews=True,
**dst_kwargs)