def _prefered_compression_method() -> str:
if not GDALVersion.runtime().at_least('2.3'):
return 'DEFLATE'
# check if we can use ZSTD (fails silently for GDAL < 2.3)
dummy_profile = dict(driver='GTiff',
height=1,
width=1,
count=1,
dtype='uint8')
try:
with warnings.catch_warnings():
warnings.simplefilter('ignore', NotGeoreferencedWarning)
with MemoryFile() as memfile, memfile.open(compress='ZSTD',
**dummy_profile):
pass
except Exception as exc:
if 'missing codec' not in str(exc):
raise
else:
return 'ZSTD'
return 'DEFLATE'
def rasterio_crs(self):
"""Return rasterio CRS."""
import rasterio
from rasterio.env import GDALVersion
if GDALVersion.runtime().major < 3:
return rasterio.crs.CRS.from_wkt(
self.crs.to_wkt(WktVersion.WKT1_GDAL))
else:
return rasterio.crs.CRS.from_wkt(self.crs.to_wkt())
def transform_geom(src_crs,
dst_crs,
geom,
antimeridian_cutting=True,
antimeridian_offset=10.0,
precision=-1):
"""Transform geometry from source coordinate reference system into target.
Parameters
------------
src_crs: CRS or dict
Source coordinate reference system, in rasterio dict format.
Example: CRS({'init': 'EPSG:4326'})
dst_crs: CRS or dict
Target coordinate reference system.
geom: GeoJSON like dict object
antimeridian_cutting: bool, optional
If True, cut geometries at the antimeridian, otherwise geometries
will not be cut (default). If False and GDAL is 2.2.0 or newer
an exception is raised. Antimeridian cutting is always on as of
GDAL 2.2.0 but this could produce an unexpected geometry.
antimeridian_offset: float
Offset from the antimeridian in degrees (default: 10) within which
any geometries will be split.
precision: float
If >= 0, geometry coordinates will be rounded to this number of decimal
places after the transform operation, otherwise original coordinate
values will be preserved (default).
Returns
---------
out: GeoJSON like dict object
Transformed geometry in GeoJSON dict format
"""
if (GDALVersion.runtime().at_least('2.2') and not antimeridian_cutting):
raise GDALBehaviorChangeException(
"Antimeridian cutting is always enabled on GDAL 2.2.0 or "
"newer, which could produce a different geometry than expected.")
return _transform_geom(src_crs, dst_crs, geom, antimeridian_cutting,
antimeridian_offset, precision)
def test_gdalversion_class_runtime():
"""Test the version of GDAL from this runtime"""
GDALVersion.runtime().major >= 1
from rasterio._base import _transform
from rasterio.enums import Resampling
from rasterio.env import GDALVersion, ensure_env, require_gdal_version
from rasterio.errors import GDALBehaviorChangeException, TransformError
from rasterio.transform import array_bounds
from rasterio._warp import (
_calculate_default_transform,
_reproject,
_transform_bounds,
_transform_geom,
)
# Gauss (7) is not supported for warp
SUPPORTED_RESAMPLING = [r for r in Resampling if r.value < 7]
GDAL2_RESAMPLING = [r for r in Resampling if r.value > 7 and r.value <= 12]
if GDALVersion.runtime().at_least('2.0'):
SUPPORTED_RESAMPLING.extend(GDAL2_RESAMPLING)
# sum supported since GDAL 3.1
if GDALVersion.runtime().at_least('3.1'):
SUPPORTED_RESAMPLING.append(Resampling.sum)
# rms supported since GDAL 3.3
if GDALVersion.runtime().at_least('3.3'):
SUPPORTED_RESAMPLING.append(Resampling.rms)
@ensure_env
def transform(src_crs, dst_crs, xs, ys, zs=None):
"""Transform vectors from source to target coordinate reference system.
Transform vectors of x, y and optionally z from source
if not os.path.exists(path):
with zipfile.ZipFile(path, 'w') as zip:
for filename in ['white-gemini-iv.vrt',
'389225main_sw_1965_1024.jpg']:
zip.write(os.path.join(data_dir, filename), filename)
return path
class MockGeoInterface(object):
"""Tiny wrapper for GeoJSON to present an object with __geo_interface__ for testing"""
def __init__(self, geojson):
self.__geo_interface__ = geojson
# Define helpers to skip tests based on GDAL version
gdal_version = GDALVersion.runtime()
requires_only_gdal1 = pytest.mark.skipif(
gdal_version.major != 1,
reason="Only relevant for GDAL 1.x")
requires_gdal2 = pytest.mark.skipif(
not gdal_version.major >= 2,
reason="Requires GDAL 2.x")
requires_gdal21 = pytest.mark.skipif(
not gdal_version.at_least('2.1'),
reason="Requires GDAL 2.1.x")
requires_gdal22 = pytest.mark.skipif(
not gdal_version.at_least('2.2'),
def test_warp_resampling(runner, path_rgb_byte_tif, tmpdir, method):
"""Resampling methods supported by this version of GDAL should run
successfully"""
outputname = str(tmpdir.join('test.tif'))
result = runner.invoke(main_group, [
'warp', path_rgb_byte_tif, outputname,
'--dst-crs', 'epsg:3857',
'--resampling', method.name])
print(result.output)
assert result.exit_code == 0
@pytest.mark.skipif(
GDALVersion.runtime().at_least('2.0'),
reason="Test only applicable to GDAL < 2.0")
@pytest.mark.parametrize("method", GDAL2_RESAMPLING)
def test_warp_resampling_not_yet_supported(
runner, path_rgb_byte_tif, tmpdir, method):
"""Resampling methods not yet supported should fail with error"""
outputname = str(tmpdir.join('test.tif'))
result = runner.invoke(main_group, [
'warp', path_rgb_byte_tif, outputname,
'--dst-crs', 'epsg:3857',
'--resampling', method.name])
assert result.exit_code == 2
assert 'Invalid value for "--resampling"' in result.output
def cog_translate( # noqa: C901
source: Union[str, pathlib.PurePath, DatasetReader, DatasetWriter,
WarpedVRT],
dst_path: Union[str, pathlib.PurePath],
dst_kwargs: Dict,
indexes: Optional[Sequence[int]] = None,
nodata: Optional[Union[str, int, float]] = None,
dtype: Optional[str] = None,
add_mask: bool = False,
overview_level: Optional[int] = None,
overview_resampling: str = "nearest",
web_optimized: bool = False,
tms: Optional[morecantile.TileMatrixSet] = None,
zoom_level_strategy: str = "auto",
zoom_level: Optional[int] = None,
aligned_levels: Optional[int] = None,
resampling: str = "nearest",
in_memory: Optional[bool] = None,
config: Optional[Dict] = None,
allow_intermediate_compression: bool = False,
forward_band_tags: bool = False,
quiet: bool = False,
temporary_compression: str = "DEFLATE",
colormap: Optional[Dict] = None,
additional_cog_metadata: Optional[Dict] = None,
use_cog_driver: bool = False,
):
"""
Create Cloud Optimized Geotiff.
Parameters
----------
source : str, PathLike object or rasterio.io.DatasetReader
A dataset path, URL or rasterio.io.DatasetReader object.
Will be opened in "r" mode.
dst_path : str or PathLike object
An output dataset path or or PathLike object.
Will be opened in "w" mode.
dst_kwargs: dict
Output dataset creation options.
indexes : tuple or int, optional
Raster band indexes to copy.
nodata, int, optional
Overwrite nodata masking values for input dataset.
dtype: str, optional
Overwrite output data type. Default will be the input data type.
add_mask, bool, optional
Force output dataset creation with a mask.
overview_level : int, optional (default: None)
COGEO overview (decimation) level. By default, inferred from data size.
overview_resampling : str, optional (default: "nearest")
Resampling algorithm for overviews
web_optimized: bool, optional (default: False)
Create web-optimized cogeo.
tms: morecantile.TileMatrixSet, optional (default: "WebMercatorQuad")
TileMatrixSet to use for reprojection, resolution and alignment.
zoom_level_strategy: str, optional (default: auto)
Strategy to determine zoom level (same as in GDAL 3.2).
LOWER will select the zoom level immediately below the theoretical computed non-integral zoom level, leading to subsampling.
On the contrary, UPPER will select the immediately above zoom level, leading to oversampling.
Defaults to AUTO which selects the closest zoom level.
ref: https://gdal.org/drivers/raster/cog.html#raster-cog
zoom_level: int, optional.
Zoom level number (starting at 0 for coarsest zoom level). If this option is specified, `--zoom-level-strategy` is ignored.
aligned_levels: int, optional.
Number of overview levels for which GeoTIFF tile and tiles defined in the tiling scheme match.
Default is to use the maximum overview levels. Note: GDAL use number of resolution levels instead of overview levels.
resampling : str, optional (default: "nearest")
Resampling algorithm.
in_memory: bool, optional
Force processing raster in memory (default: process in memory if small)
config : dict
Rasterio Env options.
allow_intermediate_compression: bool, optional (default: False)
Allow intermediate file compression to reduce memory/disk footprint.
Note: This could reduce the speed of the process.
Ref: https://github.com/cogeotiff/rio-cogeo/issues/103
forward_band_tags: bool, optional
Forward band tags to output bands.
Ref: https://github.com/cogeotiff/rio-cogeo/issues/19
quiet: bool, optional (default: False)
Mask processing steps.
temporary_compression: str, optional
Compression used for the intermediate file, default is deflate.
colormap: dict, optional
Overwrite or add a colormap to the output COG.
additional_cog_metadata: dict, optional
Additional dataset metadata to add to the COG.
use_cog_driver: bool, optional (default: False)
Use GDAL COG driver if set to True. COG driver is available starting with GDAL 3.1.
"""
tms = tms or morecantile.tms.get("WebMercatorQuad")
dst_kwargs = dst_kwargs.copy()
if isinstance(indexes, int):
indexes = (indexes, )
config = config or {}
with rasterio.Env(**config):
with ExitStack() as ctx:
if isinstance(source, (DatasetReader, DatasetWriter, WarpedVRT)):
src_dst = source
else:
src_dst = ctx.enter_context(rasterio.open(source))
meta = src_dst.meta
indexes = indexes if indexes else src_dst.indexes
nodata = nodata if nodata is not None else src_dst.nodata
dtype = dtype if dtype else src_dst.dtypes[0]
alpha = utils.has_alpha_band(src_dst)
mask = utils.has_mask_band(src_dst)
if colormap and len(indexes) > 1:
raise IncompatibleOptions(
"Cannot add a colormap for multiple bands data.")
if not add_mask and (
(nodata is not None or alpha)
and dst_kwargs.get("compress", "").lower() == "jpeg"):
warnings.warn(
"Nodata/Alpha band will be translated to an internal mask band.",
)
add_mask = True
indexes = (utils.non_alpha_indexes(src_dst)
if len(indexes) not in [1, 3] else indexes)
tilesize = min(int(dst_kwargs["blockxsize"]),
int(dst_kwargs["blockysize"]))
if src_dst.width < tilesize or src_dst.height < tilesize:
tilesize = 2**int(
math.log(min(src_dst.width, src_dst.height), 2))
if tilesize < 64:
warnings.warn(
"Raster has dimension < 64px. Output COG cannot be tiled"
" and overviews cannot be added.",
IncompatibleBlockRasterSize,
)
dst_kwargs.pop("blockxsize", None)
dst_kwargs.pop("blockysize", None)
dst_kwargs.pop("tiled")
overview_level = 0
else:
warnings.warn(
"Block Size are bigger than raster sizes. "
"Setting blocksize to {}".format(tilesize),
IncompatibleBlockRasterSize,
)
dst_kwargs["blockxsize"] = tilesize
dst_kwargs["blockysize"] = tilesize
vrt_params = {
"add_alpha": True,
"dtype": dtype,
"width": src_dst.width,
"height": src_dst.height,
"resampling": ResamplingEnums[resampling],
}
if nodata is not None:
vrt_params.update(
dict(nodata=nodata, add_alpha=False, src_nodata=nodata))
if alpha:
vrt_params.update(dict(add_alpha=False))
if web_optimized and not use_cog_driver:
params = utils.get_web_optimized_params(
src_dst,
zoom_level_strategy=zoom_level_strategy,
zoom_level=zoom_level,
aligned_levels=aligned_levels,
tms=tms,
)
vrt_params.update(**params)
with WarpedVRT(src_dst, **vrt_params) as vrt_dst:
meta = vrt_dst.meta
meta["count"] = len(indexes)
if add_mask:
meta.pop("nodata", None)
meta.pop("alpha", None)
if (dst_kwargs.get("photometric", "").upper() == "YCBCR"
and meta["count"] == 1):
warnings.warn(
"PHOTOMETRIC=YCBCR not supported on a 1-band raster"
" and has been set to 'MINISBLACK'")
dst_kwargs["photometric"] = "MINISBLACK"
meta.update(**dst_kwargs)
meta.pop("compress", None)
meta.pop("photometric", None)
if allow_intermediate_compression:
meta["compress"] = temporary_compression
if in_memory is None:
in_memory = vrt_dst.width * vrt_dst.height < IN_MEMORY_THRESHOLD
if in_memory:
tmpfile = ctx.enter_context(MemoryFile())
tmp_dst = ctx.enter_context(tmpfile.open(**meta))
else:
tmpfile = ctx.enter_context(TemporaryRasterFile(dst_path))
tmp_dst = ctx.enter_context(
rasterio.open(tmpfile.name, "w", **meta))
# Transfer color interpolation
if len(indexes) == 1 and (vrt_dst.colorinterp[indexes[0] - 1]
is not ColorInterp.palette):
tmp_dst.colorinterp = [ColorInterp.gray]
else:
tmp_dst.colorinterp = [
vrt_dst.colorinterp[b - 1] for b in indexes
]
if colormap:
if tmp_dst.colorinterp[0] is not ColorInterp.palette:
tmp_dst.colorinterp = [ColorInterp.palette]
warnings.warn(
"Dataset color interpretation was set to `Palette`"
)
tmp_dst.write_colormap(1, colormap)
elif tmp_dst.colorinterp[0] is ColorInterp.palette:
try:
tmp_dst.write_colormap(1, vrt_dst.colormap(1))
except ValueError:
warnings.warn(
"Dataset has `Palette` color interpretation"
" but is missing colormap information")
wind = list(tmp_dst.block_windows(1))
if not quiet:
click.echo("Reading input: {}".format(source), err=True)
fout = ctx.enter_context(open(os.devnull,
"w")) if quiet else sys.stderr
with click.progressbar(
wind, file=fout,
show_percent=True) as windows: # type: ignore
for _, w in windows:
matrix = vrt_dst.read(window=w, indexes=indexes)
tmp_dst.write(matrix, window=w)
if add_mask or mask:
# Cast mask to uint8 to fix rasterio 1.1.2 error (ref #115)
mask_value = vrt_dst.dataset_mask(
window=w).astype("uint8")
tmp_dst.write_mask(mask_value, window=w)
if overview_level is None:
overview_level = get_maximum_overview_level(
vrt_dst.width, vrt_dst.height, minsize=tilesize)
if not quiet and overview_level:
click.echo("Adding overviews...", err=True)
overviews = [2**j for j in range(1, overview_level + 1)]
tmp_dst.build_overviews(overviews,
ResamplingEnums[overview_resampling])
if not quiet:
click.echo("Updating dataset tags...", err=True)
for i, b in enumerate(indexes):
tmp_dst.set_band_description(i + 1,
src_dst.descriptions[b - 1])
if forward_band_tags:
tmp_dst.update_tags(i + 1, **src_dst.tags(b))
tags = src_dst.tags()
tags.update(
dict(
OVR_RESAMPLING_ALG=ResamplingEnums[overview_resampling]
.name.upper()))
if additional_cog_metadata:
tags.update(**additional_cog_metadata)
if web_optimized and not use_cog_driver:
default_zoom = tms.zoom_for_res(
max(tmp_dst.res),
max_z=30,
zoom_level_strategy=zoom_level_strategy,
)
dst_kwargs.update({
"@TILING_SCHEME_NAME":
tms.identifier,
"@TILING_SCHEME_ZOOM_LEVEL":
zoom_level if zoom_level is not None else default_zoom,
})
if aligned_levels:
dst_kwargs.update(
{"@TILING_SCHEME_ALIGNED_LEVELS": aligned_levels})
tmp_dst.update_tags(**tags)
tmp_dst._set_all_scales(
[vrt_dst.scales[b - 1] for b in indexes])
tmp_dst._set_all_offsets(
[vrt_dst.offsets[b - 1] for b in indexes])
if not quiet:
click.echo("Writing output to: {}".format(dst_path),
err=True)
if use_cog_driver:
if not GDALVersion.runtime().at_least("3.1"):
raise Exception(
"GDAL 3.1 or above required to use the COG driver."
)
dst_kwargs["driver"] = "COG"
if web_optimized:
dst_kwargs["TILING_SCHEME"] = (
"GoogleMapsCompatible" if tms.identifier
== "WebMercatorQuad" else tms.identifier)
if zoom_level is not None:
if not GDALVersion.runtime().at_least("3.5"):
warnings.warn(
"ZOOM_LEVEL option is only available with GDAL >3.5."
)
dst_kwargs["ZOOM_LEVEL"] = zoom_level
dst_kwargs["ZOOM_LEVEL_STRATEGY"] = zoom_level_strategy
if aligned_levels is not None:
# GDAL uses Number of resolution (not overviews)
# See https://github.com/OSGeo/gdal/issues/5336#issuecomment-1042946603
dst_kwargs["aligned_levels"] = aligned_levels + 1
if add_mask and dst_kwargs.get("compress", "") != "JPEG":
warnings.warn(
"With GDAL COG driver, mask band will be translated to an alpha band."
)
dst_kwargs["overview_resampling"] = overview_resampling
dst_kwargs["warp_resampling"] = resampling
dst_kwargs["blocksize"] = tilesize
dst_kwargs.pop("blockxsize", None)
dst_kwargs.pop("blockysize", None)
dst_kwargs.pop("tiled", None)
dst_kwargs.pop("interleave", None)
dst_kwargs.pop("photometric", None)
copy(tmp_dst, dst_path, **dst_kwargs)
else:
copy(tmp_dst,
dst_path,
copy_src_overviews=True,
**dst_kwargs)
from io import BytesIO
import logging
import pytest
import rasterio
from rasterio.io import MemoryFile, ZipMemoryFile
from rasterio.env import GDALVersion
logging.basicConfig(level=logging.DEBUG)
# Skip ENTIRE module if not GDAL >= 2.x.
# pytestmark is a keyword that instructs pytest to skip this module.
pytestmark = pytest.mark.skipif(
not GDALVersion.runtime().major >= 2,
reason="MemoryFile requires GDAL 2.x")
@pytest.fixture(scope='session')
def rgb_file_bytes(path_rgb_byte_tif):
"""Get the bytes of our RGB.bytes.tif file"""
return open(path_rgb_byte_tif, 'rb').read()
@pytest.fixture(scope='session')
def rgb_lzw_file_bytes():
"""Get the bytes of our RGB.bytes.tif file"""
return open('tests/data/rgb_lzw.tif', 'rb').read()
import logging
import os.path
from affine import Affine
import numpy
import pytest
import rasterio
from rasterio.io import MemoryFile, ZipMemoryFile
from rasterio.enums import MaskFlags
from rasterio.env import GDALVersion
from rasterio.shutil import copyfiles
# Skip ENTIRE module if not GDAL >= 2.x.
# pytestmark is a keyword that instructs pytest to skip this module.
pytestmark = pytest.mark.skipif(not GDALVersion.runtime().major >= 2,
reason="MemoryFile requires GDAL 2.x")
@pytest.fixture(scope='session')
def rgb_file_bytes(path_rgb_byte_tif):
"""Get the bytes of our RGB.bytes.tif file"""
return open(path_rgb_byte_tif, 'rb').read()
@pytest.fixture(scope='session')
def rgb_lzw_file_bytes():
"""Get the bytes of our RGB.bytes.tif file"""
return open('tests/data/rgb_lzw.tif', 'rb').read()
def check_raster_file(src_path: str) -> ValidationInfo: # pragma: no cover
"""
Implementation from
https://github.com/cogeotiff/rio-cogeo/blob/a07d914e2d898878417638bbc089179f01eb5b28/rio_cogeo/cogeo.py#L385
This function is the rasterio equivalent of
https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py
"""
errors: List[str] = []
warnings: List[str] = []
details: Dict[str, Any] = {}
if not GDALVersion.runtime().at_least('2.2'):
raise RuntimeError('GDAL 2.2 or above required')
config = dict(GDAL_DISABLE_READDIR_ON_OPEN='FALSE')
with rasterio.Env(**config):
with rasterio.open(src_path) as src:
if not src.driver == 'GTiff':
errors.append('The file is not a GeoTIFF')
return errors, warnings, details
if any(os.path.splitext(x)[-1] == '.ovr' for x in src.files):
errors.append(
'Overviews found in external .ovr file. They should be internal'
)
overviews = src.overviews(1)
if src.width > 512 and src.height > 512:
if not src.is_tiled:
errors.append(
'The file is greater than 512xH or 512xW, but is not tiled'
)
if not overviews:
warnings.append(
'The file is greater than 512xH or 512xW, it is recommended '
'to include internal overviews')
ifd_offset = int(src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1))
# Starting from GDAL 3.1, GeoTIFF and COG have ghost headers
# e.g:
# """
# GDAL_STRUCTURAL_METADATA_SIZE=000140 bytes
# LAYOUT=IFDS_BEFORE_DATA
# BLOCK_ORDER=ROW_MAJOR
# BLOCK_LEADER=SIZE_AS_UINT4
# BLOCK_TRAILER=LAST_4_BYTES_REPEATED
# KNOWN_INCOMPATIBLE_EDITION=NO
# """
#
# This header should be < 200bytes
if ifd_offset > 300:
errors.append(
f'The offset of the main IFD should be < 300. It is {ifd_offset} instead'
)
ifd_offsets = [ifd_offset]
details['ifd_offsets'] = {}
details['ifd_offsets']['main'] = ifd_offset
if overviews and overviews != sorted(overviews):
errors.append('Overviews should be sorted')
for ix, dec in enumerate(overviews):
# NOTE: Size check is handled in rasterio `src.overviews` methods
# https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx
# We just need to make sure the decimation level is > 1
if not dec > 1:
errors.append(
'Invalid Decimation {} for overview level {}'.format(
dec, ix))
# Check that the IFD of descending overviews are sorted by increasing
# offsets
ifd_offset = int(
src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1, ovr=ix))
ifd_offsets.append(ifd_offset)
details['ifd_offsets']['overview_{}'.format(ix)] = ifd_offset
if ifd_offsets[-1] < ifd_offsets[-2]:
if ix == 0:
errors.append(
'The offset of the IFD for overview of index {} is {}, '
'whereas it should be greater than the one of the main '
'image, which is at byte {}'.format(
ix, ifd_offsets[-1], ifd_offsets[-2]))
else:
errors.append(
'The offset of the IFD for overview of index {} is {}, '
'whereas it should be greater than the one of index {}, '
'which is at byte {}'.format(
ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2]))
block_offset = src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1)
data_offset = int(block_offset) if block_offset else 0
data_offsets = [data_offset]
details['data_offsets'] = {}
details['data_offsets']['main'] = data_offset
for ix, dec in enumerate(overviews):
block_offset = src.get_tag_item('BLOCK_OFFSET_0_0',
'TIFF',
bidx=1,
ovr=ix)
data_offset = int(block_offset) if block_offset else 0
data_offsets.append(data_offset)
details['data_offsets']['overview_{}'.format(ix)] = data_offset
if data_offsets[-1] != 0 and data_offsets[-1] < ifd_offsets[-1]:
if len(overviews) > 0:
errors.append(
'The offset of the first block of the smallest overview '
'should be after its IFD')
else:
errors.append(
'The offset of the first block of the image should '
'be after its IFD')
for i in range(len(data_offsets) - 2, 0, -1):
if data_offsets[i] < data_offsets[i + 1]:
errors.append(
'The offset of the first block of overview of index {} should '
'be after the one of the overview of index {}'.format(
i - 1, i))
if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]:
errors.append(
'The offset of the first block of the main resolution image '
'should be after the one of the overview of index {}'.
format(len(overviews) - 1))
for ix, dec in enumerate(overviews):
with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst:
if ovr_dst.width > 512 and ovr_dst.height > 512:
if not ovr_dst.is_tiled:
errors.append(
'Overview of index {} is not tiled'.format(ix))
return errors, warnings, details
def cog_validate(src_path):
"""
Validate Cloud Optimized Geotiff.
Parameters
----------
src_path : str or PathLike object
A dataset path or URL. Will be opened in "r" mode.
This script is the rasterio equivalent of
https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py
"""
errors = []
warnings = []
details = {}
if not GDALVersion.runtime().at_least("2.2"):
raise Exception("GDAL 2.2 or above required")
config = dict(GDAL_DISABLE_READDIR_ON_OPEN="FALSE")
with rasterio.Env(**config):
with rasterio.open(src_path) as src:
if not src.driver == "GTiff":
raise Exception("The file is not a GeoTIFF")
filelist = [os.path.basename(f) for f in src.files]
src_bname = os.path.basename(src_path)
if len(filelist) > 1 and src_bname + ".ovr" in filelist:
errors.append(
"Overviews found in external .ovr file. They should be internal"
)
overviews = src.overviews(1)
if src.width > 512 or src.height > 512:
if not src.is_tiled:
errors.append(
"The file is greater than 512xH or 512xW, but is not tiled"
)
if not overviews:
warnings.append(
"The file is greater than 512xH or 512xW, it is recommended "
"to include internal overviews")
ifd_offset = int(src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1))
ifd_offsets = [ifd_offset]
if ifd_offset not in (8, 16):
errors.append(
"The offset of the main IFD should be 8 for ClassicTIFF "
"or 16 for BigTIFF. It is {} instead".format(ifd_offset))
details["ifd_offsets"] = {}
details["ifd_offsets"]["main"] = ifd_offset
if overviews and overviews != sorted(overviews):
errors.append("Overviews should be sorted")
for ix, dec in enumerate(overviews):
# NOTE: Size check is handled in rasterio `src.overviews` methods
# https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx
# We just need to make sure the decimation level is > 1
if not dec > 1:
errors.append(
"Invalid Decimation {} for overview level {}".format(
dec, ix))
# Check that the IFD of descending overviews are sorted by increasing
# offsets
ifd_offset = int(
src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1, ovr=ix))
ifd_offsets.append(ifd_offset)
details["ifd_offsets"]["overview_{}".format(ix)] = ifd_offset
if ifd_offsets[-1] < ifd_offsets[-2]:
if ix == 0:
errors.append(
"The offset of the IFD for overview of index {} is {}, "
"whereas it should be greater than the one of the main "
"image, which is at byte {}".format(
ix, ifd_offsets[-1], ifd_offsets[-2]))
else:
errors.append(
"The offset of the IFD for overview of index {} is {}, "
"whereas it should be greater than the one of index {}, "
"which is at byte {}".format(
ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2]))
block_offset = int(
src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1))
if not block_offset:
errors.append("Missing BLOCK_OFFSET_0_0")
data_offset = int(block_offset) if block_offset else None
data_offsets = [data_offset]
details["data_offsets"] = {}
details["data_offsets"]["main"] = data_offset
for ix, dec in enumerate(overviews):
data_offset = int(
src.get_tag_item("BLOCK_OFFSET_0_0",
"TIFF",
bidx=1,
ovr=ix))
data_offsets.append(data_offset)
details["data_offsets"]["overview_{}".format(ix)] = data_offset
if data_offsets[-1] < ifd_offsets[-1]:
if len(overviews) > 0:
errors.append(
"The offset of the first block of the smallest overview "
"should be after its IFD")
else:
errors.append(
"The offset of the first block of the image should "
"be after its IFD")
for i in range(len(data_offsets) - 2, 0, -1):
if data_offsets[i] < data_offsets[i + 1]:
errors.append(
"The offset of the first block of overview of index {} should "
"be after the one of the overview of index {}".format(
i - 1, i))
if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]:
errors.append(
"The offset of the first block of the main resolution image "
"should be after the one of the overview of index {}".
format(len(overviews) - 1))
for ix, dec in enumerate(overviews):
with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst:
if ovr_dst.width >= 512 or ovr_dst.height >= 512:
if not ovr_dst.is_tiled:
errors.append(
"Overview of index {} is not tiled".format(ix))
if warnings:
click.secho("The following warnings were found:",
fg="yellow",
err=True)
for w in warnings:
click.echo("- " + w, err=True)
click.echo(err=True)
if errors:
click.secho("The following errors were found:", fg="red", err=True)
for e in errors:
click.echo("- " + e, err=True)
return False
return True
import logging
import os.path
from affine import Affine
import numpy
import pytest
import rasterio
from rasterio.io import MemoryFile, ZipMemoryFile
from rasterio.env import GDALVersion
# Skip ENTIRE module if not GDAL >= 2.x.
# pytestmark is a keyword that instructs pytest to skip this module.
pytestmark = pytest.mark.skipif(
not GDALVersion.runtime().major >= 2,
reason="MemoryFile requires GDAL 2.x")
@pytest.fixture(scope='session')
def rgb_file_bytes(path_rgb_byte_tif):
"""Get the bytes of our RGB.bytes.tif file"""
return open(path_rgb_byte_tif, 'rb').read()
@pytest.fixture(scope='session')
def rgb_lzw_file_bytes():
"""Get the bytes of our RGB.bytes.tif file"""
return open('tests/data/rgb_lzw.tif', 'rb').read()
def validate(src_path: str) -> bool:
"""Validate given cloud-optimized GeoTIFF"""
if not GDALVersion.runtime().at_least('2.2'):
raise RuntimeError('GDAL 2.2 or above required')
with rasterio.open(src_path) as src:
if not src.driver == 'GTiff':
# Not a GeoTIFF
return False
filelist = [os.path.basename(f) for f in src.files]
src_bname = os.path.basename(src_path)
if len(filelist) > 1 and src_bname + '.ovr' in filelist:
# Overviews found in external .ovr file. They should be internal
return False
overviews = src.overviews(1) or []
if src.width >= 512 or src.height >= 512:
if not src.is_tiled:
# The file is greater than 512xH or 512xW, but is not tiled
return False
if not overviews:
# The file is greater than 512xH or 512xW, but has no overviews
return False
ifd_offset = int(src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1))
ifd_offsets = [ifd_offset]
if ifd_offset not in (8, 16):
# The offset of the main IFD should be 8 for ClassicTIFF or 16 for BigTIFF
return False
if not overviews == sorted(overviews):
# Overviews should be sorted
return False
for ix, dec in enumerate(overviews):
if not dec > 1:
# Invalid Decimation
return False
# TODO: Check if the overviews are tiled
# NOTE: There is currently no way to do that with rasterio
# Check that the IFD of descending overviews are sorted by increasing offsets
ifd_offset = int(
src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1, ovr=ix))
ifd_offsets.append(ifd_offset)
if ifd_offsets[-1] < ifd_offsets[-2]:
return False
block_offset = int(src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF',
bidx=1))
if not block_offset:
return False
data_offset = int(block_offset)
data_offsets = [data_offset]
for ix, dec in enumerate(overviews):
data_offset = int(
src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1, ovr=ix))
data_offsets.append(data_offset)
if data_offsets[-1] < ifd_offsets[-1]:
return False
for i in range(len(data_offsets) - 2, 0, -1):
if data_offsets[i] < data_offsets[i + 1]:
return False
if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]:
return False
return True
"""Mapping of GDAL to Numpy data types.
Since 0.13 we are not importing numpy here and data types are strings.
Happily strings can be used throughout Numpy and so existing code will
not break.
"""
import numpy
from rasterio.env import GDALVersion
_GDAL_AT_LEAST_35 = GDALVersion.runtime().at_least("3.5")
bool_ = 'bool'
ubyte = uint8 = 'uint8'
sbyte = int8 = 'int8'
uint16 = 'uint16'
int16 = 'int16'
uint32 = 'uint32'
int32 = 'int32'
uint64 = 'uint32'
int64 = 'int64'
float32 = 'float32'
float64 = 'float64'
complex_ = 'complex'
complex64 = 'complex64'
complex128 = 'complex128'
complex_int16 = "complex_int16"
dtype_fwd = {
import os
import click
import numpy
from rasterio.enums import Resampling as ResamplingEnums
from rasterio.env import GDALVersion
from rasterio.rio import options
from rasterio.warp import SUPPORTED_RESAMPLING as WarpResampling
from rio_cogeo import __version__ as cogeo_version
from rio_cogeo.cogeo import cog_info, cog_translate, cog_validate
from rio_cogeo.profiles import cog_profiles
OverviewResampling = [r for r in ResamplingEnums if r.value < 8]
if GDALVersion.runtime().at_least("3.3"):
OverviewResampling.append(ResamplingEnums.rms)
IN_MEMORY_THRESHOLD = int(os.environ.get("IN_MEMORY_THRESHOLD", 10980 * 10980))
class BdxParamType(click.ParamType):
"""Band index type."""
name = "bidx"
def convert(self, value, param, ctx):
"""Validate and parse band index."""
try:
bands = [int(x) for x in value.split(",")]
assert all(b > 0 for b in bands)
def cog_validate( # noqa: C901
src_path: Union[str, pathlib.PurePath],
strict: bool = False,
quiet: bool = False) -> Tuple[bool, List[str], List[str]]:
"""
Validate Cloud Optimized Geotiff.
This script is the rasterio equivalent of
https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py
Parameters
----------
src_path: str or PathLike object
A dataset path or URL. Will be opened in "r" mode.
strict: bool
Treat warnings as errors
quiet: bool
Remove standard outputs
Returns
-------
is_valid: bool
True is src_path is a valid COG.
errors: list
List of validation errors.
warnings: list
List of validation warnings.
"""
if isinstance(src_path, str):
src_path = pathlib.Path(src_path)
errors = []
warnings = []
details: Dict[str, Any] = {}
if not GDALVersion.runtime().at_least("2.2"):
raise Exception("GDAL 2.2 or above required")
config = dict(GDAL_DISABLE_READDIR_ON_OPEN="FALSE")
with rasterio.Env(**config):
with rasterio.open(src_path) as src:
if not src.driver == "GTiff":
raise Exception("The file is not a GeoTIFF")
filelist = [pathlib.Path(f).name for f in src.files]
if len(filelist) > 1 and f"{src_path.name}.ovr" in filelist:
errors.append(
"Overviews found in external .ovr file. They should be internal"
)
overviews = src.overviews(1)
if src.width > 512 or src.height > 512:
if not src.is_tiled:
errors.append(
"The file is greater than 512xH or 512xW, but is not tiled"
)
if not overviews:
warnings.append(
"The file is greater than 512xH or 512xW, it is recommended "
"to include internal overviews")
ifd_offset = int(src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1))
# Starting from GDAL 3.1, GeoTIFF and COG have ghost headers
# e.g:
# """
# GDAL_STRUCTURAL_METADATA_SIZE=000140 bytes
# LAYOUT=IFDS_BEFORE_DATA
# BLOCK_ORDER=ROW_MAJOR
# BLOCK_LEADER=SIZE_AS_UINT4
# BLOCK_TRAILER=LAST_4_BYTES_REPEATED
# KNOWN_INCOMPATIBLE_EDITION=NO
# """
#
# This header should be < 200bytes
if ifd_offset > 300:
errors.append(
f"The offset of the main IFD should be < 300. It is {ifd_offset} instead"
)
ifd_offsets = [ifd_offset]
details["ifd_offsets"] = {}
details["ifd_offsets"]["main"] = ifd_offset
if overviews and overviews != sorted(overviews):
errors.append("Overviews should be sorted")
for ix, dec in enumerate(overviews):
# NOTE: Size check is handled in rasterio `src.overviews` methods
# https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx
# We just need to make sure the decimation level is > 1
if not dec > 1:
errors.append(
"Invalid Decimation {} for overview level {}".format(
dec, ix))
# Check that the IFD of descending overviews are sorted by increasing
# offsets
ifd_offset = int(
src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1, ovr=ix))
ifd_offsets.append(ifd_offset)
details["ifd_offsets"]["overview_{}".format(ix)] = ifd_offset
if ifd_offsets[-1] < ifd_offsets[-2]:
if ix == 0:
errors.append(
"The offset of the IFD for overview of index {} is {}, "
"whereas it should be greater than the one of the main "
"image, which is at byte {}".format(
ix, ifd_offsets[-1], ifd_offsets[-2]))
else:
errors.append(
"The offset of the IFD for overview of index {} is {}, "
"whereas it should be greater than the one of index {}, "
"which is at byte {}".format(
ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2]))
block_offset = int(
src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1))
if not block_offset:
errors.append("Missing BLOCK_OFFSET_0_0")
data_offset = int(block_offset) if block_offset else None
data_offsets = [data_offset]
details["data_offsets"] = {}
details["data_offsets"]["main"] = data_offset
for ix, dec in enumerate(overviews):
data_offset = int(
src.get_tag_item("BLOCK_OFFSET_0_0",
"TIFF",
bidx=1,
ovr=ix))
data_offsets.append(data_offset)
details["data_offsets"]["overview_{}".format(ix)] = data_offset
if data_offsets[-1] < ifd_offsets[-1]:
if len(overviews) > 0:
errors.append(
"The offset of the first block of the smallest overview "
"should be after its IFD")
else:
errors.append(
"The offset of the first block of the image should "
"be after its IFD")
for i in range(len(data_offsets) - 2, 0, -1):
if data_offsets[i] < data_offsets[i + 1]:
errors.append(
"The offset of the first block of overview of index {} should "
"be after the one of the overview of index {}".format(
i - 1, i))
if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]:
errors.append(
"The offset of the first block of the main resolution image "
"should be after the one of the overview of index {}".
format(len(overviews) - 1))
for ix, dec in enumerate(overviews):
with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst:
if ovr_dst.width >= 512 or ovr_dst.height >= 512:
if not ovr_dst.is_tiled:
errors.append(
"Overview of index {} is not tiled".format(ix))
if warnings and not quiet:
click.secho("The following warnings were found:",
fg="yellow",
err=True)
for w in warnings:
click.echo("- " + w, err=True)
click.echo(err=True)
if errors and not quiet:
click.secho("The following errors were found:", fg="red", err=True)
for e in errors:
click.echo("- " + e, err=True)
is_valid = False if errors or (warnings and strict) else True
return is_valid, errors, warnings
def test_gdalversion_class_runtime():
"""Test the version of GDAL from this runtime"""
GDALVersion.runtime().major >= 1
def test_warp_resampling(runner, path_rgb_byte_tif, tmpdir, method):
"""Resampling methods supported by this version of GDAL should run
successfully"""
outputname = str(tmpdir.join('test.tif'))
result = runner.invoke(main_group, [
'warp', path_rgb_byte_tif, outputname,
'--dst-crs', 'epsg:3857',
'--resampling', method.name])
print(result.output)
assert result.exit_code == 0
@pytest.mark.skipif(
GDALVersion.runtime().at_least('2.0'),
reason="Test only applicable to GDAL < 2.0")
@pytest.mark.parametrize("method", GDAL2_RESAMPLING)
def test_warp_resampling_not_yet_supported(
runner, path_rgb_byte_tif, tmpdir, method):
"""Resampling methods not yet supported should fail with error"""
outputname = str(tmpdir.join('test.tif'))
result = runner.invoke(main_group, [
'warp', path_rgb_byte_tif, outputname,
'--dst-crs', 'epsg:3857',
'--resampling', method.name])
assert result.exit_code == 2
assert "Invalid value for" in result.output
assert "--resampling" in result.output
def rasterize(shapes,
out_shape=None,
fill=0,
out=None,
transform=IDENTITY,
all_touched=False,
merge_alg=MergeAlg.replace,
default_value=1,
dtype=None):
"""Return an image array with input geometries burned in.
Warnings will be raised for any invalid or empty geometries, and
an exception will be raised if there are no valid shapes
to rasterize.
Parameters
----------
shapes : iterable of (`geometry`, `value`) pairs or geometries
The `geometry` can either be an object that implements the geo
interface or GeoJSON-like object. If no `value` is provided
the `default_value` will be used. If `value` is `None` the
`fill` value will be used.
out_shape : tuple or list with 2 integers
Shape of output numpy ndarray.
fill : int or float, optional
Used as fill value for all areas not covered by input
geometries.
out : numpy ndarray, optional
Array of same shape and data type as `source` in which to store
results.
transform : Affine transformation object, optional
Transformation from pixel coordinates of `source` to the
coordinate system of the input `shapes`. See the `transform`
property of dataset objects.
all_touched : boolean, optional
If True, all pixels touched by geometries will be burned in. If
false, only pixels whose center is within the polygon or that
are selected by Bresenham's line algorithm will be burned in.
merge_alg : MergeAlg, optional
Merge algorithm to use. One of:
MergeAlg.replace (default):
the new value will overwrite the existing value.
MergeAlg.add:
the new value will be added to the existing raster.
default_value : int or float, optional
Used as value for all geometries, if not provided in `shapes`.
dtype : rasterio or numpy data type, optional
Used as data type for results, if `out` is not provided.
Returns
-------
numpy ndarray
If `out` was not None then `out` is returned, it will have been
modified in-place. If `out` was None, this will be a new array.
Notes
-----
Valid data types for `fill`, `default_value`, `out`, `dtype` and
shape values are "int16", "int32", "uint8", "uint16", "uint32",
"float32", and "float64".
This function requires significant memory resources. The shapes
iterator will be materialized to a Python list and another C copy of
that list will be made. The `out` array will be copied and
additional temporary raster memory equal to 2x the smaller of `out`
data or GDAL's max cache size (controlled by GDAL_CACHEMAX, default
is 5% of the computer's physical memory) is required.
If GDAL max cache size is smaller than the output data, the array of
shapes will be iterated multiple times. Performance is thus a linear
function of buffer size. For maximum speed, ensure that
GDAL_CACHEMAX is larger than the size of `out` or `out_shape`.
"""
valid_dtypes = ('int16', 'int32', 'uint8', 'uint16', 'uint32', 'float32',
'float64')
if GDALVersion.runtime().at_least("3.5"):
valid_dtypes = valid_dtypes + ("int64", "uint64")
def format_invalid_dtype(param):
return '{0} dtype must be one of: {1}'.format(param,
', '.join(valid_dtypes))
def format_cast_error(param, dtype):
return '{0} cannot be cast to specified dtype: {1}'.format(
param, dtype)
if fill != 0:
fill_array = np.array([fill])
if not validate_dtype(fill_array, valid_dtypes):
raise ValueError(format_invalid_dtype('fill'))
if dtype is not None and not can_cast_dtype(fill_array, dtype):
raise ValueError(format_cast_error('fill', dtype))
if default_value != 1:
default_value_array = np.array([default_value])
if not validate_dtype(default_value_array, valid_dtypes):
raise ValueError(format_invalid_dtype('default_value'))
if dtype is not None and not can_cast_dtype(default_value_array,
dtype):
raise ValueError(format_cast_error('default_vaue', dtype))
if dtype is not None and _getnpdtype(dtype).name not in valid_dtypes:
raise ValueError(format_invalid_dtype('dtype'))
valid_shapes = []
shape_values = []
for index, item in enumerate(shapes):
if isinstance(item, (tuple, list)):
geom, value = item
if value is None:
value = fill
else:
geom = item
value = default_value
geom = getattr(geom, '__geo_interface__', None) or geom
if is_valid_geom(geom):
shape_values.append(value)
geom_type = geom['type']
if geom_type == 'GeometryCollection':
# GeometryCollections need to be handled as individual parts to
# avoid holes in output:
# https://github.com/rasterio/rasterio/issues/1253.
# Only 1-level deep since GeoJSON spec discourages nested
# GeometryCollections
for part in geom['geometries']:
valid_shapes.append((part, value))
elif geom_type == 'MultiPolygon':
# Same issue as above
for poly in geom['coordinates']:
valid_shapes.append(({
'type': 'Polygon',
'coordinates': poly
}, value))
else:
valid_shapes.append((geom, value))
else:
# invalid or empty geometries are skipped and raise a warning instead
warnings.warn(
'Invalid or empty shape {} at index {} will not be rasterized.'
.format(geom, index), ShapeSkipWarning)
if not valid_shapes:
raise ValueError('No valid geometry objects found for rasterize')
shape_values = np.array(shape_values)
if not validate_dtype(shape_values, valid_dtypes):
raise ValueError(format_invalid_dtype('shape values'))
if dtype is None:
dtype = get_minimum_dtype(np.append(shape_values, fill))
elif not can_cast_dtype(shape_values, dtype):
raise ValueError(format_cast_error('shape values', dtype))
if out is not None:
if _getnpdtype(out.dtype).name not in valid_dtypes:
raise ValueError(format_invalid_dtype('out'))
if not can_cast_dtype(shape_values, out.dtype):
raise ValueError(format_cast_error('shape values', out.dtype.name))
elif out_shape is not None:
if len(out_shape) != 2:
raise ValueError('Invalid out_shape, must be 2D')
out = np.empty(out_shape, dtype=dtype)
out.fill(fill)
else:
raise ValueError('Either an out_shape or image must be provided')
if min(out.shape) == 0:
raise ValueError("width and height must be > 0")
transform = guard_transform(transform)
_rasterize(valid_shapes, out, transform, all_touched, merge_alg)
return out
from rasterio.warp import (
reproject,
transform_geom,
transform,
transform_bounds,
calculate_default_transform,
aligned_target,
SUPPORTED_RESAMPLING,
GDAL2_RESAMPLING,
)
from rasterio import windows
from .conftest import requires_gdal22, requires_gdal3, requires_gdal_lt_3
gdal_version = GDALVersion.runtime()
DST_TRANSFORM = Affine(300.0, 0.0, -8789636.708, 0.0, -300.0, 2943560.235)
def flatten_coords(coordinates):
"""Yield a flat sequence of coordinates to help testing"""
for elem in coordinates:
if isinstance(elem, (float, int)):
yield elem
else:
for x in flatten_coords(elem):
yield x
assert src.overviews(3) == [2]
def test_build_overviews_two(data):
inputfile = str(data.join('RGB.byte.tif'))
with rasterio.open(inputfile, 'r+') as src:
overview_factors = [2, 4]
src.build_overviews(overview_factors,
resampling=OverviewResampling.nearest)
assert src.overviews(1) == [2, 4]
assert src.overviews(2) == [2, 4]
assert src.overviews(3) == [2, 4]
@pytest.mark.xfail(
GDALVersion.runtime() < GDALVersion.parse("2.0"),
reason="Bilinear resampling not supported by GDAL < 2.0",
)
def test_build_overviews_bilinear(data):
inputfile = str(data.join('RGB.byte.tif'))
with rasterio.open(inputfile, 'r+') as src:
overview_factors = [2, 4]
src.build_overviews(overview_factors,
resampling=OverviewResampling.bilinear)
assert src.overviews(1) == [2, 4]
assert src.overviews(2) == [2, 4]
assert src.overviews(3) == [2, 4]
def test_build_overviews_average(data):
inputfile = str(data.join('RGB.byte.tif'))
@pytest.mark.parametrize("method", SUPPORTED_RESAMPLING)
def test_warp_resampling(runner, path_rgb_byte_tif, tmpdir, method):
"""Resampling methods supported by this version of GDAL should run
successfully"""
outputname = str(tmpdir.join('test.tif'))
result = runner.invoke(main_group, [
'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857',
'--resampling', method.name
])
print(result.output)
assert result.exit_code == 0
@pytest.mark.skipif(GDALVersion.runtime().at_least('2.0'),
reason="Test only applicable to GDAL < 2.0")
@pytest.mark.parametrize("method", GDAL2_RESAMPLING)
def test_warp_resampling_not_yet_supported(runner, path_rgb_byte_tif, tmpdir,
method):
"""Resampling methods not yet supported should fail with error"""
outputname = str(tmpdir.join('test.tif'))
result = runner.invoke(main_group, [
'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857',
'--resampling', method.name
])
assert result.exit_code == 2
assert "Invalid value for" in result.output
assert "--resampling" in result.output
def check_raster_file(src_path: str) -> ValidationInfo: # pragma: no cover
"""
Implementation from
https://github.com/cogeotiff/rio-cogeo/blob/0f00a6ee1eff602014fbc88178a069bd9f4a10da/rio_cogeo/cogeo.py
This function is the rasterio equivalent of
https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py
"""
errors: List[str] = []
warnings: List[str] = []
details: Dict[str, Any] = {}
if not GDALVersion.runtime().at_least('2.2'):
raise RuntimeError('GDAL 2.2 or above required')
config = dict(GDAL_DISABLE_READDIR_ON_OPEN='FALSE')
with rasterio.Env(**config):
with rasterio.open(src_path) as src:
if not src.driver == 'GTiff':
errors.append('The file is not a GeoTIFF')
return errors, warnings, details
filelist = [os.path.basename(f) for f in src.files]
src_bname = os.path.basename(src_path)
if len(filelist) > 1 and src_bname + '.ovr' in filelist:
errors.append(
'Overviews found in external .ovr file. They should be internal'
)
overviews = src.overviews(1)
if src.width >= 512 or src.height >= 512:
if not src.is_tiled:
errors.append(
'The file is greater than 512xH or 512xW, but is not tiled'
)
if not overviews:
warnings.append(
'The file is greater than 512xH or 512xW, it is recommended '
'to include internal overviews')
ifd_offset = int(src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1))
ifd_offsets = [ifd_offset]
if ifd_offset not in (8, 16):
errors.append(
'The offset of the main IFD should be 8 for ClassicTIFF '
'or 16 for BigTIFF. It is {} instead'.format(ifd_offset))
details['ifd_offsets'] = {}
details['ifd_offsets']['main'] = ifd_offset
if not overviews == sorted(overviews):
errors.append('Overviews should be sorted')
for ix, dec in enumerate(overviews):
# NOTE: Size check is handled in rasterio `src.overviews` methods
# https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx
# We just need to make sure the decimation level is > 1
if not dec > 1:
errors.append(
'Invalid Decimation {} for overview level {}'.format(
dec, ix))
# Check that the IFD of descending overviews are sorted by increasing
# offsets
ifd_offset = int(
src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1, ovr=ix))
ifd_offsets.append(ifd_offset)
details['ifd_offsets']['overview_{}'.format(ix)] = ifd_offset
if ifd_offsets[-1] < ifd_offsets[-2]:
if ix == 0:
errors.append(
'The offset of the IFD for overview of index {} is {}, '
'whereas it should be greater than the one of the main '
'image, which is at byte {}'.format(
ix, ifd_offsets[-1], ifd_offsets[-2]))
else:
errors.append(
'The offset of the IFD for overview of index {} is {}, '
'whereas it should be greater than the one of index {}, '
'which is at byte {}'.format(
ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2]))
block_offset = int(
src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1))
if not block_offset:
errors.append('Missing BLOCK_OFFSET_0_0')
data_offset = int(block_offset) if block_offset else 0
data_offsets = [data_offset]
details['data_offsets'] = {}
details['data_offsets']['main'] = data_offset
for ix, dec in enumerate(overviews):
data_offset = int(
src.get_tag_item('BLOCK_OFFSET_0_0',
'TIFF',
bidx=1,
ovr=ix))
data_offsets.append(data_offset)
details['data_offsets']['overview_{}'.format(ix)] = data_offset
if data_offsets[-1] < ifd_offsets[-1]:
if len(overviews) > 0:
errors.append(
'The offset of the first block of the smallest overview '
'should be after its IFD')
else:
errors.append(
'The offset of the first block of the image should '
'be after its IFD')
for i in range(len(data_offsets) - 2, 0, -1):
if data_offsets[i] < data_offsets[i + 1]:
errors.append(
'The offset of the first block of overview of index {} should '
'be after the one of the overview of index {}'.format(
i - 1, i))
if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]:
errors.append(
'The offset of the first block of the main resolution image '
'should be after the one of the overview of index {}'.
format(len(overviews) - 1))
for ix, dec in enumerate(overviews):
with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst:
if ovr_dst.width >= 512 or ovr_dst.height >= 512:
if not ovr_dst.is_tiled:
errors.append(
'Overview of index {} is not tiled'.format(ix))
return errors, warnings, details
