def _prefered_compression_method() -> str: if not GDALVersion.runtime().at_least('2.3'): return 'DEFLATE' # check if we can use ZSTD (fails silently for GDAL < 2.3) dummy_profile = dict(driver='GTiff', height=1, width=1, count=1, dtype='uint8') try: with warnings.catch_warnings(): warnings.simplefilter('ignore', NotGeoreferencedWarning) with MemoryFile() as memfile, memfile.open(compress='ZSTD', **dummy_profile): pass except Exception as exc: if 'missing codec' not in str(exc): raise else: return 'ZSTD' return 'DEFLATE'
def rasterio_crs(self): """Return rasterio CRS.""" import rasterio from rasterio.env import GDALVersion if GDALVersion.runtime().major < 3: return rasterio.crs.CRS.from_wkt( self.crs.to_wkt(WktVersion.WKT1_GDAL)) else: return rasterio.crs.CRS.from_wkt(self.crs.to_wkt())
def transform_geom(src_crs, dst_crs, geom, antimeridian_cutting=True, antimeridian_offset=10.0, precision=-1): """Transform geometry from source coordinate reference system into target. Parameters ------------ src_crs: CRS or dict Source coordinate reference system, in rasterio dict format. Example: CRS({'init': 'EPSG:4326'}) dst_crs: CRS or dict Target coordinate reference system. geom: GeoJSON like dict object antimeridian_cutting: bool, optional If True, cut geometries at the antimeridian, otherwise geometries will not be cut (default). If False and GDAL is 2.2.0 or newer an exception is raised. Antimeridian cutting is always on as of GDAL 2.2.0 but this could produce an unexpected geometry. antimeridian_offset: float Offset from the antimeridian in degrees (default: 10) within which any geometries will be split. precision: float If >= 0, geometry coordinates will be rounded to this number of decimal places after the transform operation, otherwise original coordinate values will be preserved (default). Returns --------- out: GeoJSON like dict object Transformed geometry in GeoJSON dict format """ if (GDALVersion.runtime().at_least('2.2') and not antimeridian_cutting): raise GDALBehaviorChangeException( "Antimeridian cutting is always enabled on GDAL 2.2.0 or " "newer, which could produce a different geometry than expected.") return _transform_geom(src_crs, dst_crs, geom, antimeridian_cutting, antimeridian_offset, precision)
def test_gdalversion_class_runtime(): """Test the version of GDAL from this runtime""" GDALVersion.runtime().major >= 1
from rasterio._base import _transform from rasterio.enums import Resampling from rasterio.env import GDALVersion, ensure_env, require_gdal_version from rasterio.errors import GDALBehaviorChangeException, TransformError from rasterio.transform import array_bounds from rasterio._warp import ( _calculate_default_transform, _reproject, _transform_bounds, _transform_geom, ) # Gauss (7) is not supported for warp SUPPORTED_RESAMPLING = [r for r in Resampling if r.value < 7] GDAL2_RESAMPLING = [r for r in Resampling if r.value > 7 and r.value <= 12] if GDALVersion.runtime().at_least('2.0'): SUPPORTED_RESAMPLING.extend(GDAL2_RESAMPLING) # sum supported since GDAL 3.1 if GDALVersion.runtime().at_least('3.1'): SUPPORTED_RESAMPLING.append(Resampling.sum) # rms supported since GDAL 3.3 if GDALVersion.runtime().at_least('3.3'): SUPPORTED_RESAMPLING.append(Resampling.rms) @ensure_env def transform(src_crs, dst_crs, xs, ys, zs=None): """Transform vectors from source to target coordinate reference system. Transform vectors of x, y and optionally z from source
if not os.path.exists(path): with zipfile.ZipFile(path, 'w') as zip: for filename in ['white-gemini-iv.vrt', '389225main_sw_1965_1024.jpg']: zip.write(os.path.join(data_dir, filename), filename) return path class MockGeoInterface(object): """Tiny wrapper for GeoJSON to present an object with __geo_interface__ for testing""" def __init__(self, geojson): self.__geo_interface__ = geojson # Define helpers to skip tests based on GDAL version gdal_version = GDALVersion.runtime() requires_only_gdal1 = pytest.mark.skipif( gdal_version.major != 1, reason="Only relevant for GDAL 1.x") requires_gdal2 = pytest.mark.skipif( not gdal_version.major >= 2, reason="Requires GDAL 2.x") requires_gdal21 = pytest.mark.skipif( not gdal_version.at_least('2.1'), reason="Requires GDAL 2.1.x") requires_gdal22 = pytest.mark.skipif( not gdal_version.at_least('2.2'),
def test_warp_resampling(runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods supported by this version of GDAL should run successfully""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name]) print(result.output) assert result.exit_code == 0 @pytest.mark.skipif( GDALVersion.runtime().at_least('2.0'), reason="Test only applicable to GDAL < 2.0") @pytest.mark.parametrize("method", GDAL2_RESAMPLING) def test_warp_resampling_not_yet_supported( runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods not yet supported should fail with error""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name]) assert result.exit_code == 2 assert 'Invalid value for "--resampling"' in result.output
def cog_translate( # noqa: C901 source: Union[str, pathlib.PurePath, DatasetReader, DatasetWriter, WarpedVRT], dst_path: Union[str, pathlib.PurePath], dst_kwargs: Dict, indexes: Optional[Sequence[int]] = None, nodata: Optional[Union[str, int, float]] = None, dtype: Optional[str] = None, add_mask: bool = False, overview_level: Optional[int] = None, overview_resampling: str = "nearest", web_optimized: bool = False, tms: Optional[morecantile.TileMatrixSet] = None, zoom_level_strategy: str = "auto", zoom_level: Optional[int] = None, aligned_levels: Optional[int] = None, resampling: str = "nearest", in_memory: Optional[bool] = None, config: Optional[Dict] = None, allow_intermediate_compression: bool = False, forward_band_tags: bool = False, quiet: bool = False, temporary_compression: str = "DEFLATE", colormap: Optional[Dict] = None, additional_cog_metadata: Optional[Dict] = None, use_cog_driver: bool = False, ): """ Create Cloud Optimized Geotiff. Parameters ---------- source : str, PathLike object or rasterio.io.DatasetReader A dataset path, URL or rasterio.io.DatasetReader object. Will be opened in "r" mode. dst_path : str or PathLike object An output dataset path or or PathLike object. Will be opened in "w" mode. dst_kwargs: dict Output dataset creation options. indexes : tuple or int, optional Raster band indexes to copy. nodata, int, optional Overwrite nodata masking values for input dataset. dtype: str, optional Overwrite output data type. Default will be the input data type. add_mask, bool, optional Force output dataset creation with a mask. overview_level : int, optional (default: None) COGEO overview (decimation) level. By default, inferred from data size. overview_resampling : str, optional (default: "nearest") Resampling algorithm for overviews web_optimized: bool, optional (default: False) Create web-optimized cogeo. tms: morecantile.TileMatrixSet, optional (default: "WebMercatorQuad") TileMatrixSet to use for reprojection, resolution and alignment. zoom_level_strategy: str, optional (default: auto) Strategy to determine zoom level (same as in GDAL 3.2). LOWER will select the zoom level immediately below the theoretical computed non-integral zoom level, leading to subsampling. On the contrary, UPPER will select the immediately above zoom level, leading to oversampling. Defaults to AUTO which selects the closest zoom level. ref: https://gdal.org/drivers/raster/cog.html#raster-cog zoom_level: int, optional. Zoom level number (starting at 0 for coarsest zoom level). If this option is specified, `--zoom-level-strategy` is ignored. aligned_levels: int, optional. Number of overview levels for which GeoTIFF tile and tiles defined in the tiling scheme match. Default is to use the maximum overview levels. Note: GDAL use number of resolution levels instead of overview levels. resampling : str, optional (default: "nearest") Resampling algorithm. in_memory: bool, optional Force processing raster in memory (default: process in memory if small) config : dict Rasterio Env options. allow_intermediate_compression: bool, optional (default: False) Allow intermediate file compression to reduce memory/disk footprint. Note: This could reduce the speed of the process. Ref: https://github.com/cogeotiff/rio-cogeo/issues/103 forward_band_tags: bool, optional Forward band tags to output bands. Ref: https://github.com/cogeotiff/rio-cogeo/issues/19 quiet: bool, optional (default: False) Mask processing steps. temporary_compression: str, optional Compression used for the intermediate file, default is deflate. colormap: dict, optional Overwrite or add a colormap to the output COG. additional_cog_metadata: dict, optional Additional dataset metadata to add to the COG. use_cog_driver: bool, optional (default: False) Use GDAL COG driver if set to True. COG driver is available starting with GDAL 3.1. """ tms = tms or morecantile.tms.get("WebMercatorQuad") dst_kwargs = dst_kwargs.copy() if isinstance(indexes, int): indexes = (indexes, ) config = config or {} with rasterio.Env(**config): with ExitStack() as ctx: if isinstance(source, (DatasetReader, DatasetWriter, WarpedVRT)): src_dst = source else: src_dst = ctx.enter_context(rasterio.open(source)) meta = src_dst.meta indexes = indexes if indexes else src_dst.indexes nodata = nodata if nodata is not None else src_dst.nodata dtype = dtype if dtype else src_dst.dtypes[0] alpha = utils.has_alpha_band(src_dst) mask = utils.has_mask_band(src_dst) if colormap and len(indexes) > 1: raise IncompatibleOptions( "Cannot add a colormap for multiple bands data.") if not add_mask and ( (nodata is not None or alpha) and dst_kwargs.get("compress", "").lower() == "jpeg"): warnings.warn( "Nodata/Alpha band will be translated to an internal mask band.", ) add_mask = True indexes = (utils.non_alpha_indexes(src_dst) if len(indexes) not in [1, 3] else indexes) tilesize = min(int(dst_kwargs["blockxsize"]), int(dst_kwargs["blockysize"])) if src_dst.width < tilesize or src_dst.height < tilesize: tilesize = 2**int( math.log(min(src_dst.width, src_dst.height), 2)) if tilesize < 64: warnings.warn( "Raster has dimension < 64px. Output COG cannot be tiled" " and overviews cannot be added.", IncompatibleBlockRasterSize, ) dst_kwargs.pop("blockxsize", None) dst_kwargs.pop("blockysize", None) dst_kwargs.pop("tiled") overview_level = 0 else: warnings.warn( "Block Size are bigger than raster sizes. " "Setting blocksize to {}".format(tilesize), IncompatibleBlockRasterSize, ) dst_kwargs["blockxsize"] = tilesize dst_kwargs["blockysize"] = tilesize vrt_params = { "add_alpha": True, "dtype": dtype, "width": src_dst.width, "height": src_dst.height, "resampling": ResamplingEnums[resampling], } if nodata is not None: vrt_params.update( dict(nodata=nodata, add_alpha=False, src_nodata=nodata)) if alpha: vrt_params.update(dict(add_alpha=False)) if web_optimized and not use_cog_driver: params = utils.get_web_optimized_params( src_dst, zoom_level_strategy=zoom_level_strategy, zoom_level=zoom_level, aligned_levels=aligned_levels, tms=tms, ) vrt_params.update(**params) with WarpedVRT(src_dst, **vrt_params) as vrt_dst: meta = vrt_dst.meta meta["count"] = len(indexes) if add_mask: meta.pop("nodata", None) meta.pop("alpha", None) if (dst_kwargs.get("photometric", "").upper() == "YCBCR" and meta["count"] == 1): warnings.warn( "PHOTOMETRIC=YCBCR not supported on a 1-band raster" " and has been set to 'MINISBLACK'") dst_kwargs["photometric"] = "MINISBLACK" meta.update(**dst_kwargs) meta.pop("compress", None) meta.pop("photometric", None) if allow_intermediate_compression: meta["compress"] = temporary_compression if in_memory is None: in_memory = vrt_dst.width * vrt_dst.height < IN_MEMORY_THRESHOLD if in_memory: tmpfile = ctx.enter_context(MemoryFile()) tmp_dst = ctx.enter_context(tmpfile.open(**meta)) else: tmpfile = ctx.enter_context(TemporaryRasterFile(dst_path)) tmp_dst = ctx.enter_context( rasterio.open(tmpfile.name, "w", **meta)) # Transfer color interpolation if len(indexes) == 1 and (vrt_dst.colorinterp[indexes[0] - 1] is not ColorInterp.palette): tmp_dst.colorinterp = [ColorInterp.gray] else: tmp_dst.colorinterp = [ vrt_dst.colorinterp[b - 1] for b in indexes ] if colormap: if tmp_dst.colorinterp[0] is not ColorInterp.palette: tmp_dst.colorinterp = [ColorInterp.palette] warnings.warn( "Dataset color interpretation was set to `Palette`" ) tmp_dst.write_colormap(1, colormap) elif tmp_dst.colorinterp[0] is ColorInterp.palette: try: tmp_dst.write_colormap(1, vrt_dst.colormap(1)) except ValueError: warnings.warn( "Dataset has `Palette` color interpretation" " but is missing colormap information") wind = list(tmp_dst.block_windows(1)) if not quiet: click.echo("Reading input: {}".format(source), err=True) fout = ctx.enter_context(open(os.devnull, "w")) if quiet else sys.stderr with click.progressbar( wind, file=fout, show_percent=True) as windows: # type: ignore for _, w in windows: matrix = vrt_dst.read(window=w, indexes=indexes) tmp_dst.write(matrix, window=w) if add_mask or mask: # Cast mask to uint8 to fix rasterio 1.1.2 error (ref #115) mask_value = vrt_dst.dataset_mask( window=w).astype("uint8") tmp_dst.write_mask(mask_value, window=w) if overview_level is None: overview_level = get_maximum_overview_level( vrt_dst.width, vrt_dst.height, minsize=tilesize) if not quiet and overview_level: click.echo("Adding overviews...", err=True) overviews = [2**j for j in range(1, overview_level + 1)] tmp_dst.build_overviews(overviews, ResamplingEnums[overview_resampling]) if not quiet: click.echo("Updating dataset tags...", err=True) for i, b in enumerate(indexes): tmp_dst.set_band_description(i + 1, src_dst.descriptions[b - 1]) if forward_band_tags: tmp_dst.update_tags(i + 1, **src_dst.tags(b)) tags = src_dst.tags() tags.update( dict( OVR_RESAMPLING_ALG=ResamplingEnums[overview_resampling] .name.upper())) if additional_cog_metadata: tags.update(**additional_cog_metadata) if web_optimized and not use_cog_driver: default_zoom = tms.zoom_for_res( max(tmp_dst.res), max_z=30, zoom_level_strategy=zoom_level_strategy, ) dst_kwargs.update({ "@TILING_SCHEME_NAME": tms.identifier, "@TILING_SCHEME_ZOOM_LEVEL": zoom_level if zoom_level is not None else default_zoom, }) if aligned_levels: dst_kwargs.update( {"@TILING_SCHEME_ALIGNED_LEVELS": aligned_levels}) tmp_dst.update_tags(**tags) tmp_dst._set_all_scales( [vrt_dst.scales[b - 1] for b in indexes]) tmp_dst._set_all_offsets( [vrt_dst.offsets[b - 1] for b in indexes]) if not quiet: click.echo("Writing output to: {}".format(dst_path), err=True) if use_cog_driver: if not GDALVersion.runtime().at_least("3.1"): raise Exception( "GDAL 3.1 or above required to use the COG driver." ) dst_kwargs["driver"] = "COG" if web_optimized: dst_kwargs["TILING_SCHEME"] = ( "GoogleMapsCompatible" if tms.identifier == "WebMercatorQuad" else tms.identifier) if zoom_level is not None: if not GDALVersion.runtime().at_least("3.5"): warnings.warn( "ZOOM_LEVEL option is only available with GDAL >3.5." ) dst_kwargs["ZOOM_LEVEL"] = zoom_level dst_kwargs["ZOOM_LEVEL_STRATEGY"] = zoom_level_strategy if aligned_levels is not None: # GDAL uses Number of resolution (not overviews) # See https://github.com/OSGeo/gdal/issues/5336#issuecomment-1042946603 dst_kwargs["aligned_levels"] = aligned_levels + 1 if add_mask and dst_kwargs.get("compress", "") != "JPEG": warnings.warn( "With GDAL COG driver, mask band will be translated to an alpha band." ) dst_kwargs["overview_resampling"] = overview_resampling dst_kwargs["warp_resampling"] = resampling dst_kwargs["blocksize"] = tilesize dst_kwargs.pop("blockxsize", None) dst_kwargs.pop("blockysize", None) dst_kwargs.pop("tiled", None) dst_kwargs.pop("interleave", None) dst_kwargs.pop("photometric", None) copy(tmp_dst, dst_path, **dst_kwargs) else: copy(tmp_dst, dst_path, copy_src_overviews=True, **dst_kwargs)
from io import BytesIO import logging import pytest import rasterio from rasterio.io import MemoryFile, ZipMemoryFile from rasterio.env import GDALVersion logging.basicConfig(level=logging.DEBUG) # Skip ENTIRE module if not GDAL >= 2.x. # pytestmark is a keyword that instructs pytest to skip this module. pytestmark = pytest.mark.skipif( not GDALVersion.runtime().major >= 2, reason="MemoryFile requires GDAL 2.x") @pytest.fixture(scope='session') def rgb_file_bytes(path_rgb_byte_tif): """Get the bytes of our RGB.bytes.tif file""" return open(path_rgb_byte_tif, 'rb').read() @pytest.fixture(scope='session') def rgb_lzw_file_bytes(): """Get the bytes of our RGB.bytes.tif file""" return open('tests/data/rgb_lzw.tif', 'rb').read()
import logging import os.path from affine import Affine import numpy import pytest import rasterio from rasterio.io import MemoryFile, ZipMemoryFile from rasterio.enums import MaskFlags from rasterio.env import GDALVersion from rasterio.shutil import copyfiles # Skip ENTIRE module if not GDAL >= 2.x. # pytestmark is a keyword that instructs pytest to skip this module. pytestmark = pytest.mark.skipif(not GDALVersion.runtime().major >= 2, reason="MemoryFile requires GDAL 2.x") @pytest.fixture(scope='session') def rgb_file_bytes(path_rgb_byte_tif): """Get the bytes of our RGB.bytes.tif file""" return open(path_rgb_byte_tif, 'rb').read() @pytest.fixture(scope='session') def rgb_lzw_file_bytes(): """Get the bytes of our RGB.bytes.tif file""" return open('tests/data/rgb_lzw.tif', 'rb').read()
def check_raster_file(src_path: str) -> ValidationInfo: # pragma: no cover """ Implementation from https://github.com/cogeotiff/rio-cogeo/blob/a07d914e2d898878417638bbc089179f01eb5b28/rio_cogeo/cogeo.py#L385 This function is the rasterio equivalent of https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py """ errors: List[str] = [] warnings: List[str] = [] details: Dict[str, Any] = {} if not GDALVersion.runtime().at_least('2.2'): raise RuntimeError('GDAL 2.2 or above required') config = dict(GDAL_DISABLE_READDIR_ON_OPEN='FALSE') with rasterio.Env(**config): with rasterio.open(src_path) as src: if not src.driver == 'GTiff': errors.append('The file is not a GeoTIFF') return errors, warnings, details if any(os.path.splitext(x)[-1] == '.ovr' for x in src.files): errors.append( 'Overviews found in external .ovr file. They should be internal' ) overviews = src.overviews(1) if src.width > 512 and src.height > 512: if not src.is_tiled: errors.append( 'The file is greater than 512xH or 512xW, but is not tiled' ) if not overviews: warnings.append( 'The file is greater than 512xH or 512xW, it is recommended ' 'to include internal overviews') ifd_offset = int(src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1)) # Starting from GDAL 3.1, GeoTIFF and COG have ghost headers # e.g: # """ # GDAL_STRUCTURAL_METADATA_SIZE=000140 bytes # LAYOUT=IFDS_BEFORE_DATA # BLOCK_ORDER=ROW_MAJOR # BLOCK_LEADER=SIZE_AS_UINT4 # BLOCK_TRAILER=LAST_4_BYTES_REPEATED # KNOWN_INCOMPATIBLE_EDITION=NO # """ # # This header should be < 200bytes if ifd_offset > 300: errors.append( f'The offset of the main IFD should be < 300. It is {ifd_offset} instead' ) ifd_offsets = [ifd_offset] details['ifd_offsets'] = {} details['ifd_offsets']['main'] = ifd_offset if overviews and overviews != sorted(overviews): errors.append('Overviews should be sorted') for ix, dec in enumerate(overviews): # NOTE: Size check is handled in rasterio `src.overviews` methods # https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx # We just need to make sure the decimation level is > 1 if not dec > 1: errors.append( 'Invalid Decimation {} for overview level {}'.format( dec, ix)) # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int( src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) details['ifd_offsets']['overview_{}'.format(ix)] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if ix == 0: errors.append( 'The offset of the IFD for overview of index {} is {}, ' 'whereas it should be greater than the one of the main ' 'image, which is at byte {}'.format( ix, ifd_offsets[-1], ifd_offsets[-2])) else: errors.append( 'The offset of the IFD for overview of index {} is {}, ' 'whereas it should be greater than the one of index {}, ' 'which is at byte {}'.format( ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2])) block_offset = src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1) data_offset = int(block_offset) if block_offset else 0 data_offsets = [data_offset] details['data_offsets'] = {} details['data_offsets']['main'] = data_offset for ix, dec in enumerate(overviews): block_offset = src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1, ovr=ix) data_offset = int(block_offset) if block_offset else 0 data_offsets.append(data_offset) details['data_offsets']['overview_{}'.format(ix)] = data_offset if data_offsets[-1] != 0 and data_offsets[-1] < ifd_offsets[-1]: if len(overviews) > 0: errors.append( 'The offset of the first block of the smallest overview ' 'should be after its IFD') else: errors.append( 'The offset of the first block of the image should ' 'be after its IFD') for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors.append( 'The offset of the first block of overview of index {} should ' 'be after the one of the overview of index {}'.format( i - 1, i)) if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors.append( 'The offset of the first block of the main resolution image ' 'should be after the one of the overview of index {}'. format(len(overviews) - 1)) for ix, dec in enumerate(overviews): with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst: if ovr_dst.width > 512 and ovr_dst.height > 512: if not ovr_dst.is_tiled: errors.append( 'Overview of index {} is not tiled'.format(ix)) return errors, warnings, details
def cog_validate(src_path): """ Validate Cloud Optimized Geotiff. Parameters ---------- src_path : str or PathLike object A dataset path or URL. Will be opened in "r" mode. This script is the rasterio equivalent of https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py """ errors = [] warnings = [] details = {} if not GDALVersion.runtime().at_least("2.2"): raise Exception("GDAL 2.2 or above required") config = dict(GDAL_DISABLE_READDIR_ON_OPEN="FALSE") with rasterio.Env(**config): with rasterio.open(src_path) as src: if not src.driver == "GTiff": raise Exception("The file is not a GeoTIFF") filelist = [os.path.basename(f) for f in src.files] src_bname = os.path.basename(src_path) if len(filelist) > 1 and src_bname + ".ovr" in filelist: errors.append( "Overviews found in external .ovr file. They should be internal" ) overviews = src.overviews(1) if src.width > 512 or src.height > 512: if not src.is_tiled: errors.append( "The file is greater than 512xH or 512xW, but is not tiled" ) if not overviews: warnings.append( "The file is greater than 512xH or 512xW, it is recommended " "to include internal overviews") ifd_offset = int(src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1)) ifd_offsets = [ifd_offset] if ifd_offset not in (8, 16): errors.append( "The offset of the main IFD should be 8 for ClassicTIFF " "or 16 for BigTIFF. It is {} instead".format(ifd_offset)) details["ifd_offsets"] = {} details["ifd_offsets"]["main"] = ifd_offset if overviews and overviews != sorted(overviews): errors.append("Overviews should be sorted") for ix, dec in enumerate(overviews): # NOTE: Size check is handled in rasterio `src.overviews` methods # https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx # We just need to make sure the decimation level is > 1 if not dec > 1: errors.append( "Invalid Decimation {} for overview level {}".format( dec, ix)) # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int( src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) details["ifd_offsets"]["overview_{}".format(ix)] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if ix == 0: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of the main " "image, which is at byte {}".format( ix, ifd_offsets[-1], ifd_offsets[-2])) else: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of index {}, " "which is at byte {}".format( ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2])) block_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1)) if not block_offset: errors.append("Missing BLOCK_OFFSET_0_0") data_offset = int(block_offset) if block_offset else None data_offsets = [data_offset] details["data_offsets"] = {} details["data_offsets"]["main"] = data_offset for ix, dec in enumerate(overviews): data_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1, ovr=ix)) data_offsets.append(data_offset) details["data_offsets"]["overview_{}".format(ix)] = data_offset if data_offsets[-1] < ifd_offsets[-1]: if len(overviews) > 0: errors.append( "The offset of the first block of the smallest overview " "should be after its IFD") else: errors.append( "The offset of the first block of the image should " "be after its IFD") for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors.append( "The offset of the first block of overview of index {} should " "be after the one of the overview of index {}".format( i - 1, i)) if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors.append( "The offset of the first block of the main resolution image " "should be after the one of the overview of index {}". format(len(overviews) - 1)) for ix, dec in enumerate(overviews): with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst: if ovr_dst.width >= 512 or ovr_dst.height >= 512: if not ovr_dst.is_tiled: errors.append( "Overview of index {} is not tiled".format(ix)) if warnings: click.secho("The following warnings were found:", fg="yellow", err=True) for w in warnings: click.echo("- " + w, err=True) click.echo(err=True) if errors: click.secho("The following errors were found:", fg="red", err=True) for e in errors: click.echo("- " + e, err=True) return False return True
import logging import os.path from affine import Affine import numpy import pytest import rasterio from rasterio.io import MemoryFile, ZipMemoryFile from rasterio.env import GDALVersion # Skip ENTIRE module if not GDAL >= 2.x. # pytestmark is a keyword that instructs pytest to skip this module. pytestmark = pytest.mark.skipif( not GDALVersion.runtime().major >= 2, reason="MemoryFile requires GDAL 2.x") @pytest.fixture(scope='session') def rgb_file_bytes(path_rgb_byte_tif): """Get the bytes of our RGB.bytes.tif file""" return open(path_rgb_byte_tif, 'rb').read() @pytest.fixture(scope='session') def rgb_lzw_file_bytes(): """Get the bytes of our RGB.bytes.tif file""" return open('tests/data/rgb_lzw.tif', 'rb').read()
def validate(src_path: str) -> bool: """Validate given cloud-optimized GeoTIFF""" if not GDALVersion.runtime().at_least('2.2'): raise RuntimeError('GDAL 2.2 or above required') with rasterio.open(src_path) as src: if not src.driver == 'GTiff': # Not a GeoTIFF return False filelist = [os.path.basename(f) for f in src.files] src_bname = os.path.basename(src_path) if len(filelist) > 1 and src_bname + '.ovr' in filelist: # Overviews found in external .ovr file. They should be internal return False overviews = src.overviews(1) or [] if src.width >= 512 or src.height >= 512: if not src.is_tiled: # The file is greater than 512xH or 512xW, but is not tiled return False if not overviews: # The file is greater than 512xH or 512xW, but has no overviews return False ifd_offset = int(src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1)) ifd_offsets = [ifd_offset] if ifd_offset not in (8, 16): # The offset of the main IFD should be 8 for ClassicTIFF or 16 for BigTIFF return False if not overviews == sorted(overviews): # Overviews should be sorted return False for ix, dec in enumerate(overviews): if not dec > 1: # Invalid Decimation return False # TODO: Check if the overviews are tiled # NOTE: There is currently no way to do that with rasterio # Check that the IFD of descending overviews are sorted by increasing offsets ifd_offset = int( src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) if ifd_offsets[-1] < ifd_offsets[-2]: return False block_offset = int(src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1)) if not block_offset: return False data_offset = int(block_offset) data_offsets = [data_offset] for ix, dec in enumerate(overviews): data_offset = int( src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1, ovr=ix)) data_offsets.append(data_offset) if data_offsets[-1] < ifd_offsets[-1]: return False for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: return False if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: return False return True
"""Mapping of GDAL to Numpy data types. Since 0.13 we are not importing numpy here and data types are strings. Happily strings can be used throughout Numpy and so existing code will not break. """ import numpy from rasterio.env import GDALVersion _GDAL_AT_LEAST_35 = GDALVersion.runtime().at_least("3.5") bool_ = 'bool' ubyte = uint8 = 'uint8' sbyte = int8 = 'int8' uint16 = 'uint16' int16 = 'int16' uint32 = 'uint32' int32 = 'int32' uint64 = 'uint32' int64 = 'int64' float32 = 'float32' float64 = 'float64' complex_ = 'complex' complex64 = 'complex64' complex128 = 'complex128' complex_int16 = "complex_int16" dtype_fwd = {
import os import click import numpy from rasterio.enums import Resampling as ResamplingEnums from rasterio.env import GDALVersion from rasterio.rio import options from rasterio.warp import SUPPORTED_RESAMPLING as WarpResampling from rio_cogeo import __version__ as cogeo_version from rio_cogeo.cogeo import cog_info, cog_translate, cog_validate from rio_cogeo.profiles import cog_profiles OverviewResampling = [r for r in ResamplingEnums if r.value < 8] if GDALVersion.runtime().at_least("3.3"): OverviewResampling.append(ResamplingEnums.rms) IN_MEMORY_THRESHOLD = int(os.environ.get("IN_MEMORY_THRESHOLD", 10980 * 10980)) class BdxParamType(click.ParamType): """Band index type.""" name = "bidx" def convert(self, value, param, ctx): """Validate and parse band index.""" try: bands = [int(x) for x in value.split(",")] assert all(b > 0 for b in bands)
def cog_validate( # noqa: C901 src_path: Union[str, pathlib.PurePath], strict: bool = False, quiet: bool = False) -> Tuple[bool, List[str], List[str]]: """ Validate Cloud Optimized Geotiff. This script is the rasterio equivalent of https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py Parameters ---------- src_path: str or PathLike object A dataset path or URL. Will be opened in "r" mode. strict: bool Treat warnings as errors quiet: bool Remove standard outputs Returns ------- is_valid: bool True is src_path is a valid COG. errors: list List of validation errors. warnings: list List of validation warnings. """ if isinstance(src_path, str): src_path = pathlib.Path(src_path) errors = [] warnings = [] details: Dict[str, Any] = {} if not GDALVersion.runtime().at_least("2.2"): raise Exception("GDAL 2.2 or above required") config = dict(GDAL_DISABLE_READDIR_ON_OPEN="FALSE") with rasterio.Env(**config): with rasterio.open(src_path) as src: if not src.driver == "GTiff": raise Exception("The file is not a GeoTIFF") filelist = [pathlib.Path(f).name for f in src.files] if len(filelist) > 1 and f"{src_path.name}.ovr" in filelist: errors.append( "Overviews found in external .ovr file. They should be internal" ) overviews = src.overviews(1) if src.width > 512 or src.height > 512: if not src.is_tiled: errors.append( "The file is greater than 512xH or 512xW, but is not tiled" ) if not overviews: warnings.append( "The file is greater than 512xH or 512xW, it is recommended " "to include internal overviews") ifd_offset = int(src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1)) # Starting from GDAL 3.1, GeoTIFF and COG have ghost headers # e.g: # """ # GDAL_STRUCTURAL_METADATA_SIZE=000140 bytes # LAYOUT=IFDS_BEFORE_DATA # BLOCK_ORDER=ROW_MAJOR # BLOCK_LEADER=SIZE_AS_UINT4 # BLOCK_TRAILER=LAST_4_BYTES_REPEATED # KNOWN_INCOMPATIBLE_EDITION=NO # """ # # This header should be < 200bytes if ifd_offset > 300: errors.append( f"The offset of the main IFD should be < 300. It is {ifd_offset} instead" ) ifd_offsets = [ifd_offset] details["ifd_offsets"] = {} details["ifd_offsets"]["main"] = ifd_offset if overviews and overviews != sorted(overviews): errors.append("Overviews should be sorted") for ix, dec in enumerate(overviews): # NOTE: Size check is handled in rasterio `src.overviews` methods # https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx # We just need to make sure the decimation level is > 1 if not dec > 1: errors.append( "Invalid Decimation {} for overview level {}".format( dec, ix)) # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int( src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) details["ifd_offsets"]["overview_{}".format(ix)] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if ix == 0: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of the main " "image, which is at byte {}".format( ix, ifd_offsets[-1], ifd_offsets[-2])) else: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of index {}, " "which is at byte {}".format( ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2])) block_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1)) if not block_offset: errors.append("Missing BLOCK_OFFSET_0_0") data_offset = int(block_offset) if block_offset else None data_offsets = [data_offset] details["data_offsets"] = {} details["data_offsets"]["main"] = data_offset for ix, dec in enumerate(overviews): data_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1, ovr=ix)) data_offsets.append(data_offset) details["data_offsets"]["overview_{}".format(ix)] = data_offset if data_offsets[-1] < ifd_offsets[-1]: if len(overviews) > 0: errors.append( "The offset of the first block of the smallest overview " "should be after its IFD") else: errors.append( "The offset of the first block of the image should " "be after its IFD") for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors.append( "The offset of the first block of overview of index {} should " "be after the one of the overview of index {}".format( i - 1, i)) if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors.append( "The offset of the first block of the main resolution image " "should be after the one of the overview of index {}". format(len(overviews) - 1)) for ix, dec in enumerate(overviews): with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst: if ovr_dst.width >= 512 or ovr_dst.height >= 512: if not ovr_dst.is_tiled: errors.append( "Overview of index {} is not tiled".format(ix)) if warnings and not quiet: click.secho("The following warnings were found:", fg="yellow", err=True) for w in warnings: click.echo("- " + w, err=True) click.echo(err=True) if errors and not quiet: click.secho("The following errors were found:", fg="red", err=True) for e in errors: click.echo("- " + e, err=True) is_valid = False if errors or (warnings and strict) else True return is_valid, errors, warnings
def test_warp_resampling(runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods supported by this version of GDAL should run successfully""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name]) print(result.output) assert result.exit_code == 0 @pytest.mark.skipif( GDALVersion.runtime().at_least('2.0'), reason="Test only applicable to GDAL < 2.0") @pytest.mark.parametrize("method", GDAL2_RESAMPLING) def test_warp_resampling_not_yet_supported( runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods not yet supported should fail with error""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name]) assert result.exit_code == 2 assert "Invalid value for" in result.output assert "--resampling" in result.output
def rasterize(shapes, out_shape=None, fill=0, out=None, transform=IDENTITY, all_touched=False, merge_alg=MergeAlg.replace, default_value=1, dtype=None): """Return an image array with input geometries burned in. Warnings will be raised for any invalid or empty geometries, and an exception will be raised if there are no valid shapes to rasterize. Parameters ---------- shapes : iterable of (`geometry`, `value`) pairs or geometries The `geometry` can either be an object that implements the geo interface or GeoJSON-like object. If no `value` is provided the `default_value` will be used. If `value` is `None` the `fill` value will be used. out_shape : tuple or list with 2 integers Shape of output numpy ndarray. fill : int or float, optional Used as fill value for all areas not covered by input geometries. out : numpy ndarray, optional Array of same shape and data type as `source` in which to store results. transform : Affine transformation object, optional Transformation from pixel coordinates of `source` to the coordinate system of the input `shapes`. See the `transform` property of dataset objects. all_touched : boolean, optional If True, all pixels touched by geometries will be burned in. If false, only pixels whose center is within the polygon or that are selected by Bresenham's line algorithm will be burned in. merge_alg : MergeAlg, optional Merge algorithm to use. One of: MergeAlg.replace (default): the new value will overwrite the existing value. MergeAlg.add: the new value will be added to the existing raster. default_value : int or float, optional Used as value for all geometries, if not provided in `shapes`. dtype : rasterio or numpy data type, optional Used as data type for results, if `out` is not provided. Returns ------- numpy ndarray If `out` was not None then `out` is returned, it will have been modified in-place. If `out` was None, this will be a new array. Notes ----- Valid data types for `fill`, `default_value`, `out`, `dtype` and shape values are "int16", "int32", "uint8", "uint16", "uint32", "float32", and "float64". This function requires significant memory resources. The shapes iterator will be materialized to a Python list and another C copy of that list will be made. The `out` array will be copied and additional temporary raster memory equal to 2x the smaller of `out` data or GDAL's max cache size (controlled by GDAL_CACHEMAX, default is 5% of the computer's physical memory) is required. If GDAL max cache size is smaller than the output data, the array of shapes will be iterated multiple times. Performance is thus a linear function of buffer size. For maximum speed, ensure that GDAL_CACHEMAX is larger than the size of `out` or `out_shape`. """ valid_dtypes = ('int16', 'int32', 'uint8', 'uint16', 'uint32', 'float32', 'float64') if GDALVersion.runtime().at_least("3.5"): valid_dtypes = valid_dtypes + ("int64", "uint64") def format_invalid_dtype(param): return '{0} dtype must be one of: {1}'.format(param, ', '.join(valid_dtypes)) def format_cast_error(param, dtype): return '{0} cannot be cast to specified dtype: {1}'.format( param, dtype) if fill != 0: fill_array = np.array([fill]) if not validate_dtype(fill_array, valid_dtypes): raise ValueError(format_invalid_dtype('fill')) if dtype is not None and not can_cast_dtype(fill_array, dtype): raise ValueError(format_cast_error('fill', dtype)) if default_value != 1: default_value_array = np.array([default_value]) if not validate_dtype(default_value_array, valid_dtypes): raise ValueError(format_invalid_dtype('default_value')) if dtype is not None and not can_cast_dtype(default_value_array, dtype): raise ValueError(format_cast_error('default_vaue', dtype)) if dtype is not None and _getnpdtype(dtype).name not in valid_dtypes: raise ValueError(format_invalid_dtype('dtype')) valid_shapes = [] shape_values = [] for index, item in enumerate(shapes): if isinstance(item, (tuple, list)): geom, value = item if value is None: value = fill else: geom = item value = default_value geom = getattr(geom, '__geo_interface__', None) or geom if is_valid_geom(geom): shape_values.append(value) geom_type = geom['type'] if geom_type == 'GeometryCollection': # GeometryCollections need to be handled as individual parts to # avoid holes in output: # https://github.com/rasterio/rasterio/issues/1253. # Only 1-level deep since GeoJSON spec discourages nested # GeometryCollections for part in geom['geometries']: valid_shapes.append((part, value)) elif geom_type == 'MultiPolygon': # Same issue as above for poly in geom['coordinates']: valid_shapes.append(({ 'type': 'Polygon', 'coordinates': poly }, value)) else: valid_shapes.append((geom, value)) else: # invalid or empty geometries are skipped and raise a warning instead warnings.warn( 'Invalid or empty shape {} at index {} will not be rasterized.' .format(geom, index), ShapeSkipWarning) if not valid_shapes: raise ValueError('No valid geometry objects found for rasterize') shape_values = np.array(shape_values) if not validate_dtype(shape_values, valid_dtypes): raise ValueError(format_invalid_dtype('shape values')) if dtype is None: dtype = get_minimum_dtype(np.append(shape_values, fill)) elif not can_cast_dtype(shape_values, dtype): raise ValueError(format_cast_error('shape values', dtype)) if out is not None: if _getnpdtype(out.dtype).name not in valid_dtypes: raise ValueError(format_invalid_dtype('out')) if not can_cast_dtype(shape_values, out.dtype): raise ValueError(format_cast_error('shape values', out.dtype.name)) elif out_shape is not None: if len(out_shape) != 2: raise ValueError('Invalid out_shape, must be 2D') out = np.empty(out_shape, dtype=dtype) out.fill(fill) else: raise ValueError('Either an out_shape or image must be provided') if min(out.shape) == 0: raise ValueError("width and height must be > 0") transform = guard_transform(transform) _rasterize(valid_shapes, out, transform, all_touched, merge_alg) return out
from rasterio.warp import ( reproject, transform_geom, transform, transform_bounds, calculate_default_transform, aligned_target, SUPPORTED_RESAMPLING, GDAL2_RESAMPLING, ) from rasterio import windows from .conftest import requires_gdal22, requires_gdal3, requires_gdal_lt_3 gdal_version = GDALVersion.runtime() DST_TRANSFORM = Affine(300.0, 0.0, -8789636.708, 0.0, -300.0, 2943560.235) def flatten_coords(coordinates): """Yield a flat sequence of coordinates to help testing""" for elem in coordinates: if isinstance(elem, (float, int)): yield elem else: for x in flatten_coords(elem): yield x
assert src.overviews(3) == [2] def test_build_overviews_two(data): inputfile = str(data.join('RGB.byte.tif')) with rasterio.open(inputfile, 'r+') as src: overview_factors = [2, 4] src.build_overviews(overview_factors, resampling=OverviewResampling.nearest) assert src.overviews(1) == [2, 4] assert src.overviews(2) == [2, 4] assert src.overviews(3) == [2, 4] @pytest.mark.xfail( GDALVersion.runtime() < GDALVersion.parse("2.0"), reason="Bilinear resampling not supported by GDAL < 2.0", ) def test_build_overviews_bilinear(data): inputfile = str(data.join('RGB.byte.tif')) with rasterio.open(inputfile, 'r+') as src: overview_factors = [2, 4] src.build_overviews(overview_factors, resampling=OverviewResampling.bilinear) assert src.overviews(1) == [2, 4] assert src.overviews(2) == [2, 4] assert src.overviews(3) == [2, 4] def test_build_overviews_average(data): inputfile = str(data.join('RGB.byte.tif'))
@pytest.mark.parametrize("method", SUPPORTED_RESAMPLING) def test_warp_resampling(runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods supported by this version of GDAL should run successfully""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name ]) print(result.output) assert result.exit_code == 0 @pytest.mark.skipif(GDALVersion.runtime().at_least('2.0'), reason="Test only applicable to GDAL < 2.0") @pytest.mark.parametrize("method", GDAL2_RESAMPLING) def test_warp_resampling_not_yet_supported(runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods not yet supported should fail with error""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name ]) assert result.exit_code == 2 assert "Invalid value for" in result.output assert "--resampling" in result.output
def check_raster_file(src_path: str) -> ValidationInfo: # pragma: no cover """ Implementation from https://github.com/cogeotiff/rio-cogeo/blob/0f00a6ee1eff602014fbc88178a069bd9f4a10da/rio_cogeo/cogeo.py This function is the rasterio equivalent of https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py """ errors: List[str] = [] warnings: List[str] = [] details: Dict[str, Any] = {} if not GDALVersion.runtime().at_least('2.2'): raise RuntimeError('GDAL 2.2 or above required') config = dict(GDAL_DISABLE_READDIR_ON_OPEN='FALSE') with rasterio.Env(**config): with rasterio.open(src_path) as src: if not src.driver == 'GTiff': errors.append('The file is not a GeoTIFF') return errors, warnings, details filelist = [os.path.basename(f) for f in src.files] src_bname = os.path.basename(src_path) if len(filelist) > 1 and src_bname + '.ovr' in filelist: errors.append( 'Overviews found in external .ovr file. They should be internal' ) overviews = src.overviews(1) if src.width >= 512 or src.height >= 512: if not src.is_tiled: errors.append( 'The file is greater than 512xH or 512xW, but is not tiled' ) if not overviews: warnings.append( 'The file is greater than 512xH or 512xW, it is recommended ' 'to include internal overviews') ifd_offset = int(src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1)) ifd_offsets = [ifd_offset] if ifd_offset not in (8, 16): errors.append( 'The offset of the main IFD should be 8 for ClassicTIFF ' 'or 16 for BigTIFF. It is {} instead'.format(ifd_offset)) details['ifd_offsets'] = {} details['ifd_offsets']['main'] = ifd_offset if not overviews == sorted(overviews): errors.append('Overviews should be sorted') for ix, dec in enumerate(overviews): # NOTE: Size check is handled in rasterio `src.overviews` methods # https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx # We just need to make sure the decimation level is > 1 if not dec > 1: errors.append( 'Invalid Decimation {} for overview level {}'.format( dec, ix)) # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int( src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) details['ifd_offsets']['overview_{}'.format(ix)] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if ix == 0: errors.append( 'The offset of the IFD for overview of index {} is {}, ' 'whereas it should be greater than the one of the main ' 'image, which is at byte {}'.format( ix, ifd_offsets[-1], ifd_offsets[-2])) else: errors.append( 'The offset of the IFD for overview of index {} is {}, ' 'whereas it should be greater than the one of index {}, ' 'which is at byte {}'.format( ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2])) block_offset = int( src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1)) if not block_offset: errors.append('Missing BLOCK_OFFSET_0_0') data_offset = int(block_offset) if block_offset else 0 data_offsets = [data_offset] details['data_offsets'] = {} details['data_offsets']['main'] = data_offset for ix, dec in enumerate(overviews): data_offset = int( src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1, ovr=ix)) data_offsets.append(data_offset) details['data_offsets']['overview_{}'.format(ix)] = data_offset if data_offsets[-1] < ifd_offsets[-1]: if len(overviews) > 0: errors.append( 'The offset of the first block of the smallest overview ' 'should be after its IFD') else: errors.append( 'The offset of the first block of the image should ' 'be after its IFD') for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors.append( 'The offset of the first block of overview of index {} should ' 'be after the one of the overview of index {}'.format( i - 1, i)) if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors.append( 'The offset of the first block of the main resolution image ' 'should be after the one of the overview of index {}'. format(len(overviews) - 1)) for ix, dec in enumerate(overviews): with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst: if ovr_dst.width >= 512 or ovr_dst.height >= 512: if not ovr_dst.is_tiled: errors.append( 'Overview of index {} is not tiled'.format(ix)) return errors, warnings, details