def test_gdalversion_class_parse(): v = GDALVersion.parse('1.9.0') assert v.major == 1 and v.minor == 9 v = GDALVersion.parse('1.9') assert v.major == 1 and v.minor == 9 v = GDALVersion.parse('1.9a') assert v.major == 1 and v.minor == 9
def test_gdalversion_class_from_string(): v = GDALVersion.from_string('1.9.0') assert v.major == 1 and v.minor == 9 v = GDALVersion.from_string('1.9') assert v.major == 1 and v.minor == 9 v = GDALVersion.from_string('1.9a') assert v.major == 1 and v.minor == 9
def test_gdalversion_class_cmp(): assert GDALVersion(1, 0) == GDALVersion(1, 0) assert GDALVersion(2, 0) > GDALVersion(1, 0) assert GDALVersion(1, 1) > GDALVersion(1, 0) assert GDALVersion(1, 2) < GDALVersion(2, 2) # Because we don't care about patch component assert GDALVersion.parse('1.0') == GDALVersion.parse('1.0.10') assert GDALVersion.parse('1.9') < GDALVersion.parse('2.2.0') assert GDALVersion.parse('2.0.0') > GDALVersion(1, 9)
def test_gdalversion_class_at_least(): assert GDALVersion(2, 1).at_least(GDALVersion(1, 9)) assert GDALVersion(2, 1).at_least((1, 9)) assert GDALVersion(2, 1).at_least('1.9') assert not GDALVersion(2, 1).at_least(GDALVersion(2, 2)) assert not GDALVersion(2, 1).at_least((2, 2)) assert not GDALVersion(2, 1).at_least('2.2')
def _prefered_compression_method() -> str: if not GDALVersion.runtime().at_least('2.3'): return 'DEFLATE' # check if we can use ZSTD (fails silently for GDAL < 2.3) dummy_profile = dict(driver='GTiff', height=1, width=1, count=1, dtype='uint8') try: with warnings.catch_warnings(): warnings.simplefilter('ignore', NotGeoreferencedWarning) with MemoryFile() as memfile, memfile.open(compress='ZSTD', **dummy_profile): pass except Exception as exc: if 'missing codec' not in str(exc): raise else: return 'ZSTD' return 'DEFLATE'
def rasterio_crs(self): """Return rasterio CRS.""" import rasterio from rasterio.env import GDALVersion if GDALVersion.runtime().major < 3: return rasterio.crs.CRS.from_wkt( self.crs.to_wkt(WktVersion.WKT1_GDAL)) else: return rasterio.crs.CRS.from_wkt(self.crs.to_wkt())
def driver_from_extension(path): """ Attempt to auto-detect driver based on the extension. Parameters ---------- path: str or pathlike object The path to the dataset to write with. Returns ------- str: The name of the driver for the extension. """ try: # in case the path is a file handle # or a partsed path path = path.name except AttributeError: pass # dynamic driver extension lists added in GDAL 2 if GDALVersion().runtime() < GDALVersion.parse('2.0'): # basic list for GDAL 1 driver_extensions = { 'tif': 'GTiff', 'tiff': 'GTiff', 'png': 'PNG', 'jpg': 'JPEG', 'jpeg': 'JPEG', } else: driver_extensions = raster_driver_extensions() try: return driver_extensions[os.path.splitext(path)[-1].lstrip( ".").lower()] except KeyError: raise ValueError("Unable to detect driver. Please specify driver.")
def transform_geom(src_crs, dst_crs, geom, antimeridian_cutting=True, antimeridian_offset=10.0, precision=-1): """Transform geometry from source coordinate reference system into target. Parameters ------------ src_crs: CRS or dict Source coordinate reference system, in rasterio dict format. Example: CRS({'init': 'EPSG:4326'}) dst_crs: CRS or dict Target coordinate reference system. geom: GeoJSON like dict object antimeridian_cutting: bool, optional If True, cut geometries at the antimeridian, otherwise geometries will not be cut (default). If False and GDAL is 2.2.0 or newer an exception is raised. Antimeridian cutting is always on as of GDAL 2.2.0 but this could produce an unexpected geometry. antimeridian_offset: float Offset from the antimeridian in degrees (default: 10) within which any geometries will be split. precision: float If >= 0, geometry coordinates will be rounded to this number of decimal places after the transform operation, otherwise original coordinate values will be preserved (default). Returns --------- out: GeoJSON like dict object Transformed geometry in GeoJSON dict format """ if (GDALVersion.runtime().at_least('2.2') and not antimeridian_cutting): raise GDALBehaviorChangeException( "Antimeridian cutting is always enabled on GDAL 2.2.0 or " "newer, which could produce a different geometry than expected.") return _transform_geom(src_crs, dst_crs, geom, antimeridian_cutting, antimeridian_offset, precision)
def test_gdalversion_class_at_least_invalid_type(): invalids_types = ({}, {'major': 1, 'minor': 1}, [1, 2]) for invalid in invalids_types: with pytest.raises(TypeError): GDALVersion(2, 1).at_least(invalid)
def cog_validate(src_path): """ Validate Cloud Optimized Geotiff. Parameters ---------- src_path : str or PathLike object A dataset path or URL. Will be opened in "r" mode. This script is the rasterio equivalent of https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py """ errors = [] warnings = [] details = {} if not GDALVersion.runtime().at_least("2.2"): raise Exception("GDAL 2.2 or above required") config = dict(GDAL_DISABLE_READDIR_ON_OPEN="FALSE") with rasterio.Env(**config): with rasterio.open(src_path) as src: if not src.driver == "GTiff": raise Exception("The file is not a GeoTIFF") filelist = [os.path.basename(f) for f in src.files] src_bname = os.path.basename(src_path) if len(filelist) > 1 and src_bname + ".ovr" in filelist: errors.append( "Overviews found in external .ovr file. They should be internal" ) overviews = src.overviews(1) if src.width > 512 or src.height > 512: if not src.is_tiled: errors.append( "The file is greater than 512xH or 512xW, but is not tiled" ) if not overviews: warnings.append( "The file is greater than 512xH or 512xW, it is recommended " "to include internal overviews") ifd_offset = int(src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1)) ifd_offsets = [ifd_offset] if ifd_offset not in (8, 16): errors.append( "The offset of the main IFD should be 8 for ClassicTIFF " "or 16 for BigTIFF. It is {} instead".format(ifd_offset)) details["ifd_offsets"] = {} details["ifd_offsets"]["main"] = ifd_offset if overviews and overviews != sorted(overviews): errors.append("Overviews should be sorted") for ix, dec in enumerate(overviews): # NOTE: Size check is handled in rasterio `src.overviews` methods # https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx # We just need to make sure the decimation level is > 1 if not dec > 1: errors.append( "Invalid Decimation {} for overview level {}".format( dec, ix)) # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int( src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) details["ifd_offsets"]["overview_{}".format(ix)] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if ix == 0: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of the main " "image, which is at byte {}".format( ix, ifd_offsets[-1], ifd_offsets[-2])) else: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of index {}, " "which is at byte {}".format( ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2])) block_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1)) if not block_offset: errors.append("Missing BLOCK_OFFSET_0_0") data_offset = int(block_offset) if block_offset else None data_offsets = [data_offset] details["data_offsets"] = {} details["data_offsets"]["main"] = data_offset for ix, dec in enumerate(overviews): data_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1, ovr=ix)) data_offsets.append(data_offset) details["data_offsets"]["overview_{}".format(ix)] = data_offset if data_offsets[-1] < ifd_offsets[-1]: if len(overviews) > 0: errors.append( "The offset of the first block of the smallest overview " "should be after its IFD") else: errors.append( "The offset of the first block of the image should " "be after its IFD") for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors.append( "The offset of the first block of overview of index {} should " "be after the one of the overview of index {}".format( i - 1, i)) if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors.append( "The offset of the first block of the main resolution image " "should be after the one of the overview of index {}". format(len(overviews) - 1)) for ix, dec in enumerate(overviews): with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst: if ovr_dst.width >= 512 or ovr_dst.height >= 512: if not ovr_dst.is_tiled: errors.append( "Overview of index {} is not tiled".format(ix)) if warnings: click.secho("The following warnings were found:", fg="yellow", err=True) for w in warnings: click.echo("- " + w, err=True) click.echo(err=True) if errors: click.secho("The following errors were found:", fg="red", err=True) for e in errors: click.echo("- " + e, err=True) return False return True
def test_gdalversion_class_str(): assert str(GDALVersion(2, 1)) == '2.1'
from rasterio.warp import ( reproject, transform_geom, transform, transform_bounds, calculate_default_transform, aligned_target, SUPPORTED_RESAMPLING, GDAL2_RESAMPLING, ) from rasterio import windows from .conftest import requires_gdal22, requires_gdal3, requires_gdal_lt_3 gdal_version = GDALVersion.runtime() DST_TRANSFORM = Affine(300.0, 0.0, -8789636.708, 0.0, -300.0, 2943560.235) def flatten_coords(coordinates): """Yield a flat sequence of coordinates to help testing""" for elem in coordinates: if isinstance(elem, (float, int)): yield elem else: for x in flatten_coords(elem): yield x
def test_gdalversion_class_parse_err(): invalids = ('foo', 'foo.bar', '1', '1.', '1.a', '.1') for invalid in invalids: with pytest.raises(ValueError): GDALVersion.parse(invalid)
if not os.path.exists(path): with zipfile.ZipFile(path, 'w') as zip: for filename in ['white-gemini-iv.vrt', '389225main_sw_1965_1024.jpg']: zip.write(os.path.join(data_dir, filename), filename) return path class MockGeoInterface(object): """Tiny wrapper for GeoJSON to present an object with __geo_interface__ for testing""" def __init__(self, geojson): self.__geo_interface__ = geojson # Define helpers to skip tests based on GDAL version gdal_version = GDALVersion.runtime() requires_only_gdal1 = pytest.mark.skipif( gdal_version.major != 1, reason="Only relevant for GDAL 1.x") requires_gdal2 = pytest.mark.skipif( not gdal_version.major >= 2, reason="Requires GDAL 2.x") requires_gdal21 = pytest.mark.skipif( not gdal_version.at_least('2.1'), reason="Requires GDAL 2.1.x") requires_gdal22 = pytest.mark.skipif( not gdal_version.at_least('2.2'),
def test_warp_resampling(runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods supported by this version of GDAL should run successfully""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name]) print(result.output) assert result.exit_code == 0 @pytest.mark.skipif( GDALVersion.runtime().at_least('2.0'), reason="Test only applicable to GDAL < 2.0") @pytest.mark.parametrize("method", GDAL2_RESAMPLING) def test_warp_resampling_not_yet_supported( runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods not yet supported should fail with error""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name]) assert result.exit_code == 2 assert "Invalid value for" in result.output assert "--resampling" in result.output
def check_raster_file(src_path: str) -> ValidationInfo: # pragma: no cover """ Implementation from https://github.com/cogeotiff/rio-cogeo/blob/0f00a6ee1eff602014fbc88178a069bd9f4a10da/rio_cogeo/cogeo.py This function is the rasterio equivalent of https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py """ errors: List[str] = [] warnings: List[str] = [] details: Dict[str, Any] = {} if not GDALVersion.runtime().at_least('2.2'): raise RuntimeError('GDAL 2.2 or above required') config = dict(GDAL_DISABLE_READDIR_ON_OPEN='FALSE') with rasterio.Env(**config): with rasterio.open(src_path) as src: if not src.driver == 'GTiff': errors.append('The file is not a GeoTIFF') return errors, warnings, details filelist = [os.path.basename(f) for f in src.files] src_bname = os.path.basename(src_path) if len(filelist) > 1 and src_bname + '.ovr' in filelist: errors.append( 'Overviews found in external .ovr file. They should be internal' ) overviews = src.overviews(1) if src.width >= 512 or src.height >= 512: if not src.is_tiled: errors.append( 'The file is greater than 512xH or 512xW, but is not tiled' ) if not overviews: warnings.append( 'The file is greater than 512xH or 512xW, it is recommended ' 'to include internal overviews') ifd_offset = int(src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1)) ifd_offsets = [ifd_offset] if ifd_offset not in (8, 16): errors.append( 'The offset of the main IFD should be 8 for ClassicTIFF ' 'or 16 for BigTIFF. It is {} instead'.format(ifd_offset)) details['ifd_offsets'] = {} details['ifd_offsets']['main'] = ifd_offset if not overviews == sorted(overviews): errors.append('Overviews should be sorted') for ix, dec in enumerate(overviews): # NOTE: Size check is handled in rasterio `src.overviews` methods # https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx # We just need to make sure the decimation level is > 1 if not dec > 1: errors.append( 'Invalid Decimation {} for overview level {}'.format( dec, ix)) # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int( src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) details['ifd_offsets']['overview_{}'.format(ix)] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if ix == 0: errors.append( 'The offset of the IFD for overview of index {} is {}, ' 'whereas it should be greater than the one of the main ' 'image, which is at byte {}'.format( ix, ifd_offsets[-1], ifd_offsets[-2])) else: errors.append( 'The offset of the IFD for overview of index {} is {}, ' 'whereas it should be greater than the one of index {}, ' 'which is at byte {}'.format( ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2])) block_offset = int( src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1)) if not block_offset: errors.append('Missing BLOCK_OFFSET_0_0') data_offset = int(block_offset) if block_offset else 0 data_offsets = [data_offset] details['data_offsets'] = {} details['data_offsets']['main'] = data_offset for ix, dec in enumerate(overviews): data_offset = int( src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1, ovr=ix)) data_offsets.append(data_offset) details['data_offsets']['overview_{}'.format(ix)] = data_offset if data_offsets[-1] < ifd_offsets[-1]: if len(overviews) > 0: errors.append( 'The offset of the first block of the smallest overview ' 'should be after its IFD') else: errors.append( 'The offset of the first block of the image should ' 'be after its IFD') for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors.append( 'The offset of the first block of overview of index {} should ' 'be after the one of the overview of index {}'.format( i - 1, i)) if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors.append( 'The offset of the first block of the main resolution image ' 'should be after the one of the overview of index {}'. format(len(overviews) - 1)) for ix, dec in enumerate(overviews): with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst: if ovr_dst.width >= 512 or ovr_dst.height >= 512: if not ovr_dst.is_tiled: errors.append( 'Overview of index {} is not tiled'.format(ix)) return errors, warnings, details
from io import BytesIO import logging import pytest import rasterio from rasterio.io import MemoryFile, ZipMemoryFile from rasterio.env import GDALVersion logging.basicConfig(level=logging.DEBUG) # Skip ENTIRE module if not GDAL >= 2.x. # pytestmark is a keyword that instructs pytest to skip this module. pytestmark = pytest.mark.skipif( not GDALVersion.runtime().major >= 2, reason="MemoryFile requires GDAL 2.x") @pytest.fixture(scope='session') def rgb_file_bytes(path_rgb_byte_tif): """Get the bytes of our RGB.bytes.tif file""" return open(path_rgb_byte_tif, 'rb').read() @pytest.fixture(scope='session') def rgb_lzw_file_bytes(): """Get the bytes of our RGB.bytes.tif file""" return open('tests/data/rgb_lzw.tif', 'rb').read()
assert src.overviews(3) == [2] def test_build_overviews_two(data): inputfile = str(data.join('RGB.byte.tif')) with rasterio.open(inputfile, 'r+') as src: overview_factors = [2, 4] src.build_overviews(overview_factors, resampling=OverviewResampling.nearest) assert src.overviews(1) == [2, 4] assert src.overviews(2) == [2, 4] assert src.overviews(3) == [2, 4] @pytest.mark.xfail( GDALVersion.runtime() < GDALVersion.parse("2.0"), reason="Bilinear resampling not supported by GDAL < 2.0", ) def test_build_overviews_bilinear(data): inputfile = str(data.join('RGB.byte.tif')) with rasterio.open(inputfile, 'r+') as src: overview_factors = [2, 4] src.build_overviews(overview_factors, resampling=OverviewResampling.bilinear) assert src.overviews(1) == [2, 4] assert src.overviews(2) == [2, 4] assert src.overviews(3) == [2, 4] def test_build_overviews_average(data): inputfile = str(data.join('RGB.byte.tif'))
def validate(src_path: str) -> bool: """Validate given cloud-optimized GeoTIFF""" if not GDALVersion.runtime().at_least('2.2'): raise RuntimeError('GDAL 2.2 or above required') with rasterio.open(src_path) as src: if not src.driver == 'GTiff': # Not a GeoTIFF return False filelist = [os.path.basename(f) for f in src.files] src_bname = os.path.basename(src_path) if len(filelist) > 1 and src_bname + '.ovr' in filelist: # Overviews found in external .ovr file. They should be internal return False overviews = src.overviews(1) or [] if src.width >= 512 or src.height >= 512: if not src.is_tiled: # The file is greater than 512xH or 512xW, but is not tiled return False if not overviews: # The file is greater than 512xH or 512xW, but has no overviews return False ifd_offset = int(src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1)) ifd_offsets = [ifd_offset] if ifd_offset not in (8, 16): # The offset of the main IFD should be 8 for ClassicTIFF or 16 for BigTIFF return False if not overviews == sorted(overviews): # Overviews should be sorted return False for ix, dec in enumerate(overviews): if not dec > 1: # Invalid Decimation return False # TODO: Check if the overviews are tiled # NOTE: There is currently no way to do that with rasterio # Check that the IFD of descending overviews are sorted by increasing offsets ifd_offset = int( src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) if ifd_offsets[-1] < ifd_offsets[-2]: return False block_offset = int(src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1)) if not block_offset: return False data_offset = int(block_offset) data_offsets = [data_offset] for ix, dec in enumerate(overviews): data_offset = int( src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1, ovr=ix)) data_offsets.append(data_offset) if data_offsets[-1] < ifd_offsets[-1]: return False for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: return False if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: return False return True
import logging import os.path from affine import Affine import numpy import pytest import rasterio from rasterio.io import MemoryFile, ZipMemoryFile from rasterio.env import GDALVersion # Skip ENTIRE module if not GDAL >= 2.x. # pytestmark is a keyword that instructs pytest to skip this module. pytestmark = pytest.mark.skipif( not GDALVersion.runtime().major >= 2, reason="MemoryFile requires GDAL 2.x") @pytest.fixture(scope='session') def rgb_file_bytes(path_rgb_byte_tif): """Get the bytes of our RGB.bytes.tif file""" return open(path_rgb_byte_tif, 'rb').read() @pytest.fixture(scope='session') def rgb_lzw_file_bytes(): """Get the bytes of our RGB.bytes.tif file""" return open('tests/data/rgb_lzw.tif', 'rb').read()
with caplog.at_level(logging.INFO): with rasterio.open(path_rgb_byte_tif) as src: _ = src.colorinterp t, w, h = calculate_default_transform( 'PROJCS["unknown",GEOGCS["unknown",DATUM["unknown",SPHEROID["GRS 1980",6378137,298.257222096042]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433]],PROJECTION["Geostationary_Satellite"],PARAMETER["central_meridian",-137],PARAMETER["satellite_height",35786023],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],EXTENSION["PROJ4","+proj=geos +sweep=x +lon_0=-137 +h=35786023 +x_0=0 +y_0=0 +ellps=GRS80 +units=m +no_defs"]]', "EPSG:4326", 21696, 21696, -5434894.885056, -5434894.885056, 5434894.885056, 5434894.885056, ) _ = src.colorinterp @pytest.mark.xfail(gdal_version < GDALVersion(3, 3), reason="GDAL <3.3 will not warn") @pytest.mark.xfail(gdal_version > GDALVersion(3, 3), reason="GDAL > 3.3 will not warn") def test_issue2353bis(caplog, path_rgb_byte_tif): """Ensure VRT doesn't leave errors behind.""" from rasterio.vrt import WarpedVRT with caplog.at_level(logging.INFO): with rasterio.open('tests/data/goes.tif') as src: with WarpedVRT(src, dst_crs="EPSG:3857") as vrt: pass assert "Ignoring error" in caplog.text _ = src.colorinterp
import os import click import numpy from rasterio.enums import Resampling as ResamplingEnums from rasterio.env import GDALVersion from rasterio.rio import options from rasterio.warp import SUPPORTED_RESAMPLING as WarpResampling from rio_cogeo import __version__ as cogeo_version from rio_cogeo.cogeo import cog_info, cog_translate, cog_validate from rio_cogeo.profiles import cog_profiles OverviewResampling = [r for r in ResamplingEnums if r.value < 8] if GDALVersion.runtime().at_least("3.3"): OverviewResampling.append(ResamplingEnums.rms) IN_MEMORY_THRESHOLD = int(os.environ.get("IN_MEMORY_THRESHOLD", 10980 * 10980)) class BdxParamType(click.ParamType): """Band index type.""" name = "bidx" def convert(self, value, param, ctx): """Validate and parse band index.""" try: bands = [int(x) for x in value.split(",")] assert all(b > 0 for b in bands)
def cog_validate( # noqa: C901 src_path: Union[str, pathlib.PurePath], strict: bool = False, quiet: bool = False) -> Tuple[bool, List[str], List[str]]: """ Validate Cloud Optimized Geotiff. This script is the rasterio equivalent of https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py Parameters ---------- src_path: str or PathLike object A dataset path or URL. Will be opened in "r" mode. strict: bool Treat warnings as errors quiet: bool Remove standard outputs Returns ------- is_valid: bool True is src_path is a valid COG. errors: list List of validation errors. warnings: list List of validation warnings. """ if isinstance(src_path, str): src_path = pathlib.Path(src_path) errors = [] warnings = [] details: Dict[str, Any] = {} if not GDALVersion.runtime().at_least("2.2"): raise Exception("GDAL 2.2 or above required") config = dict(GDAL_DISABLE_READDIR_ON_OPEN="FALSE") with rasterio.Env(**config): with rasterio.open(src_path) as src: if not src.driver == "GTiff": raise Exception("The file is not a GeoTIFF") filelist = [pathlib.Path(f).name for f in src.files] if len(filelist) > 1 and f"{src_path.name}.ovr" in filelist: errors.append( "Overviews found in external .ovr file. They should be internal" ) overviews = src.overviews(1) if src.width > 512 or src.height > 512: if not src.is_tiled: errors.append( "The file is greater than 512xH or 512xW, but is not tiled" ) if not overviews: warnings.append( "The file is greater than 512xH or 512xW, it is recommended " "to include internal overviews") ifd_offset = int(src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1)) # Starting from GDAL 3.1, GeoTIFF and COG have ghost headers # e.g: # """ # GDAL_STRUCTURAL_METADATA_SIZE=000140 bytes # LAYOUT=IFDS_BEFORE_DATA # BLOCK_ORDER=ROW_MAJOR # BLOCK_LEADER=SIZE_AS_UINT4 # BLOCK_TRAILER=LAST_4_BYTES_REPEATED # KNOWN_INCOMPATIBLE_EDITION=NO # """ # # This header should be < 200bytes if ifd_offset > 300: errors.append( f"The offset of the main IFD should be < 300. It is {ifd_offset} instead" ) ifd_offsets = [ifd_offset] details["ifd_offsets"] = {} details["ifd_offsets"]["main"] = ifd_offset if overviews and overviews != sorted(overviews): errors.append("Overviews should be sorted") for ix, dec in enumerate(overviews): # NOTE: Size check is handled in rasterio `src.overviews` methods # https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx # We just need to make sure the decimation level is > 1 if not dec > 1: errors.append( "Invalid Decimation {} for overview level {}".format( dec, ix)) # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int( src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) details["ifd_offsets"]["overview_{}".format(ix)] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if ix == 0: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of the main " "image, which is at byte {}".format( ix, ifd_offsets[-1], ifd_offsets[-2])) else: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of index {}, " "which is at byte {}".format( ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2])) block_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1)) if not block_offset: errors.append("Missing BLOCK_OFFSET_0_0") data_offset = int(block_offset) if block_offset else None data_offsets = [data_offset] details["data_offsets"] = {} details["data_offsets"]["main"] = data_offset for ix, dec in enumerate(overviews): data_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1, ovr=ix)) data_offsets.append(data_offset) details["data_offsets"]["overview_{}".format(ix)] = data_offset if data_offsets[-1] < ifd_offsets[-1]: if len(overviews) > 0: errors.append( "The offset of the first block of the smallest overview " "should be after its IFD") else: errors.append( "The offset of the first block of the image should " "be after its IFD") for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors.append( "The offset of the first block of overview of index {} should " "be after the one of the overview of index {}".format( i - 1, i)) if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors.append( "The offset of the first block of the main resolution image " "should be after the one of the overview of index {}". format(len(overviews) - 1)) for ix, dec in enumerate(overviews): with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst: if ovr_dst.width >= 512 or ovr_dst.height >= 512: if not ovr_dst.is_tiled: errors.append( "Overview of index {} is not tiled".format(ix)) if warnings and not quiet: click.secho("The following warnings were found:", fg="yellow", err=True) for w in warnings: click.echo("- " + w, err=True) click.echo(err=True) if errors and not quiet: click.secho("The following errors were found:", fg="red", err=True) for e in errors: click.echo("- " + e, err=True) is_valid = False if errors or (warnings and strict) else True return is_valid, errors, warnings
def test_warp_resampling(runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods supported by this version of GDAL should run successfully""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name]) print(result.output) assert result.exit_code == 0 @pytest.mark.skipif( GDALVersion.runtime().at_least('2.0'), reason="Test only applicable to GDAL < 2.0") @pytest.mark.parametrize("method", GDAL2_RESAMPLING) def test_warp_resampling_not_yet_supported( runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods not yet supported should fail with error""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name]) assert result.exit_code == 2 assert 'Invalid value for "--resampling"' in result.output
import logging import os.path from affine import Affine import numpy import pytest import rasterio from rasterio.io import MemoryFile, ZipMemoryFile from rasterio.enums import MaskFlags from rasterio.env import GDALVersion from rasterio.shutil import copyfiles # Skip ENTIRE module if not GDAL >= 2.x. # pytestmark is a keyword that instructs pytest to skip this module. pytestmark = pytest.mark.skipif(not GDALVersion.runtime().major >= 2, reason="MemoryFile requires GDAL 2.x") @pytest.fixture(scope='session') def rgb_file_bytes(path_rgb_byte_tif): """Get the bytes of our RGB.bytes.tif file""" return open(path_rgb_byte_tif, 'rb').read() @pytest.fixture(scope='session') def rgb_lzw_file_bytes(): """Get the bytes of our RGB.bytes.tif file""" return open('tests/data/rgb_lzw.tif', 'rb').read()
assert src.overviews(2) == [2] assert src.overviews(3) == [2] def test_build_overviews_two(data): inputfile = str(data.join('RGB.byte.tif')) with rasterio.open(inputfile, 'r+') as src: overview_factors = [2, 4] src.build_overviews(overview_factors, resampling=Resampling.nearest) assert src.overviews(1) == [2, 4] assert src.overviews(2) == [2, 4] assert src.overviews(3) == [2, 4] @pytest.mark.xfail( gdal_version < GDALVersion.parse('2.0'), reason="Bilinear resampling not supported by GDAL < 2.0") def test_build_overviews_bilinear(data): inputfile = str(data.join('RGB.byte.tif')) with rasterio.open(inputfile, 'r+') as src: overview_factors = [2, 4] src.build_overviews(overview_factors, resampling=Resampling.bilinear) assert src.overviews(1) == [2, 4] assert src.overviews(2) == [2, 4] assert src.overviews(3) == [2, 4] def test_build_overviews_average(data): inputfile = str(data.join('RGB.byte.tif')) with rasterio.open(inputfile, 'r+') as src: overview_factors = [2, 4]
def rasterize(shapes, out_shape=None, fill=0, out=None, transform=IDENTITY, all_touched=False, merge_alg=MergeAlg.replace, default_value=1, dtype=None): """Return an image array with input geometries burned in. Warnings will be raised for any invalid or empty geometries, and an exception will be raised if there are no valid shapes to rasterize. Parameters ---------- shapes : iterable of (`geometry`, `value`) pairs or geometries The `geometry` can either be an object that implements the geo interface or GeoJSON-like object. If no `value` is provided the `default_value` will be used. If `value` is `None` the `fill` value will be used. out_shape : tuple or list with 2 integers Shape of output numpy ndarray. fill : int or float, optional Used as fill value for all areas not covered by input geometries. out : numpy ndarray, optional Array of same shape and data type as `source` in which to store results. transform : Affine transformation object, optional Transformation from pixel coordinates of `source` to the coordinate system of the input `shapes`. See the `transform` property of dataset objects. all_touched : boolean, optional If True, all pixels touched by geometries will be burned in. If false, only pixels whose center is within the polygon or that are selected by Bresenham's line algorithm will be burned in. merge_alg : MergeAlg, optional Merge algorithm to use. One of: MergeAlg.replace (default): the new value will overwrite the existing value. MergeAlg.add: the new value will be added to the existing raster. default_value : int or float, optional Used as value for all geometries, if not provided in `shapes`. dtype : rasterio or numpy data type, optional Used as data type for results, if `out` is not provided. Returns ------- numpy ndarray If `out` was not None then `out` is returned, it will have been modified in-place. If `out` was None, this will be a new array. Notes ----- Valid data types for `fill`, `default_value`, `out`, `dtype` and shape values are "int16", "int32", "uint8", "uint16", "uint32", "float32", and "float64". This function requires significant memory resources. The shapes iterator will be materialized to a Python list and another C copy of that list will be made. The `out` array will be copied and additional temporary raster memory equal to 2x the smaller of `out` data or GDAL's max cache size (controlled by GDAL_CACHEMAX, default is 5% of the computer's physical memory) is required. If GDAL max cache size is smaller than the output data, the array of shapes will be iterated multiple times. Performance is thus a linear function of buffer size. For maximum speed, ensure that GDAL_CACHEMAX is larger than the size of `out` or `out_shape`. """ valid_dtypes = ('int16', 'int32', 'uint8', 'uint16', 'uint32', 'float32', 'float64') if GDALVersion.runtime().at_least("3.5"): valid_dtypes = valid_dtypes + ("int64", "uint64") def format_invalid_dtype(param): return '{0} dtype must be one of: {1}'.format(param, ', '.join(valid_dtypes)) def format_cast_error(param, dtype): return '{0} cannot be cast to specified dtype: {1}'.format( param, dtype) if fill != 0: fill_array = np.array([fill]) if not validate_dtype(fill_array, valid_dtypes): raise ValueError(format_invalid_dtype('fill')) if dtype is not None and not can_cast_dtype(fill_array, dtype): raise ValueError(format_cast_error('fill', dtype)) if default_value != 1: default_value_array = np.array([default_value]) if not validate_dtype(default_value_array, valid_dtypes): raise ValueError(format_invalid_dtype('default_value')) if dtype is not None and not can_cast_dtype(default_value_array, dtype): raise ValueError(format_cast_error('default_vaue', dtype)) if dtype is not None and _getnpdtype(dtype).name not in valid_dtypes: raise ValueError(format_invalid_dtype('dtype')) valid_shapes = [] shape_values = [] for index, item in enumerate(shapes): if isinstance(item, (tuple, list)): geom, value = item if value is None: value = fill else: geom = item value = default_value geom = getattr(geom, '__geo_interface__', None) or geom if is_valid_geom(geom): shape_values.append(value) geom_type = geom['type'] if geom_type == 'GeometryCollection': # GeometryCollections need to be handled as individual parts to # avoid holes in output: # https://github.com/rasterio/rasterio/issues/1253. # Only 1-level deep since GeoJSON spec discourages nested # GeometryCollections for part in geom['geometries']: valid_shapes.append((part, value)) elif geom_type == 'MultiPolygon': # Same issue as above for poly in geom['coordinates']: valid_shapes.append(({ 'type': 'Polygon', 'coordinates': poly }, value)) else: valid_shapes.append((geom, value)) else: # invalid or empty geometries are skipped and raise a warning instead warnings.warn( 'Invalid or empty shape {} at index {} will not be rasterized.' .format(geom, index), ShapeSkipWarning) if not valid_shapes: raise ValueError('No valid geometry objects found for rasterize') shape_values = np.array(shape_values) if not validate_dtype(shape_values, valid_dtypes): raise ValueError(format_invalid_dtype('shape values')) if dtype is None: dtype = get_minimum_dtype(np.append(shape_values, fill)) elif not can_cast_dtype(shape_values, dtype): raise ValueError(format_cast_error('shape values', dtype)) if out is not None: if _getnpdtype(out.dtype).name not in valid_dtypes: raise ValueError(format_invalid_dtype('out')) if not can_cast_dtype(shape_values, out.dtype): raise ValueError(format_cast_error('shape values', out.dtype.name)) elif out_shape is not None: if len(out_shape) != 2: raise ValueError('Invalid out_shape, must be 2D') out = np.empty(out_shape, dtype=dtype) out.fill(fill) else: raise ValueError('Either an out_shape or image must be provided') if min(out.shape) == 0: raise ValueError("width and height must be > 0") transform = guard_transform(transform) _rasterize(valid_shapes, out, transform, all_touched, merge_alg) return out
def test_gdalversion_class_runtime(): """Test the version of GDAL from this runtime""" GDALVersion.runtime().major >= 1
def test_gdalversion_class_repr(): assert (GDALVersion(2, 1)).__repr__() == 'GDALVersion(major=2, minor=1)'
def check_raster_file(src_path: str) -> ValidationInfo: # pragma: no cover """ Implementation from https://github.com/cogeotiff/rio-cogeo/blob/a07d914e2d898878417638bbc089179f01eb5b28/rio_cogeo/cogeo.py#L385 This function is the rasterio equivalent of https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py """ errors: List[str] = [] warnings: List[str] = [] details: Dict[str, Any] = {} if not GDALVersion.runtime().at_least('2.2'): raise RuntimeError('GDAL 2.2 or above required') config = dict(GDAL_DISABLE_READDIR_ON_OPEN='FALSE') with rasterio.Env(**config): with rasterio.open(src_path) as src: if not src.driver == 'GTiff': errors.append('The file is not a GeoTIFF') return errors, warnings, details if any(os.path.splitext(x)[-1] == '.ovr' for x in src.files): errors.append( 'Overviews found in external .ovr file. They should be internal' ) overviews = src.overviews(1) if src.width > 512 and src.height > 512: if not src.is_tiled: errors.append( 'The file is greater than 512xH or 512xW, but is not tiled' ) if not overviews: warnings.append( 'The file is greater than 512xH or 512xW, it is recommended ' 'to include internal overviews') ifd_offset = int(src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1)) # Starting from GDAL 3.1, GeoTIFF and COG have ghost headers # e.g: # """ # GDAL_STRUCTURAL_METADATA_SIZE=000140 bytes # LAYOUT=IFDS_BEFORE_DATA # BLOCK_ORDER=ROW_MAJOR # BLOCK_LEADER=SIZE_AS_UINT4 # BLOCK_TRAILER=LAST_4_BYTES_REPEATED # KNOWN_INCOMPATIBLE_EDITION=NO # """ # # This header should be < 200bytes if ifd_offset > 300: errors.append( f'The offset of the main IFD should be < 300. It is {ifd_offset} instead' ) ifd_offsets = [ifd_offset] details['ifd_offsets'] = {} details['ifd_offsets']['main'] = ifd_offset if overviews and overviews != sorted(overviews): errors.append('Overviews should be sorted') for ix, dec in enumerate(overviews): # NOTE: Size check is handled in rasterio `src.overviews` methods # https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx # We just need to make sure the decimation level is > 1 if not dec > 1: errors.append( 'Invalid Decimation {} for overview level {}'.format( dec, ix)) # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int( src.get_tag_item('IFD_OFFSET', 'TIFF', bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) details['ifd_offsets']['overview_{}'.format(ix)] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if ix == 0: errors.append( 'The offset of the IFD for overview of index {} is {}, ' 'whereas it should be greater than the one of the main ' 'image, which is at byte {}'.format( ix, ifd_offsets[-1], ifd_offsets[-2])) else: errors.append( 'The offset of the IFD for overview of index {} is {}, ' 'whereas it should be greater than the one of index {}, ' 'which is at byte {}'.format( ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2])) block_offset = src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1) data_offset = int(block_offset) if block_offset else 0 data_offsets = [data_offset] details['data_offsets'] = {} details['data_offsets']['main'] = data_offset for ix, dec in enumerate(overviews): block_offset = src.get_tag_item('BLOCK_OFFSET_0_0', 'TIFF', bidx=1, ovr=ix) data_offset = int(block_offset) if block_offset else 0 data_offsets.append(data_offset) details['data_offsets']['overview_{}'.format(ix)] = data_offset if data_offsets[-1] != 0 and data_offsets[-1] < ifd_offsets[-1]: if len(overviews) > 0: errors.append( 'The offset of the first block of the smallest overview ' 'should be after its IFD') else: errors.append( 'The offset of the first block of the image should ' 'be after its IFD') for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors.append( 'The offset of the first block of overview of index {} should ' 'be after the one of the overview of index {}'.format( i - 1, i)) if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors.append( 'The offset of the first block of the main resolution image ' 'should be after the one of the overview of index {}'. format(len(overviews) - 1)) for ix, dec in enumerate(overviews): with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst: if ovr_dst.width > 512 and ovr_dst.height > 512: if not ovr_dst.is_tiled: errors.append( 'Overview of index {} is not tiled'.format(ix)) return errors, warnings, details
"""Mapping of GDAL to Numpy data types. Since 0.13 we are not importing numpy here and data types are strings. Happily strings can be used throughout Numpy and so existing code will not break. """ import numpy from rasterio.env import GDALVersion _GDAL_AT_LEAST_35 = GDALVersion.runtime().at_least("3.5") bool_ = 'bool' ubyte = uint8 = 'uint8' sbyte = int8 = 'int8' uint16 = 'uint16' int16 = 'int16' uint32 = 'uint32' int32 = 'int32' uint64 = 'uint32' int64 = 'int64' float32 = 'float32' float64 = 'float64' complex_ = 'complex' complex64 = 'complex64' complex128 = 'complex128' complex_int16 = "complex_int16" dtype_fwd = {
from rasterio._base import _transform from rasterio.enums import Resampling from rasterio.env import GDALVersion, ensure_env, require_gdal_version from rasterio.errors import GDALBehaviorChangeException, TransformError from rasterio.transform import array_bounds from rasterio._warp import ( _calculate_default_transform, _reproject, _transform_bounds, _transform_geom, ) # Gauss (7) is not supported for warp SUPPORTED_RESAMPLING = [r for r in Resampling if r.value < 7] GDAL2_RESAMPLING = [r for r in Resampling if r.value > 7 and r.value <= 12] if GDALVersion.runtime().at_least('2.0'): SUPPORTED_RESAMPLING.extend(GDAL2_RESAMPLING) # sum supported since GDAL 3.1 if GDALVersion.runtime().at_least('3.1'): SUPPORTED_RESAMPLING.append(Resampling.sum) # rms supported since GDAL 3.3 if GDALVersion.runtime().at_least('3.3'): SUPPORTED_RESAMPLING.append(Resampling.rms) @ensure_env def transform(src_crs, dst_crs, xs, ys, zs=None): """Transform vectors from source to target coordinate reference system. Transform vectors of x, y and optionally z from source
@pytest.mark.parametrize("method", SUPPORTED_RESAMPLING) def test_warp_resampling(runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods supported by this version of GDAL should run successfully""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name ]) print(result.output) assert result.exit_code == 0 @pytest.mark.skipif(GDALVersion.runtime().at_least('2.0'), reason="Test only applicable to GDAL < 2.0") @pytest.mark.parametrize("method", GDAL2_RESAMPLING) def test_warp_resampling_not_yet_supported(runner, path_rgb_byte_tif, tmpdir, method): """Resampling methods not yet supported should fail with error""" outputname = str(tmpdir.join('test.tif')) result = runner.invoke(main_group, [ 'warp', path_rgb_byte_tif, outputname, '--dst-crs', 'epsg:3857', '--resampling', method.name ]) assert result.exit_code == 2 assert "Invalid value for" in result.output assert "--resampling" in result.output
"""Tests of overview counting and creation.""" import pytest from .conftest import requires_gdal2 import rasterio from rasterio.enums import Resampling from rasterio.env import GDALVersion from rasterio.errors import OverviewCreationError gdal_version = GDALVersion() def test_count_overviews_zero(data): inputfile = str(data.join('RGB.byte.tif')) with rasterio.open(inputfile) as src: assert src.overviews(1) == [] assert src.overviews(2) == [] assert src.overviews(3) == [] def test_build_overviews_one(data): inputfile = str(data.join('RGB.byte.tif')) with rasterio.open(inputfile, 'r+') as src: overview_factors = [2] src.build_overviews(overview_factors, resampling=Resampling.nearest) assert src.overviews(1) == [2] assert src.overviews(2) == [2] assert src.overviews(3) == [2]