def test_gdal_cog_compareWeb(runner):
"""Test GDAL COG."""
with runner.isolated_filesystem():
profile = cog_profiles.get("jpeg")
profile["blockxsize"] = 256
profile["blockysize"] = 256
# rio cogeo GDAL COG
cog_translate(
raster_path_rgba,
"gdalcogeo.tif",
profile.copy(),
quiet=True,
use_cog_driver=True,
web_optimized=True,
)
# pure COG
copy(
raster_path_rgba,
"cog.tif",
driver="COG",
blocksize=256,
compress="JPEG",
TILING_SCHEME="GoogleMapsCompatible",
)
with rasterio.open("gdalcogeo.tif") as gdalcogeo, rasterio.open(
"cog.tif"
) as cog:
assert cog.meta == gdalcogeo.meta
def convert_to_cog(source_file, destination_file, resampling=rasterio.enums.Resampling.gauss, blocksize=256,
overview_blocksize=256, creation_options=None):
"""Convert source file to a Cloud Optimized GeoTiff new file.
:param source_file: path to the original raster
:param destination_file: path to the new raster
:param resampling: which Resampling to use on reading, default Resampling.gauss
:param blocksize: the size of the blocks default 256
:param overview_blocksize: the block size of the overviews, default 256
:param creation_options: <dictioanry>, options that can override the source raster profile,
notice that you can't override tiled=True, and the blocksize
"""
with rasterio.open(source_file) as src:
# creation_options overrides proile
source_profile = src.profile
creation_options = _creation_options_for_cog(creation_options, source_profile, blocksize)
with rasterio.Env(GDAL_TIFF_INTERNAL_MASK=True, GDAL_TIFF_OVR_BLOCKSIZE=overview_blocksize):
with TemporaryDirectory() as temp_dir:
temp_file = os.path.join(temp_dir, 'temp.tif')
rasterio_sh.copy(source_file, temp_file, **creation_options)
with rasterio.open(temp_file, 'r+') as dest:
factors = _calc_overviews_factors(dest)
dest.build_overviews(factors, resampling=resampling)
dest.update_tags(ns='rio_overview', resampling=resampling.name)
telluric_tags = _get_telluric_tags(source_file)
if telluric_tags:
dest.update_tags(**telluric_tags)
rasterio_sh.copy(temp_file, destination_file,
COPY_SRC_OVERVIEWS=True, **creation_options)
def compress_image(self, image_path, save_directory, save=True):
if not os.path.exists(save_directory):
os.makedirs(save_directory)
print("Compressing image ", self.IND)
self.IND += 1
meta = ""
name = self.get_file_name(image_path)
band = []
if not os.path.exists(save_directory + "/" + name):
dst_crs = rst.crs.CRS.from_epsg(
4326) # Coordinate system Hu Tzu Shan 1950
dst_width = self.max_width
dst_height = self.max_height
xres = (self.maxRight - self.minLeft) / dst_width
yres = (self.maxTop - self.minBottom) / dst_height
dst_transform = affine.Affine(xres, 0.0, self.minLeft, 0.0, -yres,
self.maxTop)
vrt_options = {
'resampling': rst.enums.Resampling.cubic,
'crs': dst_crs,
'transform': dst_transform,
'height': dst_height,
'width': dst_width
}
if (save):
raster = rst.open(image_path)
with WarpedVRT(raster, **vrt_options) as vrt:
rio_shutil.copy(vrt,
save_directory + "/" + name,
driver='GTiff')
raster.close()
return name, band, meta
def write_cog(layout, location, data, filename_template):
"""
Write a GPS tile out as a Cloud-Optimized GeoTIFF
Uses GDAL to write a COG out to disk, utilizing the given layout and location
to generate the GeoTIFF header.
Args:
layout (``gps.LayoutDefinition``): The layout for the source layer
location (``gps.SpatialKey`` or ``gps.SpaceTimeKey``): The cell identifier
in the layer corresponding to the image data
data (``gps.Tile``): The image data
filename_template (str): A pattern giving the destination for the target
image. Contains two '{}' tokens which will be ``.format``ed with the
column and row of the ``location``, in that order. May be an S3 uri or
local path.
"""
bands, w, h = data.cells.shape
nodata = data.no_data_value
dtype = data.cells.dtype
cw, ch = cell_size(layout)
ex = extent_for_cell(layout, location)
overview_level = int(log(w) / log(2) - 8)
with rstr.io.MemoryFile() as memfile:
with memfile.open(driver='GTiff',
count=bands,
width=w,
height=h,
transform=Affine(cw, 0.0, ex.xmin, 0.0, -ch,
ex.ymax),
crs=rstr.crs.CRS.from_epsg(4326),
nodata=nodata,
dtype=dtype,
compress='lzw',
tiled=True) as mem:
windows = list(mem.block_windows(1))
for _, w in windows:
segment = data.cells[:, w.row_off:(w.row_off + w.height),
w.col_off:(w.col_off + w.width)]
mem.write(segment, window=w)
mask_value = np.all(segment != nodata, axis=0).astype(
np.uint8) * 255
mem.write_mask(mask_value, window=w)
overviews = [2**j for j in range(1, overview_level + 1)]
mem.build_overviews(overviews, rwarp.Resampling.nearest)
mem.update_tags(ns='rio_oveview',
resampling=rwarp.Resampling.nearest.value)
uri = urlparse.urlparse(filename_template)
if uri.scheme == 's3':
boto3.resource('s3').Bucket(uri.netloc).upload_fileobj(
memfile,
uri.path.format(location.col, location.row)[1:])
else:
rshutil.copy(mem,
filename_template.format(location.col,
location.row),
copy_src_overviews=True)
def to_vrt(data,
filename,
resampling=None,
nodata=None,
init_dest_nodata=True,
warp_mem_limit=128):
"""
Writes a file to a VRT file
Args:
data (DataArray): The ``xarray.DataArray`` to write.
filename (str): The output file name to write to.
resampling (Optional[object]): The resampling algorithm for ``rasterio.vrt.WarpedVRT``. Default is 'nearest'.
nodata (Optional[float or int]): The 'no data' value for ``rasterio.vrt.WarpedVRT``.
init_dest_nodata (Optional[bool]): Whether or not to initialize output to ``nodata`` for ``rasterio.vrt.WarpedVRT``.
warp_mem_limit (Optional[int]): The GDAL memory limit for ``rasterio.vrt.WarpedVRT``.
Example:
>>> import geowombat as gw
>>> from rasterio.enums import Resampling
>>>
>>> with gw.config.update(ref_crs=102033):
>>>
>>> with gw.open('image.tif') as ds:
>>>
>>> gw.to_vrt(ds,
>>> 'image.vrt',
>>> resampling=Resampling.cubic,
>>> warp_mem_limit=256)
"""
if not resampling:
resampling = Resampling.nearest
# Open the input file on disk
with rio.open(data.filename) as src:
with WarpedVRT(
src,
src_crs=src.crs, # the original CRS
crs=data.crs, # the transformed CRS
src_transform=src.transform, # the original transform
transform=data.transform, # the new transform
dtype=data.gw.dtype,
resampling=resampling,
nodata=nodata,
init_dest_nodata=init_dest_nodata,
warp_mem_limit=warp_mem_limit) as vrt:
rio_shutil.copy(vrt, filename, driver='VRT')
def tileserver_optimized_raster(src, dest):
""" This method converts a raster to a tileserver optimized raster.
The method will reproject the raster to align to the xyz system, in resolution and projection
It will also create overviews
And finally it will arragne the raster in a cog way.
You could take the dest file upload it to a web server that supports ranges and user GeoRaster.get_tile
on it,
You are geranteed that you will get as minimal data as possible
"""
src_raster = tl.GeoRaster2.open(src)
bounding_box = src_raster.footprint().get_shape(
tl.constants.WGS84_CRS).bounds
tile = mercantile.bounding_tile(*bounding_box)
dest_resolution = mercator_upper_zoom_level(src_raster)
bounds = tl.GeoVector.from_xyz(tile.x, tile.y, tile.z).get_bounds(
tl.constants.WEB_MERCATOR_CRS)
create_options = {
"tiled": "YES",
"blocksize": 256,
"compress": "DEFLATE",
"photometric": "MINISBLACK"
}
with TemporaryDirectory() as temp_dir:
temp_file = os.path.join(temp_dir, 'temp.tif')
warp(src,
temp_file,
dst_crs=tl.constants.WEB_MERCATOR_CRS,
resolution=dest_resolution,
dst_bounds=bounds,
create_options=create_options)
with rasterio.Env(GDAL_TIFF_INTERNAL_MASK=True,
GDAL_TIFF_OVR_BLOCKSIZE=256):
resampling = rasterio.enums.Resampling.gauss
with rasterio.open(temp_file, 'r+') as tmp_raster:
factors = _calc_overviews_factors(tmp_raster)
tmp_raster.build_overviews(factors, resampling=resampling)
tmp_raster.update_tags(ns='rio_overview',
resampling=resampling.name)
telluric_tags = _get_telluric_tags(src)
if telluric_tags:
tmp_raster.update_tags(**telluric_tags)
rasterio_sh.copy(temp_file,
dest,
COPY_SRC_OVERVIEWS=True,
tiled=True,
compress='DEFLATE',
photometric='MINISBLACK')
def __init__(self):
if not os.path.exists("Reduced"):
os.makedirs("Reduced")
reduction_size = float(sys.argv[2])
dst_crs = rst.crs.CRS.from_epsg(4326)
for image_path in self.image_locations(sys.argv[1]):
name = self.get_file_name(image_path)
print(name)
if not os.path.exists("Reduced/" + name):
with rst.open(image_path) as src:
band = src.read(1)
meta = src.profile
top = 0
left = 0
bottom = band.shape[0]
right = band.shape[1]
print("top: ", top)
print("bottom: ", bottom)
print("left: ", left)
print("right: ", right)
dst_width = int((right - left) * reduction_size)
dst_height = int((bottom - top) * reduction_size)
print("previous width: ", band.shape[1])
print("dst_width: ", dst_width)
print("previous height: ", band.shape[0])
print("dst_height: ", dst_height)
leftCoord, topCoord = src.xy(top, left)
rightCoord, bottomCoord = src.xy(bottom, right)
xres = (rightCoord - leftCoord) / dst_width
yres = (topCoord - bottomCoord) / dst_height
dst_transform = affine.Affine(xres, 0.0, leftCoord, 0.0,
-yres, topCoord)
vrt_options = {
'resampling': rst.enums.Resampling.cubic,
'crs': dst_crs,
'transform': dst_transform,
'height': dst_height,
'width': dst_width
}
with WarpedVRT(src, **vrt_options) as vrt:
rio_shutil.copy(vrt, "Reduced/" + name, driver='GTiff')
def normalize_to_std_grid(self, inputs, resamplemethod='nearest'):
"""
Uses rasterio virtual raster to standardize grids of different crs, resolution, boundaries based on a shapefile geometry feature
:param inputs: a list of (daily) raster input files for the water balance.
:param outloc: output locations 'temp' for the virtual files
:return: list of numpy arrays
"""
outputs = []
npy_outputs = []
if resamplemethod == 'nearest':
rs = Resampling.nearest
else:
print('only nearest neighbor resampling is supported at this time')
sys.exit(0)
for i, warpfile in enumerate(inputs):
# print('warpfile', warpfile)
with rasterio.open(warpfile) as src:
# TODO - make the default configurable.
# if src.crs == None:
# src.crs = CRS.from_epsg(4326)
# create the virtual raster based on the standard rasterio attributes from the sample tiff and shapefile feature.
with WarpedVRT(src,
resampling=rs,
crs=self.crs,
transform=self.transform,
height=self.rows,
width=self.cols) as vrt:
data = vrt.read()
# print(type(vrt))
# save the file as an enumerated tiff. reopen outside this loop with the outputs list
outwarp = os.path.join(self.temp_folder,
'temp_{}.tif'.format(i))
rio_shutil.copy(vrt, outwarp, driver='GTiff')
outputs.append(outwarp)
# output each virtual file as a temporary .tif file in a temp folder somewhere in the outputs directory.
# for each file in the temp directory read in the raster as a numpy array and return the list of numpy arrays
# from this method for us in the rest of the code.
for ow in outputs:
with rasterio.open(ow, 'r') as src:
arr = src.read(1)
npy_outputs.append(arr)
return npy_outputs
def test_crs_should_be_set(path_rgb_byte_tif, tmpdir, complex):
"""When ``dst_height``, ``dst_width``, and ``dst_transform`` are set
:py:class:`rasterio.warp.WarpedVRT` calls ``GDALCreateWarpedVRT()``,
which requires the caller to then set a projection with
``GDALSetProjection()``.
Permalink to ``GDALCreateWarpedVRT()`` call:
https://github.com/mapbox/rasterio/blob/1f759e5f67628f163ea2550d8926b91545245712/rasterio/_warp.pyx#L753
"""
vrt_path = str(tmpdir.join('test_crs_should_be_set.vrt'))
with rasterio.open(path_rgb_byte_tif) as src:
dst_crs = 'EPSG:4326'
dst_height = dst_width = 10
dst_bounds = transform_bounds(src.crs, dst_crs, *src.bounds)
# Destination transform
left, bottom, right, top = dst_bounds
xres = (right - left) / dst_width
yres = (top - bottom) / dst_height
dst_transform = affine.Affine(
xres, 0.0, left, 0.0, -yres, top)
# The 'complex' test case hits the affected code path
vrt_options = {'dst_crs': dst_crs}
if complex:
vrt_options.update(
dst_crs=dst_crs,
dst_height=dst_height,
dst_width=dst_width,
dst_transform=dst_transform,
resampling=Resampling.nearest)
with WarpedVRT(src, **vrt_options) as vrt:
rio_shutil.copy(vrt, vrt_path, driver='VRT')
with rasterio.open(vrt_path) as src:
assert src.crs
def test_crs_should_be_set(path_rgb_byte_tif, tmpdir, complex):
"""When ``dst_height``, ``dst_width``, and ``dst_transform`` are set
:py:class:`rasterio.warp.WarpedVRT` calls ``GDALCreateWarpedVRT()``,
which requires the caller to then set a projection with
``GDALSetProjection()``.
Permalink to ``GDALCreateWarpedVRT()`` call:
https://github.com/mapbox/rasterio/blob/1f759e5f67628f163ea2550d8926b91545245712/rasterio/_warp.pyx#L753
"""
vrt_path = str(tmpdir.join("test_crs_should_be_set.vrt"))
with rasterio.open(path_rgb_byte_tif) as src:
dst_crs = "EPSG:4326"
dst_height = dst_width = 10
dst_bounds = transform_bounds(src.crs, dst_crs, *src.bounds)
# Destination transform
left, bottom, right, top = dst_bounds
xres = (right - left) / dst_width
yres = (top - bottom) / dst_height
dst_transform = affine.Affine(xres, 0.0, left, 0.0, -yres, top)
# The 'complex' test case hits the affected code path
vrt_options = {"dst_crs": dst_crs}
if complex:
vrt_options.update(
dst_crs=dst_crs,
dst_height=dst_height,
dst_width=dst_width,
dst_transform=dst_transform,
resampling=Resampling.nearest,
)
with WarpedVRT(src, **vrt_options) as vrt:
rio_shutil.copy(vrt, vrt_path, driver="VRT")
with rasterio.open(vrt_path) as src:
assert src.crs
def test_validate_optimized(tmpdir):
from terracotta import cog
outfile = str(tmpdir / 'raster.tif')
raster_data = 1000 * np.random.rand(512, 512).astype(np.uint16)
profile = BASE_PROFILE.copy()
profile.update(height=raster_data.shape[0],
width=raster_data.shape[1],
tiled=True,
blockxsize=256,
blockysize=256)
with MemoryFile() as mf, mf.open(**profile) as dst:
dst.write(raster_data, 1)
overviews = [2**j for j in range(1, 4)]
dst.build_overviews(overviews, Resampling.nearest)
copy(dst, outfile, copy_src_overviews=True, **profile)
assert cog.validate(outfile)
def test_gdal_cog_compare(runner):
"""Test GDAL COG."""
with runner.isolated_filesystem():
profile = cog_profiles.get("jpeg")
profile["blockxsize"] = 256
profile["blockysize"] = 256
# rio cogeo GDAL COG
cog_translate(
raster_path_rgba,
"gdalcogeo.tif",
profile.copy(),
quiet=True,
use_cog_driver=True,
)
# pure COG
copy(raster_path_rgba, "cog.tif", driver="COG", blocksize=256, compress="JPEG")
# rio cogeo cog
cog_translate(
raster_path_rgba,
"riocogeo.tif",
profile.copy(),
indexes=(
1,
2,
3,
),
add_mask=True,
quiet=True,
)
with rasterio.open("riocogeo.tif") as riocogeo, rasterio.open(
"gdalcogeo.tif"
) as gdalcogeo, rasterio.open("cog.tif") as cog:
assert cog.profile == gdalcogeo.profile == riocogeo.profile
assert cog.overviews(1) == gdalcogeo.overviews(1) == riocogeo.overviews(1)
def cog_translate(
src_path,
dst_path,
dst_kwargs,
indexes=None,
overview_level=5,
overview_resampling=None,
config=None,
):
"""
Create Cloud Optimized Geotiff.
Parameters
----------
src_path : str or PathLike object
A dataset path or URL. Will be opened in "r" mode.
dst_path : str or Path-like object
An output dataset path or or PathLike object.
Will be opened in "w" mode.
dst_kwargs: dict
output dataset creation options.
indexes : tuple, int, optional
Raster band indexes to copy.
overview_level : int, optional (default: 6)
COGEO overview (decimation) level
overview_resampling : str, [average, nearest, mode]
config : dict
Rasterio Env options.
"""
config = config or {}
nodata_mask = None
src = gdal.Open(src_path, gdal.GA_ReadOnly)
band = src.GetRasterBand(1)
nodata = band.GetNoDataValue()
# Update nodata mask only if nodata is a negative integer value
if band.DataType == gdal.GDT_Byte and nodata and nodata < 0:
nodata_mask = 255
with rasterio.Env(**config):
with rasterio.open(src_path) as src:
indexes = indexes if indexes else src.indexes
meta = src.meta
meta["count"] = len(indexes)
meta.pop("alpha", None)
meta.update(**dst_kwargs)
meta.pop("compress", None)
meta.pop("photometric", None)
if nodata_mask is not None:
meta['nodata'] = nodata
meta['dtype'] = 'int16'
meta['stats'] = True
with MemoryFile() as memfile:
with memfile.open(**meta) as mem:
wind = list(mem.block_windows(1))
for ij, w in wind:
matrix = src.read(window=w, indexes=indexes)
if nodata_mask is not None:
matrix = numpy.array(matrix, dtype='int16')
matrix[matrix == nodata_mask] = nodata
mem.write(matrix, window=w)
if overview_resampling is not None:
overviews = [
2**j for j in range(1, overview_level + 1)
]
mem.build_overviews(overviews,
Resampling[overview_resampling])
mem.update_tags(
OVR_RESAMPLING_ALG=Resampling[overview_resampling].
name.upper())
try:
copy(mem, dst_path, **dst_kwargs)
LOG.info(
f"Created a cloud optimized GeoTIFF file, {dst_path}"
)
except Exception:
LOG.exception(
f"Error while creating a cloud optimized GeoTIFF file, {dst_path}"
)
raise
def cog_translate( # noqa: C901
source: Union[str, pathlib.PurePath, DatasetReader, DatasetWriter,
WarpedVRT],
dst_path: Union[str, pathlib.PurePath],
dst_kwargs: Dict,
indexes: Optional[Sequence[int]] = None,
nodata: Optional[Union[str, int, float]] = None,
dtype: Optional[str] = None,
add_mask: bool = False,
overview_level: Optional[int] = None,
overview_resampling: str = "nearest",
web_optimized: bool = False,
tms: morecantile.TileMatrixSet = morecantile.tms.get("WebMercatorQuad"),
zoom_level_strategy: str = "auto",
aligned_levels: Optional[int] = None,
resampling: str = "nearest",
in_memory: Optional[bool] = None,
config: Optional[Dict] = None,
allow_intermediate_compression: bool = False,
forward_band_tags: bool = False,
quiet: bool = False,
temporary_compression: str = "DEFLATE",
):
"""
Create Cloud Optimized Geotiff.
Parameters
----------
source : str, PathLike object or rasterio.io.DatasetReader
A dataset path, URL or rasterio.io.DatasetReader object.
Will be opened in "r" mode.
dst_path : str or PathLike object
An output dataset path or or PathLike object.
Will be opened in "w" mode.
dst_kwargs: dict
Output dataset creation options.
indexes : tuple or int, optional
Raster band indexes to copy.
nodata, int, optional
Overwrite nodata masking values for input dataset.
dtype: str, optional
Overwrite output data type. Default will be the input data type.
add_mask, bool, optional
Force output dataset creation with a mask.
overview_level : int, optional (default: None)
COGEO overview (decimation) level. By default, inferred from data size.
overview_resampling : str, optional (default: "nearest")
Resampling algorithm for overviews
web_optimized: bool, optional (default: False)
Create web-optimized cogeo.
tms: morecantile.TileMatrixSet, optional (default: "WebMercatorQuad")
TileMatrixSet to use for reprojection, resolution and alignment.
zoom_level_strategy: str, optional (default: auto)
Strategy to determine zoom level (same as in GDAL 3.2).
LOWER will select the zoom level immediately below the theoretical computed non-integral zoom level, leading to subsampling.
On the contrary, UPPER will select the immediately above zoom level, leading to oversampling.
Defaults to AUTO which selects the closest zoom level.
ref: https://gdal.org/drivers/raster/cog.html#raster-cog
aligned_levels: int, optional.
Number of overview levels for which GeoTIFF tile and tiles defined in the tiling scheme match.
Default is to use the maximum overview levels.
resampling : str, optional (default: "nearest")
Resampling algorithm.
in_memory: bool, optional
Force processing raster in memory (default: process in memory if small)
config : dict
Rasterio Env options.
allow_intermediate_compression: bool, optional (default: False)
Allow intermediate file compression to reduce memory/disk footprint.
Note: This could reduce the speed of the process.
Ref: https://github.com/cogeotiff/rio-cogeo/issues/103
forward_band_tags: bool, optional
Forward band tags to output bands.
Ref: https://github.com/cogeotiff/rio-cogeo/issues/19
quiet: bool, optional (default: False)
Mask processing steps.
temporary_compression: str, optional
Compression used for the intermediate file, default is deflate.
"""
if isinstance(indexes, int):
indexes = (indexes, )
config = config or {}
with rasterio.Env(**config):
with ExitStack() as ctx:
if isinstance(source, (DatasetReader, DatasetWriter, WarpedVRT)):
src_dst = source
else:
src_dst = ctx.enter_context(rasterio.open(source))
meta = src_dst.meta
indexes = indexes if indexes else src_dst.indexes
nodata = nodata if nodata is not None else src_dst.nodata
dtype = dtype if dtype else src_dst.dtypes[0]
alpha = utils.has_alpha_band(src_dst)
mask = utils.has_mask_band(src_dst)
if not add_mask and (
(nodata is not None or alpha)
and dst_kwargs.get("compress") in ["JPEG", "jpeg"]):
warnings.warn(
"Using lossy compression with Nodata or Alpha band "
"can results in unwanted artefacts.",
LossyCompression,
)
tilesize = min(int(dst_kwargs["blockxsize"]),
int(dst_kwargs["blockysize"]))
if src_dst.width < tilesize or src_dst.height < tilesize:
tilesize = 2**int(
math.log(min(src_dst.width, src_dst.height), 2))
if tilesize < 64:
warnings.warn(
"Raster has dimension < 64px. Output COG cannot be tiled"
" and overviews cannot be added.",
IncompatibleBlockRasterSize,
)
dst_kwargs.pop("blockxsize", None)
dst_kwargs.pop("blockysize", None)
dst_kwargs.pop("tiled")
overview_level = 0
else:
warnings.warn(
"Block Size are bigger than raster sizes. "
"Setting blocksize to {}".format(tilesize),
IncompatibleBlockRasterSize,
)
dst_kwargs["blockxsize"] = tilesize
dst_kwargs["blockysize"] = tilesize
vrt_params = {
"add_alpha": True,
"dtype": dtype,
"width": src_dst.width,
"height": src_dst.height,
}
if nodata is not None:
vrt_params.update(
dict(nodata=nodata, add_alpha=False, src_nodata=nodata))
if alpha:
vrt_params.update(dict(add_alpha=False))
if web_optimized:
params = utils.get_web_optimized_params(
src_dst,
tilesize=tilesize,
warp_resampling=resampling,
zoom_level_strategy=zoom_level_strategy,
aligned_levels=aligned_levels,
tms=tms,
)
vrt_params.update(**params)
with WarpedVRT(src_dst, **vrt_params) as vrt_dst:
meta = vrt_dst.meta
meta["count"] = len(indexes)
if add_mask:
meta.pop("nodata", None)
meta.pop("alpha", None)
if (dst_kwargs.get("photometric", "").upper() == "YCBCR"
and meta["count"] == 1):
warnings.warn(
"PHOTOMETRIC=YCBCR not supported on a 1-band raster"
" and has been set to 'MINISBLACK'")
dst_kwargs["photometric"] = "MINISBLACK"
meta.update(**dst_kwargs)
meta.pop("compress", None)
meta.pop("photometric", None)
if allow_intermediate_compression:
meta["compress"] = temporary_compression
if in_memory is None:
in_memory = vrt_dst.width * vrt_dst.height < IN_MEMORY_THRESHOLD
if in_memory:
tmpfile = ctx.enter_context(MemoryFile())
tmp_dst = ctx.enter_context(tmpfile.open(**meta))
else:
tmpfile = ctx.enter_context(TemporaryRasterFile(dst_path))
tmp_dst = ctx.enter_context(
rasterio.open(tmpfile.name, "w", **meta))
# Transfer color interpolation
if len(indexes) == 1 and (vrt_dst.colorinterp[indexes[0] - 1]
is not ColorInterp.palette):
tmp_dst.colorinterp = [ColorInterp.gray]
else:
tmp_dst.colorinterp = [
vrt_dst.colorinterp[b - 1] for b in indexes
]
if tmp_dst.colorinterp[0] is ColorInterp.palette:
try:
tmp_dst.write_colormap(1, vrt_dst.colormap(1))
except ValueError:
warnings.warn(
"Dataset has `Palette` color interpretation"
" but is missing colormap information")
wind = list(tmp_dst.block_windows(1))
if not quiet:
click.echo("Reading input: {}".format(source), err=True)
fout = os.devnull if quiet else sys.stderr
with click.progressbar(
wind, file=fout,
show_percent=True) as windows: # type: ignore
for _, w in windows:
matrix = vrt_dst.read(window=w, indexes=indexes)
tmp_dst.write(matrix, window=w)
if add_mask or mask:
# Cast mask to uint8 to fix rasterio 1.1.2 error (ref #115)
mask_value = vrt_dst.dataset_mask(
window=w).astype("uint8")
tmp_dst.write_mask(mask_value, window=w)
if overview_level is None:
overview_level = get_maximum_overview_level(
vrt_dst.width, vrt_dst.height, minsize=tilesize)
if not quiet and overview_level:
click.echo("Adding overviews...", err=True)
overviews = [2**j for j in range(1, overview_level + 1)]
tmp_dst.build_overviews(overviews,
ResamplingEnums[overview_resampling])
if not quiet:
click.echo("Updating dataset tags...", err=True)
for i, b in enumerate(indexes):
tmp_dst.set_band_description(i + 1,
src_dst.descriptions[b - 1])
if forward_band_tags:
tmp_dst.update_tags(i + 1, **src_dst.tags(b))
tags = src_dst.tags()
tags.update(
dict(
OVR_RESAMPLING_ALG=ResamplingEnums[overview_resampling]
.name.upper()))
tmp_dst.update_tags(**tags)
tmp_dst._set_all_scales(
[vrt_dst.scales[b - 1] for b in indexes])
tmp_dst._set_all_offsets(
[vrt_dst.offsets[b - 1] for b in indexes])
if not quiet:
click.echo("Writing output to: {}".format(dst_path),
err=True)
copy(tmp_dst, dst_path, copy_src_overviews=True, **dst_kwargs)
def optimize_rasters(raster_files: Sequence[Sequence[Path]],
output_folder: Path,
overwrite: bool = False,
resampling_method: str = 'average',
reproject: bool = False,
in_memory: bool = None,
compression: str = 'auto',
quiet: bool = False) -> None:
"""Optimize a collection of raster files for use with Terracotta.
First argument is a list of input files or glob patterns.
Example:
$ terracotta optimize-rasters rasters/*.tif -o cloud-optimized/
Note that all rasters may only contain a single band.
"""
raster_files_flat = sorted(set(itertools.chain.from_iterable(raster_files)))
if not raster_files_flat:
click.echo('No files given')
return
rs_method = RESAMPLING_METHODS[resampling_method]
if compression == 'auto':
compression = _prefered_compression_method()
total_pixels = 0
for f in raster_files_flat:
if not f.is_file():
raise click.BadParameter(f'Input raster {f!s} is not a file')
with rasterio.open(str(f), 'r') as src:
if src.count > 1 and not quiet:
click.echo(
f'Warning: raster file {f!s} has more than one band. '
'Only the first one will be used.', err=True
)
total_pixels += src.height * src.width
output_folder.mkdir(exist_ok=True)
if not quiet:
# insert newline for nicer progress bar style
click.echo('')
sub_pbar_args = dict(
disable=quiet,
leave=False,
bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt}'
)
with contextlib.ExitStack() as outer_env:
pbar = outer_env.enter_context(tqdm.tqdm(
total=total_pixels, smoothing=0, disable=quiet,
bar_format='{l_bar}{bar}| [{elapsed}<{remaining}{postfix}]',
desc='Optimizing rasters'
))
outer_env.enter_context(rasterio.Env(**GDAL_CONFIG))
for input_file in raster_files_flat:
if len(input_file.name) > 30:
short_name = input_file.name[:13] + '...' + input_file.name[-13:]
else:
short_name = input_file.name
pbar.set_postfix(file=short_name)
output_file = output_folder / input_file.with_suffix('.tif').name
if not overwrite and output_file.is_file():
raise click.BadParameter(
f'Output file {output_file!s} exists (use --overwrite to ignore)'
)
with contextlib.ExitStack() as es, warnings.catch_warnings():
warnings.filterwarnings('ignore', message='invalid value encountered.*')
src = es.enter_context(rasterio.open(str(input_file)))
if reproject:
vrt = es.enter_context(_get_vrt(src, rs_method=rs_method))
else:
vrt = src
profile = vrt.profile.copy()
profile.update(COG_PROFILE)
if in_memory is None:
in_memory = vrt.width * vrt.height < IN_MEMORY_THRESHOLD
if in_memory:
memfile = es.enter_context(MemoryFile())
dst = es.enter_context(memfile.open(**profile))
else:
tempraster = es.enter_context(TemporaryRasterFile(basedir=output_folder))
dst = es.enter_context(rasterio.open(tempraster, 'w', **profile))
# iterate over blocks
windows = list(dst.block_windows(1))
for _, w in tqdm.tqdm(windows, desc='Reading', **sub_pbar_args):
block_data = vrt.read(window=w, indexes=[1])
dst.write(block_data, window=w)
block_mask = vrt.dataset_mask(window=w).astype('uint8')
dst.write_mask(block_mask, window=w)
# add overviews
if not in_memory:
# work around bug mapbox/rasterio#1497
dst.close()
dst = es.enter_context(rasterio.open(tempraster, 'r+'))
max_overview_level = math.ceil(math.log2(max(
dst.height // profile['blockysize'],
dst.width // profile['blockxsize']
)))
overviews = [2 ** j for j in range(1, max_overview_level + 1)]
with tqdm.tqdm(desc='Creating overviews', total=1, **sub_pbar_args):
dst.build_overviews(overviews, rs_method)
dst.update_tags(ns='rio_overview', resampling=rs_method.value)
# copy to destination (this is necessary to push overviews to start of file)
with tqdm.tqdm(desc='Compressing', total=1, **sub_pbar_args):
copy(
dst, str(output_file), copy_src_overviews=True,
compress=compression, **COG_PROFILE
)
pbar.update(dst.height * dst.width)
def cog_translate(
src_path,
dst_path,
dst_kwargs,
indexes=None,
nodata=None,
alpha=None,
overview_level=6,
overview_resampling="nearest",
config=None,
):
"""
Create Cloud Optimized Geotiff.
Parameters
----------
src_path : str or PathLike object
A dataset path or URL. Will be opened in "r" mode.
dst_path : str or Path-like object
An output dataset path or or PathLike object.
Will be opened in "w" mode.
dst_kwargs: dict
output dataset creation options.
indexes : tuple, int, optional
Raster band indexes to copy.
nodata, int, optional
nodata value for mask creation.
alpha, int, optional
alpha band index for mask creation.
overview_level : int, optional (default: 6)
COGEO overview (decimation) level
config : dict
Rasterio Env options.
"""
config = config or {}
with rasterio.Env(**config):
with rasterio.open(src_path) as src:
indexes = indexes if indexes else src.indexes
meta = src.meta
meta["count"] = len(indexes)
meta.pop("nodata", None)
meta.pop("alpha", None)
meta.update(**dst_kwargs)
meta.pop("compress", None)
meta.pop("photometric", None)
with MemoryFile() as memfile:
with memfile.open(**meta) as mem:
wind = list(mem.block_windows(1))
with click.progressbar(wind,
length=len(wind),
file=sys.stderr,
show_percent=True) as windows:
for ij, w in windows:
matrix = src.read(window=w, indexes=indexes)
mem.write(matrix, window=w)
if nodata is not None:
mask_value = (
numpy.all(matrix != nodata, axis=0).astype(
numpy.uint8) * 255)
elif alpha is not None:
mask_value = src.read(alpha, window=w)
else:
mask_value = src.dataset_mask(window=w)
mem.write_mask(mask_value, window=w)
overviews = [2**j for j in range(1, overview_level + 1)]
mem.build_overviews(overviews,
Resampling[overview_resampling])
mem.update_tags(
ns="rio_overview",
resampling=Resampling[overview_resampling].value,
)
copy(mem, dst_path, copy_src_overviews=True, **dst_kwargs)
def cog_translate(
src_path,
dst_path,
dst_kwargs,
indexes=None,
nodata=None,
add_mask=None,
overview_level=None,
overview_resampling="nearest",
web_optimized=False,
latitude_adjustment=True,
resampling="nearest",
in_memory=None,
config=None,
quiet=False,
):
"""
Create Cloud Optimized Geotiff.
Parameters
----------
src_path : str or PathLike object
A dataset path or URL. Will be opened in "r" mode.
dst_path : str or Path-like object
An output dataset path or or PathLike object.
Will be opened in "w" mode.
dst_kwargs: dict
Output dataset creation options.
indexes : tuple, int, optional
Raster band indexes to copy.
nodata, int, optional
Overwrite nodata masking values for input dataset.
add_mask, bool, optional
Force output dataset creation with a mask.
overview_level : int, optional (default: 6)
COGEO overview (decimation) level
overview_resampling : str, optional (default: "nearest")
Resampling algorithm for overviews
web_optimized: bool, option (default: False)
Create web-optimized cogeo.
latitude_adjustment: bool, option (default: True)
Use mercator meters for zoom calculation or ensure max zoom equality.
resampling : str, optional (default: "nearest")
Resampling algorithm.
in_memory: bool, optional
Force processing raster in memory (default: process in memory if small)
config : dict
Rasterio Env options.
quiet: bool, optional (default: False)
Mask processing steps.
"""
config = config or {}
with rasterio.Env(**config):
with rasterio.open(src_path) as src_dst:
meta = src_dst.meta
indexes = indexes if indexes else src_dst.indexes
nodata = nodata if nodata is not None else src_dst.nodata
alpha = has_alpha_band(src_dst)
mask = has_mask_band(src_dst)
if not add_mask and (
(nodata is not None or alpha)
and dst_kwargs.get("compress") in ["JPEG", "jpeg"]):
warnings.warn(
"Using lossy compression with Nodata or Alpha band "
"can results in unwanted artefacts.",
LossyCompression,
)
tilesize = min(int(dst_kwargs["blockxsize"]),
int(dst_kwargs["blockysize"]))
if src_dst.width < tilesize or src_dst.height < tilesize:
tilesize = 2**int(
math.log(min(src_dst.width, src_dst.height), 2))
if tilesize < 64:
warnings.warn(
"Raster has dimension < 64px. Output COG cannot be tiled"
" and overviews cannot be added.",
IncompatibleBlockRasterSize,
)
dst_kwargs.pop("blockxsize", None)
dst_kwargs.pop("blockysize", None)
dst_kwargs.pop("tiled")
overview_level = 0
else:
warnings.warn(
"Block Size are bigger than raster sizes. "
"Setting blocksize to {}".format(tilesize),
IncompatibleBlockRasterSize,
)
dst_kwargs["blockxsize"] = tilesize
dst_kwargs["blockysize"] = tilesize
vrt_params = dict(add_alpha=True)
if nodata is not None:
vrt_params.update(
dict(nodata=nodata, add_alpha=False, src_nodata=nodata))
if alpha:
vrt_params.update(dict(add_alpha=False))
if web_optimized:
bounds = list(
transform_bounds(*[src_dst.crs, "epsg:4326"] +
list(src_dst.bounds),
densify_pts=21))
center = [(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2]
lat = 0 if latitude_adjustment else center[1]
max_zoom = get_max_zoom(src_dst, lat=lat, tilesize=tilesize)
extrema = tile_extrema(bounds, max_zoom)
w, n = mercantile.xy(*mercantile.ul(
extrema["x"]["min"], extrema["y"]["min"], max_zoom))
vrt_res = _meters_per_pixel(max_zoom, 0, tilesize=tilesize)
vrt_transform = Affine(vrt_res, 0, w, 0, -vrt_res, n)
vrt_width = (extrema["x"]["max"] -
extrema["x"]["min"]) * tilesize
vrt_height = (extrema["y"]["max"] -
extrema["y"]["min"]) * tilesize
vrt_params.update(
dict(
crs="epsg:3857",
transform=vrt_transform,
width=vrt_width,
height=vrt_height,
resampling=ResamplingEnums[resampling],
))
with WarpedVRT(src_dst, **vrt_params) as vrt_dst:
meta = vrt_dst.meta
meta["count"] = len(indexes)
if add_mask:
meta.pop("nodata", None)
meta.pop("alpha", None)
meta.update(**dst_kwargs)
meta.pop("compress", None)
meta.pop("photometric", None)
if in_memory is None:
in_memory = vrt_dst.width * vrt_dst.height < IN_MEMORY_THRESHOLD
with ExitStack() as ctx:
if in_memory:
tmpfile = ctx.enter_context(MemoryFile())
tmp_dst = ctx.enter_context(tmpfile.open(**meta))
else:
tmpfile = ctx.enter_context(
TemporaryRasterFile(dst_path))
tmp_dst = ctx.enter_context(
rasterio.open(tmpfile.name, "w", **meta))
wind = list(tmp_dst.block_windows(1))
if not quiet:
click.echo("Reading input: {}".format(src_path),
err=True)
fout = os.devnull if quiet else sys.stderr
with click.progressbar(wind,
length=len(wind),
file=fout,
show_percent=True) as windows:
for ij, w in windows:
matrix = vrt_dst.read(window=w, indexes=indexes)
tmp_dst.write(matrix, window=w)
if add_mask or mask:
mask_value = vrt_dst.dataset_mask(window=w)
tmp_dst.write_mask(mask_value, window=w)
if overview_level is None:
overview_level = get_maximum_overview_level(
vrt_dst, tilesize)
if not quiet and overview_level:
click.echo("Adding overviews...", err=True)
overviews = [2**j for j in range(1, overview_level + 1)]
tmp_dst.build_overviews(
overviews, ResamplingEnums[overview_resampling])
if not quiet:
click.echo("Updating dataset tags...", err=True)
for i, b in enumerate(indexes):
tmp_dst.set_band_description(
i + 1, src_dst.descriptions[b - 1])
tags = src_dst.tags()
tags.update(
dict(OVR_RESAMPLING_ALG=ResamplingEnums[
overview_resampling].name.upper()))
tmp_dst.update_tags(**tags)
if not quiet:
click.echo("Writing output to: {}".format(dst_path),
err=True)
copy(tmp_dst,
dst_path,
copy_src_overviews=True,
**dst_kwargs)
def region_of_interest(self, image_path, band_num=4):
dst_crs = rst.crs.CRS.from_epsg(4326)
if not os.path.exists("IntRegImgs"):
os.makedirs("IntRegImgs")
name = self.get_file_name(image_path)
if not os.path.exists("IntRegImgs/" + name):
try:
with rst.open(image_path) as src:
# Write bands
band = src.read(band_num)
meta = src.profile
meta['count'] = 1
meta['dtype'] = 'uint8'
max_val = np.max(band)
band = band / max_val
band = band * 255
band = band.astype(np.uint8)
if not os.path.exists("Band4"):
os.makedirs("Band4")
raster = []
if not os.path.exists("Band4/" + name):
raster = rst.open("Band4" + '/' + name, 'w', **meta)
raster.write(band, 1)
no_black_pix = np.where(band != 0)
top = np.amin(no_black_pix[0])
left = np.amin(no_black_pix[1])
bottom = np.amax(no_black_pix[0])
right = np.amax(no_black_pix[1])
dst_width = right - left
dst_height = bottom - top
leftCoord, topCoord = src.xy(top, left)
rightCoord, bottomCoord = src.xy(bottom, right)
if self.index == 0:
self.minBottom = bottomCoord
self.minLeft = leftCoord
self.maxTop = topCoord
self.maxRight = rightCoord
self.max_width = dst_width
self.max_height = dst_height
self.index += 1
else:
if bottomCoord < self.minBottom:
self.minBottom = bottomCoord
if leftCoord < self.minLeft:
self.minLeft = leftCoord
if rightCoord > self.maxRight:
self.maxRight = rightCoord
if topCoord > self.maxTop:
self.maxTop = topCoord
if dst_width > self.max_width:
self.max_width = dst_width
if dst_height > self.max_height:
self.max_height = dst_height
xres = meta['transform'][0]
yres = -meta['transform'][4]
dst_transform = affine.Affine(xres, 0.0, leftCoord,
0.0, -yres, topCoord)
vrt_options = {
'resampling': rst.enums.Resampling.cubic,
'crs': dst_crs,
'transform': dst_transform,
'height': dst_height,
'width': dst_width
}
with WarpedVRT(raster, **vrt_options) as vrt:
rio_shutil.copy(vrt,
"IntRegImgs/" + name,
driver='GTiff')
raster.close()
else:
raster = rst.open("Band4/" + name)
test_band = raster.read(1)
no_black_pix = np.where(band != 0)
top = np.amin(no_black_pix[0])
left = np.amin(no_black_pix[1])
bottom = np.amax(no_black_pix[0])
right = np.amax(no_black_pix[1])
dst_width = right - left
dst_height = bottom - top
leftCoord, topCoord = src.xy(top, left, offset='ul')
rightCoord, bottomCoord = src.xy(bottom,
right,
offset='lr')
if self.index == 0:
self.minBottom = bottomCoord
self.minLeft = leftCoord
self.maxTop = topCoord
self.maxRight = rightCoord
self.max_width = dst_width
self.max_height = dst_height
self.index += 1
else:
if bottomCoord < self.minBottom:
self.minBottom = bottomCoord
if leftCoord < self.minLeft:
self.minLeft = leftCoord
if rightCoord > self.maxRight:
self.maxRight = rightCoord
if topCoord > self.maxTop:
self.maxTop = topCoord
if dst_width > self.max_width:
self.max_width = dst_width
if dst_height > self.max_height:
self.max_height = dst_height
xres = meta['transform'][0]
yres = -meta['transform'][4]
dst_transform = affine.Affine(xres, 0.0, leftCoord,
0.0, -yres, topCoord)
vrt_options = {
'resampling': rst.enums.Resampling.cubic,
'crs': dst_crs,
'transform': dst_transform,
'height': dst_height,
'width': dst_width
}
with WarpedVRT(raster, **vrt_options) as vrt:
rio_shutil.copy(vrt,
"IntRegImgs/" + name,
driver='GTiff')
raster.close()
except:
print("Could not open the file: ", name)
else:
raster = rst.open("IntRegImgs/" + name)
test_band = raster.read(1)
leftCoord, topCoord = raster.xy(0, 0)
rightCoord, bottomCoord = raster.xy(test_band.shape[0] - 1,
test_band.shape[1] - 1)
if self.index == 0:
self.minBottom = bottomCoord
self.minLeft = leftCoord
self.maxTop = topCoord
self.maxRight = rightCoord
self.max_width = dst_width
self.max_height = dst_height
self.index += 1
else:
if bottomCoord < self.minBottom:
self.minBottom = bottomCoord
if leftCoord < self.minLeft:
self.minLeft = leftCoord
if rightCoord > self.maxRight:
self.maxRight = rightCoord
if topCoord > self.maxTop:
self.maxTop = topCoord
if dst_width > self.max_width:
self.max_width = dst_width
if dst_height > self.max_height:
self.max_height = dst_height
def create_overview_cogs(
mosaic_path: str,
output_profile: Dict,
prefix: str = "mosaic_ovr",
max_overview_level: int = 6,
method: str = "first",
config: Dict = None,
threads=1,
in_memory: bool = True,
) -> None:
"""
Create Low resolution mosaic image from a mosaicJSON.
The output will be a web optimized COG with bounds matching the mosaicJSON bounds and
with its resolution matching the mosaic MinZoom - 1.
Attributes
----------
mosaic_path : str, required
Mosaic definition path.
output_profile : dict, required
prefix : str
max_overview_level : int
method: str, optional
pixel_selection method name (default is 'first').
config : dict
Rasterio Env options.
threads: int, optional
maximum number of threads to use (default is 1).
in_memory: bool, optional
Force COG creation in memory (default is True).
"""
pixel_method = PIXSEL_METHODS[method]
with MosaicBackend(mosaic_path) as mosaic:
base_zoom = mosaic.metadata["minzoom"] - 1
mosaic_quadkey_zoom = mosaic.quadkey_zoom
bounds = mosaic.metadata["bounds"]
mosaic_quadkeys = set(mosaic._quadkeys)
# Select a random quakey/asset and get dataset info
tile = mercantile.quadkey_to_tile(random.sample(mosaic_quadkeys, 1)[0])
assets = mosaic.assets_for_tile(*tile)
info = _get_info(assets[0])
extrema = tile_extrema(bounds, base_zoom)
tilesize = 256
resolution = _meters_per_pixel(base_zoom, 0, tilesize=tilesize)
# Create multiples files if coverage is too big
extremas = _split_extrema(extrema, max_ovr=max_overview_level)
for ix, extrema in enumerate(extremas):
click.echo(f"Part {1 + ix}/{len(extremas)}", err=True)
output_path = f"{prefix}_{ix}.tif"
blocks = list(_get_blocks(extrema, tilesize))
random.shuffle(blocks)
width = (extrema["x"]["max"] - extrema["x"]["min"]) * tilesize
height = (extrema["y"]["max"] - extrema["y"]["min"]) * tilesize
w, n = mercantile.xy(*mercantile.ul(
extrema["x"]["min"], extrema["y"]["min"], base_zoom))
params = dict(
driver="GTiff",
dtype=info["dtype"],
count=len(info["band_descriptions"]),
width=width,
height=height,
crs="epsg:3857",
transform=Affine(resolution, 0, w, 0, -resolution, n),
nodata=info["nodata_value"],
)
params.update(**output_profile)
config = config or {}
with rasterio.Env(**config):
with ExitStack() as ctx:
if in_memory:
tmpfile = ctx.enter_context(MemoryFile())
tmp_dst = ctx.enter_context(tmpfile.open(**params))
else:
tmpfile = ctx.enter_context(
TemporaryRasterFile(output_path))
tmp_dst = ctx.enter_context(
rasterio.open(tmpfile.name, "w", **params))
def _get_tile(wind):
idx, window = wind
x = extrema["x"]["min"] + idx[1]
y = extrema["y"]["min"] + idx[0]
t = mercantile.Tile(x, y, base_zoom)
kds = set(find_quadkeys(t, mosaic_quadkey_zoom))
if not mosaic_quadkeys.intersection(kds):
return window, None, None
try:
(tile, mask), _ = mosaic.tile(
t.x,
t.y,
t.z,
tilesize=tilesize,
pixel_selection=pixel_method(),
)
except NoAssetFoundError:
return window, None, None
return window, tile, mask
with futures.ThreadPoolExecutor(
max_workers=threads) as executor:
future_work = [
executor.submit(_get_tile, item) for item in blocks
]
with click.progressbar(
futures.as_completed(future_work),
length=len(future_work),
show_percent=True,
label="Loading tiles",
) as future:
for res in future:
pass
for f in _filter_futures(future_work):
window, tile, mask = f
if tile is None:
continue
tmp_dst.write(tile, window=window)
if info["nodata_type"] == "Mask":
tmp_dst.write_mask(mask.astype("uint8"),
window=window)
min_tile_size = tilesize = min(
int(output_profile["blockxsize"]),
int(output_profile["blockysize"]),
)
overview_level = get_maximum_overview_level(
tmp_dst.width, tmp_dst.height, minsize=min_tile_size)
overviews = [2**j for j in range(1, overview_level + 1)]
tmp_dst.build_overviews(overviews)
copy(tmp_dst,
output_path,
copy_src_overviews=True,
**params)
def close(self, exc_type=None, exc_value=None, exc_traceback=None):
"""Build overviews and write file."""
try:
# only in case no Exception was raised
if not exc_type:
# build overviews
if self.overviews and self.dst is not None:
logger.debug(
"build overviews using %s resampling and levels %s",
self.overviews_resampling, self.overviews_levels)
self.dst.build_overviews(
self.overviews_levels,
Resampling[self.overviews_resampling])
self.dst.update_tags(ns='rio_overview',
resampling=self.overviews_resampling)
# write
if self.cog:
if path_is_remote(self.path):
# remote COG: copy to tempfile and upload to destination
logger.debug("upload to %s", self.path)
# TODO this writes a memoryfile to disk and uploads the file,
# this is inefficient but until we find a solution to copy
# from one memoryfile to another the rasterio way (rasterio needs
# to rearrange the data so the overviews are at the beginning of
# the GTiff in order to be a valid COG).
with NamedTemporaryFile() as tmp_dst:
copy(self.dst,
tmp_dst.name,
copy_src_overviews=True,
**self._profile)
self._bucket_resource.upload_file(
Filename=tmp_dst.name,
Key="/".join(self.path.split("/")[3:]),
)
else:
# local COG: copy to destination
logger.debug("write to %s", self.path)
copy(self.dst,
self.path,
copy_src_overviews=True,
**self._profile)
else:
if path_is_remote(self.path):
# remote GTiff: upload memfile or tempfile to destination
logger.debug("upload to %s", self.path)
if self.in_memory:
self._bucket_resource.put_object(
Body=self._memfile,
Key="/".join(self.path.split("/")[3:]),
)
else:
self._bucket_resource.upload_file(
Filename=self._tempfile.name,
Key="/".join(self.path.split("/")[3:]),
)
else:
# local GTiff: already written, do nothing
pass
finally:
self._ctx.close()
def cog_translate(
src_path,
dst_path,
dst_kwargs,
indexes=None,
nodata=None,
add_mask=None,
overview_level=None,
overview_resampling="nearest",
config=None,
quiet=False,
):
"""
Create Cloud Optimized Geotiff.
Parameters
----------
src_path : str or PathLike object
A dataset path or URL. Will be opened in "r" mode.
dst_path : str or Path-like object
An output dataset path or or PathLike object.
Will be opened in "w" mode.
dst_kwargs: dict
output dataset creation options.
indexes : tuple, int, optional
Raster band indexes to copy.
nodata, int, optional
Overwrite nodata masking values for input dataset.
add_mask, bool, optional
Force output dataset creation with a mask.
overview_level : int, optional (default: 6)
COGEO overview (decimation) level
overview_resampling : str, optional (default: "nearest")
Resampling algorithm for overviews
config : dict
Rasterio Env options.
quiet: bool, optional (default: False)
Mask processing steps.
"""
config = config or {}
if overview_level is None:
overview_level = get_maximum_overview_level(
src_path,
min(int(dst_kwargs["blockxsize"]), int(dst_kwargs["blockysize"])))
with rasterio.Env(**config):
with rasterio.open(src_path) as src_dst:
meta = src_dst.meta
indexes = indexes if indexes else src_dst.indexes
nodata = nodata if nodata is not None else src_dst.nodata
alpha = has_alpha_band(src_dst)
mask = has_mask_band(src_dst)
if not add_mask and (
(nodata is not None or alpha)
and dst_kwargs.get("compress") in ["JPEG", "jpeg"]):
warnings.warn(
"Using lossy compression with Nodata or Alpha band "
"can results in unwanted artefacts.",
LossyCompression,
)
vrt_params = dict(add_alpha=True)
if nodata is not None:
vrt_params.update(
dict(nodata=nodata, add_alpha=False, src_nodata=nodata))
if alpha:
vrt_params.update(dict(add_alpha=False))
with WarpedVRT(src_dst, **vrt_params) as vrt_dst:
meta = vrt_dst.meta
meta["count"] = len(indexes)
if add_mask:
meta.pop("nodata", None)
meta.pop("alpha", None)
meta.update(**dst_kwargs)
meta.pop("compress", None)
meta.pop("photometric", None)
with MemoryFile() as memfile:
with memfile.open(**meta) as mem:
wind = list(mem.block_windows(1))
if not quiet:
click.echo("Reading input: {}".format(src_path),
err=True)
fout = os.devnull if quiet else sys.stderr
with click.progressbar(wind,
length=len(wind),
file=fout,
show_percent=True) as windows:
for ij, w in windows:
matrix = vrt_dst.read(window=w,
indexes=indexes)
mem.write(matrix, window=w)
if add_mask or mask:
mask_value = vrt_dst.dataset_mask(window=w)
mem.write_mask(mask_value, window=w)
if not quiet:
click.echo("Adding overviews...", err=True)
overviews = [
2**j for j in range(1, overview_level + 1)
]
mem.build_overviews(overviews,
Resampling[overview_resampling])
if not quiet:
click.echo("Updating dataset tags...", err=True)
for i, b in enumerate(indexes):
mem.set_band_description(
i + 1, src_dst.descriptions[b - 1])
tags = src_dst.tags()
tags.update(
dict(OVR_RESAMPLING_ALG=Resampling[
overview_resampling].name.upper()))
mem.update_tags(**tags)
if not quiet:
click.echo(
"Writing output to: {}".format(dst_path),
err=True)
copy(mem,
dst_path,
copy_src_overviews=True,
**dst_kwargs)
def save_file(self, path=None):
if path is None:
path = self.path
shutil.copy(self.writable_file, path, driver='GTiff')
return True
def save_file_as_georeferenced(self, path=None):
if path is None:
path = self.path
shutil.copy(self.file, path, driver='GTiff')
return True
def _optimize_single_raster(input_file: Path, output_folder: Path,
reproject: bool, rs_method: Any,
in_memory: Union[bool, None], compression: str,
quiet: bool, progress_suffix: str) -> None:
output_file = _output_file(output_folder, input_file)
if not quiet:
click.echo(f'\r{input_file.name} ... {progress_suffix}')
with contextlib.ExitStack() as es, warnings.catch_warnings():
warnings.filterwarnings('ignore',
message='invalid value encountered.*')
src = es.enter_context(rasterio.open(str(input_file)))
if reproject:
vrt = es.enter_context(_get_vrt(src, rs_method=rs_method))
else:
vrt = src
profile = vrt.profile.copy()
profile.update(COG_PROFILE)
if in_memory is None:
in_memory = vrt.width * vrt.height < IN_MEMORY_THRESHOLD
if in_memory:
memfile = es.enter_context(MemoryFile())
dst = es.enter_context(memfile.open(**profile))
else:
tempraster = es.enter_context(
TemporaryRasterFile(basedir=output_folder))
dst = es.enter_context(rasterio.open(tempraster, 'w', **profile))
# iterate over blocks
windows = list(dst.block_windows(1))
for _, w in windows:
block_data = vrt.read(window=w, indexes=[1])
dst.write(block_data, window=w)
block_mask = vrt.dataset_mask(window=w).astype('uint8')
dst.write_mask(block_mask, window=w)
# add overviews
if not in_memory:
# work around bug mapbox/rasterio#1497
dst.close()
dst = es.enter_context(rasterio.open(tempraster, 'r+'))
max_overview_level = math.ceil(
math.log2(
max(dst.height // profile['blockysize'],
dst.width // profile['blockxsize'], 1)))
if max_overview_level > 0:
overviews = [2**j for j in range(1, max_overview_level + 1)]
dst.build_overviews(overviews, rs_method)
dst.update_tags(ns='rio_overview', resampling=rs_method.value)
# copy to destination (this is necessary to push overviews to start of file)
copy(dst,
str(output_file),
copy_src_overviews=True,
compress=compression,
**COG_PROFILE)
def to_vrt(data,
filename,
overwrite=False,
resampling=None,
nodata=None,
init_dest_nodata=True,
warp_mem_limit=128):
"""
Writes a file to a VRT file
Args:
data (DataArray): The ``xarray.DataArray`` to write.
filename (str): The output file name to write to.
overwrite (Optional[bool]): Whether to overwrite an existing VRT file.
resampling (Optional[object]): The resampling algorithm for ``rasterio.vrt.WarpedVRT``. Default is 'nearest'.
nodata (Optional[float or int]): The 'no data' value for ``rasterio.vrt.WarpedVRT``.
init_dest_nodata (Optional[bool]): Whether or not to initialize output to ``nodata`` for ``rasterio.vrt.WarpedVRT``.
warp_mem_limit (Optional[int]): The GDAL memory limit for ``rasterio.vrt.WarpedVRT``.
Example:
>>> import geowombat as gw
>>> from rasterio.enums import Resampling
>>>
>>> # Transform a CRS and save to VRT
>>> with gw.config.update(ref_crs=102033):
>>> with gw.open('image.tif') as src:
>>> gw.to_vrt(src,
>>> 'output.vrt',
>>> resampling=Resampling.cubic,
>>> warp_mem_limit=256)
>>>
>>> # Load multiple files set to a common geographic extent
>>> bounds = (left, bottom, right, top)
>>> with gw.config.update(ref_bounds=bounds):
>>> with gw.open(['image1.tif', 'image2.tif'], mosaic=True) as src:
>>> gw.to_vrt(src, 'output.vrt')
"""
if Path(filename).is_file():
if overwrite:
Path(filename).unlink()
else:
logger.warning(f' The VRT file {filename} already exists.')
return
if not resampling:
resampling = Resampling.nearest
if isinstance(data.attrs['filename'], str) or isinstance(data.attrs['filename'], Path):
# Open the input file on disk
with rio.open(data.attrs['filename']) as src:
with WarpedVRT(src,
src_crs=src.crs, # the original CRS
crs=data.crs, # the transformed CRS
src_transform=src.gw.transform, # the original transform
transform=data.gw.transform, # the new transform
dtype=data.dtype,
resampling=resampling,
nodata=nodata,
init_dest_nodata=init_dest_nodata,
warp_mem_limit=warp_mem_limit) as vrt:
rio_shutil.copy(vrt, filename, driver='VRT')
else:
if not data.gw.filenames:
logger.exception(' The data filenames attribute is empty. Use gw.open(..., persist_filenames=True).')
raise KeyError
separate = True if data.gw.data_are_separate and data.gw.data_are_stacked else False
vrt_options = gdal.BuildVRTOptions(outputBounds=data.gw.bounds,
xRes=data.gw.cellx,
yRes=data.gw.celly,
separate=separate,
outputSRS=data.crs)
ds = gdal.BuildVRT(filename, data.gw.filenames, options=vrt_options)
ds = None
fname2 = sys.argv[2] fname3 = sys.argv[3] #Open up our data files: a = rio.open(fname1, mode='r', driver='ISIS3') a1 = a.read()[0] b = rio.open(fname2, mode='r', driver='ISIS3') b1 = b.read()[0] b.close #keeping memory footprint as small as possible by deallocating the import #objects and running garbage collection del b #Create dummy cube to hold merged data shutil.copy(a, fname3, driver='ISIS3') a.close del a gc.collect() #Mask arrays so we don't run into overflow errors with null values a1 = np.ma.masked_where(a1 < 0, a1) b1 = np.ma.masked_where(b1 < 0, b1) #Trying to process the entire array at once tends to crash memory #so we're going to take a divide and conquer approach by processing #the array in six segments. array_subdivs_x = np.linspace(0, a1.shape[1], num=8, dtype=int) array_subdivs_y = np.linspace(0, a1.shape[0], num=4, dtype=int)
def cog_translate(
source,
dst_path,
dst_kwargs,
indexes=None,
nodata=None,
dtype=None,
add_mask=None,
overview_level=None,
overview_resampling="nearest",
web_optimized=False,
latitude_adjustment=True,
resampling="nearest",
in_memory=None,
config=None,
allow_intermediate_compression=False,
forward_band_tags=False,
quiet=False,
):
"""
Create Cloud Optimized Geotiff.
Parameters
----------
source : str, PathLike object or rasterio.io.DatasetReader
A dataset path, URL or rasterio.io.DatasetReader object.
Will be opened in "r" mode.
dst_path : str or Path-like object
An output dataset path or or PathLike object.
Will be opened in "w" mode.
dst_kwargs: dict
Output dataset creation options.
indexes : tuple or int, optional
Raster band indexes to copy.
nodata, int, optional
Overwrite nodata masking values for input dataset.
dtype: str, optional
Overwrite output data type. Default will be the input data type.
add_mask, bool, optional
Force output dataset creation with a mask.
overview_level : int, optional (default: 6)
COGEO overview (decimation) level
overview_resampling : str, optional (default: "nearest")
Resampling algorithm for overviews
web_optimized: bool, option (default: False)
Create web-optimized cogeo.
latitude_adjustment: bool, option (default: True)
Use mercator meters for zoom calculation or ensure max zoom equality.
resampling : str, optional (default: "nearest")
Resampling algorithm.
in_memory: bool, optional
Force processing raster in memory (default: process in memory if small)
config : dict
Rasterio Env options.
allow_intermediate_compression: bool, optional (default: False)
Allow intermediate file compression to reduce memory/disk footprint.
Note: This could reduce the speed of the process.
Ref: https://github.com/cogeotiff/rio-cogeo/issues/103
forward_band_tags: bool, optional
Forward band tags to output bands.
Ref: https://github.com/cogeotiff/rio-cogeo/issues/19
quiet: bool, optional (default: False)
Mask processing steps.
"""
if isinstance(indexes, int):
indexes = (indexes, )
config = config or {}
with rasterio.Env(**config):
with ExitStack() as ctx:
if isinstance(source, (DatasetReader, DatasetWriter, WarpedVRT)):
src_dst = source
else:
src_dst = ctx.enter_context(rasterio.open(source))
meta = src_dst.meta
indexes = indexes if indexes else src_dst.indexes
nodata = nodata if nodata is not None else src_dst.nodata
dtype = dtype if dtype else src_dst.dtypes[0]
alpha = has_alpha_band(src_dst)
mask = has_mask_band(src_dst)
if not add_mask and (
(nodata is not None or alpha)
and dst_kwargs.get("compress") in ["JPEG", "jpeg"]):
warnings.warn(
"Using lossy compression with Nodata or Alpha band "
"can results in unwanted artefacts.",
LossyCompression,
)
tilesize = min(int(dst_kwargs["blockxsize"]),
int(dst_kwargs["blockysize"]))
if src_dst.width < tilesize or src_dst.height < tilesize:
tilesize = 2**int(
math.log(min(src_dst.width, src_dst.height), 2))
if tilesize < 64:
warnings.warn(
"Raster has dimension < 64px. Output COG cannot be tiled"
" and overviews cannot be added.",
IncompatibleBlockRasterSize,
)
dst_kwargs.pop("blockxsize", None)
dst_kwargs.pop("blockysize", None)
dst_kwargs.pop("tiled")
overview_level = 0
else:
warnings.warn(
"Block Size are bigger than raster sizes. "
"Setting blocksize to {}".format(tilesize),
IncompatibleBlockRasterSize,
)
dst_kwargs["blockxsize"] = tilesize
dst_kwargs["blockysize"] = tilesize
vrt_params = dict(add_alpha=True, dtype=dtype)
if nodata is not None:
vrt_params.update(
dict(nodata=nodata, add_alpha=False, src_nodata=nodata))
if alpha:
vrt_params.update(dict(add_alpha=False))
if web_optimized:
bounds = list(
transform_bounds(*[src_dst.crs, "epsg:4326"] +
list(src_dst.bounds),
densify_pts=21))
center = [(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2]
lat = 0 if latitude_adjustment else center[1]
max_zoom = get_max_zoom(src_dst, lat=lat, tilesize=tilesize)
extrema = tile_extrema(bounds, max_zoom)
w, n = mercantile.xy(*mercantile.ul(
extrema["x"]["min"], extrema["y"]["min"], max_zoom))
vrt_res = _meters_per_pixel(max_zoom, 0, tilesize=tilesize)
vrt_transform = Affine(vrt_res, 0, w, 0, -vrt_res, n)
vrt_width = (extrema["x"]["max"] -
extrema["x"]["min"]) * tilesize
vrt_height = (extrema["y"]["max"] -
extrema["y"]["min"]) * tilesize
vrt_params.update(
dict(
crs="epsg:3857",
transform=vrt_transform,
width=vrt_width,
height=vrt_height,
resampling=ResamplingEnums[resampling],
))
with WarpedVRT(src_dst, **vrt_params) as vrt_dst:
meta = vrt_dst.meta
meta["count"] = len(indexes)
if add_mask:
meta.pop("nodata", None)
meta.pop("alpha", None)
if (dst_kwargs.get("photometric", "").upper() == "YCBCR"
and meta["count"] == 1):
warnings.warn(
"PHOTOMETRIC=YCBCR not supported on a 1-band raster"
" and has been set to 'MINISBLACK'")
dst_kwargs["photometric"] = "MINISBLACK"
meta.update(**dst_kwargs)
if not allow_intermediate_compression:
meta.pop("compress", None)
meta.pop("photometric", None)
if in_memory is None:
in_memory = vrt_dst.width * vrt_dst.height < IN_MEMORY_THRESHOLD
if in_memory:
tmpfile = ctx.enter_context(MemoryFile())
tmp_dst = ctx.enter_context(tmpfile.open(**meta))
else:
tmpfile = ctx.enter_context(TemporaryRasterFile(dst_path))
tmp_dst = ctx.enter_context(
rasterio.open(tmpfile.name, "w", **meta))
# Transfer color interpolation
if len(indexes) == 1 and (vrt_dst.colorinterp[indexes[0] - 1]
is not ColorInterp.palette):
tmp_dst.colorinterp = [ColorInterp.gray]
else:
tmp_dst.colorinterp = [
vrt_dst.colorinterp[b - 1] for b in indexes
]
if tmp_dst.colorinterp[0] is ColorInterp.palette:
try:
tmp_dst.write_colormap(1, vrt_dst.colormap(1))
except ValueError:
warnings.warn(
"Dataset has `Palette` color interpretation"
" but is missing colormap information")
wind = list(tmp_dst.block_windows(1))
if not quiet:
click.echo("Reading input: {}".format(source), err=True)
fout = os.devnull if quiet else sys.stderr
with click.progressbar(wind,
length=len(wind),
file=fout,
show_percent=True) as windows:
for ij, w in windows:
matrix = vrt_dst.read(window=w, indexes=indexes)
tmp_dst.write(matrix, window=w)
if add_mask or mask:
# Cast mask to uint8 to fix rasterio 1.1.2 error (ref #115)
mask_value = vrt_dst.dataset_mask(
window=w).astype("uint8")
tmp_dst.write_mask(mask_value, window=w)
if overview_level is None:
overview_level = get_maximum_overview_level(
vrt_dst, tilesize)
if not quiet and overview_level:
click.echo("Adding overviews...", err=True)
overviews = [2**j for j in range(1, overview_level + 1)]
tmp_dst.build_overviews(overviews,
ResamplingEnums[overview_resampling])
if not quiet:
click.echo("Updating dataset tags...", err=True)
for i, b in enumerate(indexes):
tmp_dst.set_band_description(i + 1,
src_dst.descriptions[b - 1])
if forward_band_tags:
tmp_dst.update_tags(i + 1, **src_dst.tags(b))
tags = src_dst.tags()
tags.update(
dict(
OVR_RESAMPLING_ALG=ResamplingEnums[overview_resampling]
.name.upper()))
tmp_dst.update_tags(**tags)
tmp_dst._set_all_scales(
[vrt_dst.scales[b - 1] for b in indexes])
tmp_dst._set_all_offsets(
[vrt_dst.offsets[b - 1] for b in indexes])
if not quiet:
click.echo("Writing output to: {}".format(dst_path),
err=True)
copy(tmp_dst, dst_path, copy_src_overviews=True, **dst_kwargs)
def processAlgorithmRasterio(self, parameters, context, feedback):
"""
Here is where the processing itself takes place.
"""
import rasterio
from osgeo import gdal
from rasterio import shutil as rio_shutil
from rasterio.vrt import WarpedVRT
def get_inputfilepath(layer):
return os.path.realpath(layer.source().split("|layername")[0])
img_ref = get_inputfilepath(
self.parameterAsRasterLayer(parameters, self.IMG_REF, context))
file_in = get_inputfilepath(
self.parameterAsRasterLayer(parameters, self.INPUT, context))
output_file = self.parameterAsOutputLayer(parameters, self.OUTPUT,
context)
feedback.pushInfo("Co-registration:")
feedback.pushInfo("\nProcessing file: " + file_in)
# extract some info
with rasterio.open(img_ref) as target:
dst_crs = target.crs
x_res, y_res = target.res
vrt_options = {
'crs': target.crs,
'transform': target.transform,
'height': target.height,
'width': target.width,
'nodata': target.nodata
}
with rasterio.open(file_in) as src:
src_crs = src.crs
# ----- reprojection
if src_crs != dst_crs:
feedback.pushInfo(
"--> reprojection is required, to CRS: {}".format(dst_crs))
# reproject
reprj_file_tmp = tempfile.NamedTemporaryFile(suffix=".tif",
delete=True)
reprj_file = reprj_file_tmp.name
resample = gdal.GRA_NearestNeighbour
gdal.Warp(reprj_file,
file_in,
srcSRS=src_crs,
dstSRS=dst_crs,
xRes=x_res,
yRes=y_res,
resampleAlg=resample)
else:
reprj_file_tmp = False
reprj_file = file_in
# ----- set extent and align pixels based on PU
feedback.pushInfo("--> set extent and align pixels")
if target.nodata is not None:
feedback.pushInfo("--> nodata as: {}".format(target.nodata))
with rasterio.open(reprj_file) as src:
with WarpedVRT(src, **vrt_options) as vrt:
rio_shutil.copy(vrt, output_file, driver='GTiff')
feedback.pushInfo("--> done\n")
if reprj_file_tmp:
reprj_file_tmp.close()
return {self.OUTPUT: output_file}
def cog_translate( # noqa: C901
source: Union[str, pathlib.PurePath, DatasetReader, DatasetWriter,
WarpedVRT],
dst_path: Union[str, pathlib.PurePath],
dst_kwargs: Dict,
indexes: Optional[Sequence[int]] = None,
nodata: Optional[Union[str, int, float]] = None,
dtype: Optional[str] = None,
add_mask: bool = False,
overview_level: Optional[int] = None,
overview_resampling: str = "nearest",
web_optimized: bool = False,
tms: Optional[morecantile.TileMatrixSet] = None,
zoom_level_strategy: str = "auto",
zoom_level: Optional[int] = None,
aligned_levels: Optional[int] = None,
resampling: str = "nearest",
in_memory: Optional[bool] = None,
config: Optional[Dict] = None,
allow_intermediate_compression: bool = False,
forward_band_tags: bool = False,
quiet: bool = False,
temporary_compression: str = "DEFLATE",
colormap: Optional[Dict] = None,
additional_cog_metadata: Optional[Dict] = None,
use_cog_driver: bool = False,
):
"""
Create Cloud Optimized Geotiff.
Parameters
----------
source : str, PathLike object or rasterio.io.DatasetReader
A dataset path, URL or rasterio.io.DatasetReader object.
Will be opened in "r" mode.
dst_path : str or PathLike object
An output dataset path or or PathLike object.
Will be opened in "w" mode.
dst_kwargs: dict
Output dataset creation options.
indexes : tuple or int, optional
Raster band indexes to copy.
nodata, int, optional
Overwrite nodata masking values for input dataset.
dtype: str, optional
Overwrite output data type. Default will be the input data type.
add_mask, bool, optional
Force output dataset creation with a mask.
overview_level : int, optional (default: None)
COGEO overview (decimation) level. By default, inferred from data size.
overview_resampling : str, optional (default: "nearest")
Resampling algorithm for overviews
web_optimized: bool, optional (default: False)
Create web-optimized cogeo.
tms: morecantile.TileMatrixSet, optional (default: "WebMercatorQuad")
TileMatrixSet to use for reprojection, resolution and alignment.
zoom_level_strategy: str, optional (default: auto)
Strategy to determine zoom level (same as in GDAL 3.2).
LOWER will select the zoom level immediately below the theoretical computed non-integral zoom level, leading to subsampling.
On the contrary, UPPER will select the immediately above zoom level, leading to oversampling.
Defaults to AUTO which selects the closest zoom level.
ref: https://gdal.org/drivers/raster/cog.html#raster-cog
zoom_level: int, optional.
Zoom level number (starting at 0 for coarsest zoom level). If this option is specified, `--zoom-level-strategy` is ignored.
aligned_levels: int, optional.
Number of overview levels for which GeoTIFF tile and tiles defined in the tiling scheme match.
Default is to use the maximum overview levels. Note: GDAL use number of resolution levels instead of overview levels.
resampling : str, optional (default: "nearest")
Resampling algorithm.
in_memory: bool, optional
Force processing raster in memory (default: process in memory if small)
config : dict
Rasterio Env options.
allow_intermediate_compression: bool, optional (default: False)
Allow intermediate file compression to reduce memory/disk footprint.
Note: This could reduce the speed of the process.
Ref: https://github.com/cogeotiff/rio-cogeo/issues/103
forward_band_tags: bool, optional
Forward band tags to output bands.
Ref: https://github.com/cogeotiff/rio-cogeo/issues/19
quiet: bool, optional (default: False)
Mask processing steps.
temporary_compression: str, optional
Compression used for the intermediate file, default is deflate.
colormap: dict, optional
Overwrite or add a colormap to the output COG.
additional_cog_metadata: dict, optional
Additional dataset metadata to add to the COG.
use_cog_driver: bool, optional (default: False)
Use GDAL COG driver if set to True. COG driver is available starting with GDAL 3.1.
"""
tms = tms or morecantile.tms.get("WebMercatorQuad")
dst_kwargs = dst_kwargs.copy()
if isinstance(indexes, int):
indexes = (indexes, )
config = config or {}
with rasterio.Env(**config):
with ExitStack() as ctx:
if isinstance(source, (DatasetReader, DatasetWriter, WarpedVRT)):
src_dst = source
else:
src_dst = ctx.enter_context(rasterio.open(source))
meta = src_dst.meta
indexes = indexes if indexes else src_dst.indexes
nodata = nodata if nodata is not None else src_dst.nodata
dtype = dtype if dtype else src_dst.dtypes[0]
alpha = utils.has_alpha_band(src_dst)
mask = utils.has_mask_band(src_dst)
if colormap and len(indexes) > 1:
raise IncompatibleOptions(
"Cannot add a colormap for multiple bands data.")
if not add_mask and (
(nodata is not None or alpha)
and dst_kwargs.get("compress", "").lower() == "jpeg"):
warnings.warn(
"Nodata/Alpha band will be translated to an internal mask band.",
)
add_mask = True
indexes = (utils.non_alpha_indexes(src_dst)
if len(indexes) not in [1, 3] else indexes)
tilesize = min(int(dst_kwargs["blockxsize"]),
int(dst_kwargs["blockysize"]))
if src_dst.width < tilesize or src_dst.height < tilesize:
tilesize = 2**int(
math.log(min(src_dst.width, src_dst.height), 2))
if tilesize < 64:
warnings.warn(
"Raster has dimension < 64px. Output COG cannot be tiled"
" and overviews cannot be added.",
IncompatibleBlockRasterSize,
)
dst_kwargs.pop("blockxsize", None)
dst_kwargs.pop("blockysize", None)
dst_kwargs.pop("tiled")
overview_level = 0
else:
warnings.warn(
"Block Size are bigger than raster sizes. "
"Setting blocksize to {}".format(tilesize),
IncompatibleBlockRasterSize,
)
dst_kwargs["blockxsize"] = tilesize
dst_kwargs["blockysize"] = tilesize
vrt_params = {
"add_alpha": True,
"dtype": dtype,
"width": src_dst.width,
"height": src_dst.height,
"resampling": ResamplingEnums[resampling],
}
if nodata is not None:
vrt_params.update(
dict(nodata=nodata, add_alpha=False, src_nodata=nodata))
if alpha:
vrt_params.update(dict(add_alpha=False))
if web_optimized and not use_cog_driver:
params = utils.get_web_optimized_params(
src_dst,
zoom_level_strategy=zoom_level_strategy,
zoom_level=zoom_level,
aligned_levels=aligned_levels,
tms=tms,
)
vrt_params.update(**params)
with WarpedVRT(src_dst, **vrt_params) as vrt_dst:
meta = vrt_dst.meta
meta["count"] = len(indexes)
if add_mask:
meta.pop("nodata", None)
meta.pop("alpha", None)
if (dst_kwargs.get("photometric", "").upper() == "YCBCR"
and meta["count"] == 1):
warnings.warn(
"PHOTOMETRIC=YCBCR not supported on a 1-band raster"
" and has been set to 'MINISBLACK'")
dst_kwargs["photometric"] = "MINISBLACK"
meta.update(**dst_kwargs)
meta.pop("compress", None)
meta.pop("photometric", None)
if allow_intermediate_compression:
meta["compress"] = temporary_compression
if in_memory is None:
in_memory = vrt_dst.width * vrt_dst.height < IN_MEMORY_THRESHOLD
if in_memory:
tmpfile = ctx.enter_context(MemoryFile())
tmp_dst = ctx.enter_context(tmpfile.open(**meta))
else:
tmpfile = ctx.enter_context(TemporaryRasterFile(dst_path))
tmp_dst = ctx.enter_context(
rasterio.open(tmpfile.name, "w", **meta))
# Transfer color interpolation
if len(indexes) == 1 and (vrt_dst.colorinterp[indexes[0] - 1]
is not ColorInterp.palette):
tmp_dst.colorinterp = [ColorInterp.gray]
else:
tmp_dst.colorinterp = [
vrt_dst.colorinterp[b - 1] for b in indexes
]
if colormap:
if tmp_dst.colorinterp[0] is not ColorInterp.palette:
tmp_dst.colorinterp = [ColorInterp.palette]
warnings.warn(
"Dataset color interpretation was set to `Palette`"
)
tmp_dst.write_colormap(1, colormap)
elif tmp_dst.colorinterp[0] is ColorInterp.palette:
try:
tmp_dst.write_colormap(1, vrt_dst.colormap(1))
except ValueError:
warnings.warn(
"Dataset has `Palette` color interpretation"
" but is missing colormap information")
wind = list(tmp_dst.block_windows(1))
if not quiet:
click.echo("Reading input: {}".format(source), err=True)
fout = ctx.enter_context(open(os.devnull,
"w")) if quiet else sys.stderr
with click.progressbar(
wind, file=fout,
show_percent=True) as windows: # type: ignore
for _, w in windows:
matrix = vrt_dst.read(window=w, indexes=indexes)
tmp_dst.write(matrix, window=w)
if add_mask or mask:
# Cast mask to uint8 to fix rasterio 1.1.2 error (ref #115)
mask_value = vrt_dst.dataset_mask(
window=w).astype("uint8")
tmp_dst.write_mask(mask_value, window=w)
if overview_level is None:
overview_level = get_maximum_overview_level(
vrt_dst.width, vrt_dst.height, minsize=tilesize)
if not quiet and overview_level:
click.echo("Adding overviews...", err=True)
overviews = [2**j for j in range(1, overview_level + 1)]
tmp_dst.build_overviews(overviews,
ResamplingEnums[overview_resampling])
if not quiet:
click.echo("Updating dataset tags...", err=True)
for i, b in enumerate(indexes):
tmp_dst.set_band_description(i + 1,
src_dst.descriptions[b - 1])
if forward_band_tags:
tmp_dst.update_tags(i + 1, **src_dst.tags(b))
tags = src_dst.tags()
tags.update(
dict(
OVR_RESAMPLING_ALG=ResamplingEnums[overview_resampling]
.name.upper()))
if additional_cog_metadata:
tags.update(**additional_cog_metadata)
if web_optimized and not use_cog_driver:
default_zoom = tms.zoom_for_res(
max(tmp_dst.res),
max_z=30,
zoom_level_strategy=zoom_level_strategy,
)
dst_kwargs.update({
"@TILING_SCHEME_NAME":
tms.identifier,
"@TILING_SCHEME_ZOOM_LEVEL":
zoom_level if zoom_level is not None else default_zoom,
})
if aligned_levels:
dst_kwargs.update(
{"@TILING_SCHEME_ALIGNED_LEVELS": aligned_levels})
tmp_dst.update_tags(**tags)
tmp_dst._set_all_scales(
[vrt_dst.scales[b - 1] for b in indexes])
tmp_dst._set_all_offsets(
[vrt_dst.offsets[b - 1] for b in indexes])
if not quiet:
click.echo("Writing output to: {}".format(dst_path),
err=True)
if use_cog_driver:
if not GDALVersion.runtime().at_least("3.1"):
raise Exception(
"GDAL 3.1 or above required to use the COG driver."
)
dst_kwargs["driver"] = "COG"
if web_optimized:
dst_kwargs["TILING_SCHEME"] = (
"GoogleMapsCompatible" if tms.identifier
== "WebMercatorQuad" else tms.identifier)
if zoom_level is not None:
if not GDALVersion.runtime().at_least("3.5"):
warnings.warn(
"ZOOM_LEVEL option is only available with GDAL >3.5."
)
dst_kwargs["ZOOM_LEVEL"] = zoom_level
dst_kwargs["ZOOM_LEVEL_STRATEGY"] = zoom_level_strategy
if aligned_levels is not None:
# GDAL uses Number of resolution (not overviews)
# See https://github.com/OSGeo/gdal/issues/5336#issuecomment-1042946603
dst_kwargs["aligned_levels"] = aligned_levels + 1
if add_mask and dst_kwargs.get("compress", "") != "JPEG":
warnings.warn(
"With GDAL COG driver, mask band will be translated to an alpha band."
)
dst_kwargs["overview_resampling"] = overview_resampling
dst_kwargs["warp_resampling"] = resampling
dst_kwargs["blocksize"] = tilesize
dst_kwargs.pop("blockxsize", None)
dst_kwargs.pop("blockysize", None)
dst_kwargs.pop("tiled", None)
dst_kwargs.pop("interleave", None)
dst_kwargs.pop("photometric", None)
copy(tmp_dst, dst_path, **dst_kwargs)
else:
copy(tmp_dst,
dst_path,
copy_src_overviews=True,
**dst_kwargs)
