def cogeo(
input,
output,
bidx,
cogeo_profile,
nodata,
alpha,
overview_level,
threads,
creation_options,
):
"""Create Cloud Optimized Geotiff."""
if nodata is not None and alpha is not None:
raise click.ClickException('Incompatible options "alpha" and "nodata"')
output_profile = cog_profiles.get(cogeo_profile)
output_profile.update(dict(BIGTIFF=os.environ.get("BIGTIFF", "IF_SAFER")))
if creation_options:
output_profile.update(creation_options)
block_size = min(output_profile["blockxsize"], output_profile["blockysize"])
config = dict(
NUM_THREADS=threads,
GDAL_TIFF_INTERNAL_MASK=os.environ.get("GDAL_TIFF_INTERNAL_MASK", True),
GDAL_TIFF_OVR_BLOCKSIZE=os.environ.get("GDAL_TIFF_OVR_BLOCKSIZE", block_size),
)
cog_translate(
input, output, output_profile, bidx, nodata, alpha, overview_level, config
)
def create_cog_in_s3(services,
profile,
path,
raster,
bucket_name,
nodata=None,
tags=None):
with MemoryFile() as dst_file:
with MemoryFile() as memfile:
with memfile.open(**profile) as mem:
if nodata:
mem.nodata = nodata
mem.write_band(1, raster)
if tags:
mem.update_tags(**tags)
dst_profile = cog_profiles.get("deflate")
cog_translate(mem,
dst_file.name,
dst_profile,
in_memory=True,
quiet=True)
services.upload_fileobj_S3(dst_file,
path, {'ACL': 'public-read'},
bucket_name=bucket_name)
return True
def test_cog_translate_validCustom():
"""Should work as expected (create cogeo file)."""
runner = CliRunner()
with runner.isolated_filesystem():
config = dict(GDAL_TIFF_OVR_BLOCKSIZE=256)
ycbcr_profile.update({"blockxsize": 256, "blockysize": 256})
cog_translate(
raster_path_rgba,
"cogeo_env.tif",
ycbcr_profile,
indexes=[1, 2, 3],
config=config,
)
with rasterio.open("cogeo_env.tif") as src:
assert src.height == 512
assert src.width == 512
assert src.meta["dtype"] == "uint8"
assert src.is_tiled
assert src.compression.value == "JPEG"
assert src.profile["blockxsize"] == 256
assert src.profile["blockysize"] == 256
assert src.photometric.value == "YCbCr"
assert src.interleaving.value == "PIXEL"
assert src.overviews(1) == [2, 4, 8, 16, 32, 64]
def cogify(fin, fout, nodata=None):
""" Turn a geospatial image into a COG """
logger.debug(f"Turning {fin} into COG named {fout}")
output_profile = cog_profiles.get('deflate')
output_profile.update(dict(BIGTIFF=os.environ.get("BIGTIFF", "IF_SAFER")))
output_profile['blockxsize'] = 256
output_profile['blockysize'] = 256
threads = 1
overview_blocksize = 128
config = dict(
NUM_THREADS=threads,
GDAL_TIFF_INTERNAL_MASK=os.environ.get("GDAL_TIFF_INTERNAL_MASK",
True),
GDAL_TIFF_OVR_BLOCKSIZE=str(overview_blocksize),
)
cog_translate(fin,
fout,
output_profile,
config=config,
nodata=nodata,
overview_resampling="bilinear",
add_mask=False,
web_optimized=False)
return fout
def mem_cog(raster, meta, filename):
config = dict(
GDAL_NUM_THREADS="ALL_CPUS",
GDAL_TIFF_INTERNAL_MASK=False,
GDAL_TIFF_OVR_BLOCKSIZE="128",
)
with MemoryFile() as memfile:
with memfile.open(**meta) as mem:
# Populate the input file with numpy array
mem.write(raster)
dst_profile = cog_profiles.get("deflate")
dst_profile.update(dict(BIGTIFF="IF_SAFER"))
cog_translate(mem,
filename,
dst_profile,
nodata=0,
add_mask=False,
config=config,
in_memory=True,
quiet=True)
if cog_validate(filename):
print('COGs created and validated')
with open('cog_list_' + time_frame + '.txt', 'a') as f:
print(f"{filename}", file=f)
def test_cog_translate_web_align():
"""
Test Web-Optimized COG.
- Test COG bounds (thus block) is aligned with Zoom levels
"""
tms = morecantile.tms.get("WebMercatorQuad")
runner = CliRunner()
with runner.isolated_filesystem():
web_profile = cog_profiles.get("raw")
web_profile.update({"blockxsize": 256, "blockysize": 256})
config = dict(GDAL_TIFF_OVR_BLOCKSIZE="256")
with rasterio.open(raster_path_web) as src_dst:
_, max_zoom = get_zooms(src_dst)
cog_translate(
raster_path_web,
"cogeo.tif",
web_profile,
quiet=True,
web_optimized=True,
config=config,
aligned_levels=2,
)
with COGReader(raster_path_web) as src_dst:
bounds = src_dst.bounds
with COGReader("cogeo.tif") as cog:
_, max_zoom = get_zooms(cog.dataset)
ulTile = tms.xy_bounds(tms.tile(bounds[0], bounds[3], max_zoom - 2))
assert round(cog.dataset.bounds[0], 5) == round(ulTile.left, 5)
assert round(cog.dataset.bounds[3], 5) == round(ulTile.top, 5)
lrTile = tms.xy_bounds(tms.tile(bounds[2], bounds[1], max_zoom - 2))
assert round(cog.dataset.bounds[2], 5) == round(lrTile.right, 5)
assert round(cog.dataset.bounds[1], 5) == round(lrTile.bottom, 5)
cog_translate(
raster_path_web,
"cogeo.tif",
web_profile,
quiet=True,
web_optimized=True,
config=config,
aligned_levels=3,
)
with COGReader(raster_path_web) as src_dst:
bounds = src_dst.bounds
with COGReader("cogeo.tif") as cog_dst:
_, max_zoom = get_zooms(cog_dst.dataset)
ulTile = tms.xy_bounds(tms.tile(bounds[0], bounds[3], max_zoom - 3))
assert round(cog_dst.dataset.bounds[0], 5) == round(ulTile.left, 5)
assert round(cog_dst.dataset.bounds[3], 5) == round(ulTile.top, 5)
lrTile = tms.xy_bounds(tms.tile(bounds[2], bounds[1], max_zoom - 3))
assert round(cog_dst.dataset.bounds[2], 5) == round(lrTile.right, 5)
assert round(cog_dst.dataset.bounds[1], 5) == round(lrTile.bottom, 5)
def test_gdal_cog_compareWeb(runner):
"""Test GDAL COG."""
with runner.isolated_filesystem():
profile = cog_profiles.get("jpeg")
profile["blockxsize"] = 256
profile["blockysize"] = 256
# rio cogeo GDAL COG
cog_translate(
raster_path_rgba,
"gdalcogeo.tif",
profile.copy(),
quiet=True,
use_cog_driver=True,
web_optimized=True,
)
# pure COG
copy(
raster_path_rgba,
"cog.tif",
driver="COG",
blocksize=256,
compress="JPEG",
TILING_SCHEME="GoogleMapsCompatible",
)
with rasterio.open("gdalcogeo.tif") as gdalcogeo, rasterio.open(
"cog.tif"
) as cog:
assert cog.meta == gdalcogeo.meta
def generate_tiff_from_array(meta, array, obs_path):
temp_tiff_file = tempfile.mktemp(suffix=".tiff")
temp_cog_file = tempfile.mktemp(suffix=".tiff")
with rasterio.open(temp_tiff_file, mode='w', driver=meta.get('driver'), width=meta.get('width'),
height=meta.get('height'), transform=meta.get('transform'), crs=meta.get('crs'),
count=meta.get('count'), nodata=meta.get('nodata'), dtype=array.dtype) as dst:
for band in range(0, meta.get('count')):
dst.write_band(band + 1, array[band])
try:
# convert tiff to cog
output_profile = cog_profiles.get("deflate")
output_profile.update({"BLOCKXSIZE": 256, "BLOCKYSIZE": 256})
config = dict(
NUM_THREADS=8,
GDAL_TIFF_INTERNAL_MASK=os.environ.get("GDAL_TIFF_INTERNAL_MASK", True),
GDAL_TIFF_OVR_BLOCKSIZE=str(os.environ.get("GDAL_TIFF_OVR_BLOCKSIZE", 128))
)
cog_translate(temp_tiff_file, temp_cog_file, output_profile, add_mask=True, web_optimized=False,
config=config)
s3 = S3()
return s3.upload(local_file=temp_cog_file, obs_path=obs_path)
except Exception as e:
raise ConvertCogError(e.__str__())
finally:
if os.path.exists(temp_tiff_file):
os.remove(temp_tiff_file)
if os.path.exists(temp_cog_file):
os.remove(temp_cog_file)
def to_obstiff(arr, obs_path, proj="EPSG:4326", spec=None, bands=None, **kwargs):
temp_file = tempfile.mktemp(suffix=".tiff")
temp_cog_file = tempfile.mktemp(suffix=".tiff")
# save to tiff
to_geotiff(arr, path=temp_file, proj=proj, spec=spec, bands=bands, **kwargs)
try:
# convert tiff to cog
output_profile = cog_profiles.get("deflate")
output_profile.update({"BLOCKXSIZE": 256, "BLOCKYSIZE": 256})
config = dict(
NUM_THREADS=8,
GDAL_TIFF_INTERNAL_MASK=os.environ.get("GDAL_TIFF_INTERNAL_MASK", True),
GDAL_TIFF_OVR_BLOCKSIZE=str(os.environ.get("GDAL_TIFF_OVR_BLOCKSIZE", 128))
)
cog_translate(temp_file, temp_cog_file, output_profile, add_mask=True, web_optimized=False, config=config)
s3 = S3()
return s3.upload(local_file=temp_cog_file, obs_path=obs_path)
except Exception as e:
raise ConvertCogError(e.__str__())
finally:
if os.path.exists(temp_file):
os.remove(temp_file)
if os.path.exists(temp_cog_file):
os.remove(temp_cog_file)
def jp2_to_cog(band_src_path):
'''
Given the path of a band of sentinel (.jp2) generates a Cloud Optimized GeoTiff version.
'''
config = dict(NUM_THREADS=100, GDAL_TIFF_OVR_BLOCKSIZE=128)
output_profile = {
"driver": "GTiff",
"interleave": "pixel",
"tiled": True,
"blockxsize": 256,
"blockysize": 256,
"compress": "DEFLATE",
}
cog_path = f"{band_src_path[band_src_path.rfind('/')+1:band_src_path.rfind('.')]}.tif"
cog_translate(
band_src_path,
cog_path,
output_profile,
nodata=0,
in_memory=False,
config=config,
quiet=True,
)
return cog_path
def _translate(src_path,
dst_path,
profile="webp",
profile_options={},
**options):
"""Convert image to COG."""
output_profile = cog_profiles.get(profile)
output_profile.update(dict(BIGTIFF="IF_SAFER"))
output_profile.update(profile_options)
config = dict(
GDAL_NUM_THREADS="ALL_CPUS",
GDAL_TIFF_INTERNAL_MASK=True,
GDAL_TIFF_OVR_BLOCKSIZE="128",
)
cog_translate(
src_path,
dst_path,
output_profile,
config=config,
in_memory=False,
quiet=True,
allow_intermediate_compression=True,
**options,
)
return True
def _translate(src_path,
dst_path,
profile="webp",
profile_options={},
**options):
#source: https://github.com/cogeotiff/rio-cogeo
"""Convert image to COG."""
# Format creation option (see gdalwarp `-co` option)
output_profile = cog_profiles.get(profile)
output_profile.update(dict(BIGTIFF="IF_SAFER"))
output_profile.update(profile_options)
# Dataset Open option (see gdalwarp `-oo` option)
config = dict(
GDAL_NUM_THREADS="ALL_CPUS",
GDAL_TIFF_INTERNAL_MASK=True,
GDAL_TIFF_OVR_BLOCKSIZE="128",
)
cog_translate(
src_path,
dst_path,
output_profile,
config=config,
in_memory=False,
quiet=True,
**options,
)
return True
def _create_cog(self, src_path, dest_path):
"""Overwrite non-cloud-optimized GeoTIFF with a COG."""
cog_translate(src_path=src_path,
dst_path=dest_path,
dst_kwargs={'crs': self.dest_crs},
resampling=self.resampling,
latitude_adjustment=False)
def test_cog_translate_valid():
"""Should work as expected (create cogeo file)."""
runner = CliRunner()
with runner.isolated_filesystem():
cog_translate(raster_path_rgb, "cogeo.tif", jpeg_profile, quiet=True)
with rasterio.open("cogeo.tif") as src:
assert src.height == 512
assert src.width == 512
assert src.meta["dtype"] == "uint8"
assert src.is_tiled
assert src.profile["blockxsize"] == 64
assert src.profile["blockysize"] == 64
assert src.compression.value == "JPEG"
assert src.photometric.value == "YCbCr"
assert src.interleaving.value == "PIXEL"
assert src.overviews(1) == [2, 4, 8]
assert src.tags()["OVR_RESAMPLING_ALG"] == "NEAREST"
assert not has_mask_band(src)
cog_translate(raster_path_rgb,
"cogeo.tif",
jpeg_profile,
add_mask=True,
quiet=True)
with rasterio.open("cogeo.tif") as src:
assert has_mask_band(src)
def test_cog_translate_NodataMask():
"""Should work as expected (create cogeo and translate nodata to mask)."""
runner = CliRunner()
with runner.isolated_filesystem():
cog_translate(
raster_path_missingnodata,
"cogeo.tif",
deflate_profile,
nodata=-9999,
add_mask=True,
quiet=True,
)
with rasterio.open("cogeo.tif") as src:
assert src.nodata is None
assert has_mask_band(src)
assert not src.dataset_mask().all()
cog_translate(raster_path_nodata,
"cogeo.tif",
deflate_profile,
add_mask=True,
quiet=True)
with rasterio.open("cogeo.tif") as src:
assert src.nodata is None
assert has_mask_band(src)
assert not src.dataset_mask().all()
def test_cog_translate_valid_blocksize(runner):
"""Should work as expected (create cogeo file)."""
with runner.isolated_filesystem():
with pytest.warns(IncompatibleBlockRasterSize):
cog_translate(raster_path_small,
"cogeo.tif",
default_profile,
quiet=True)
assert cog_validate("cogeo.tif")
with rasterio.open("cogeo.tif") as src:
assert src.height == 171
assert src.width == 171
assert src.is_tiled
assert src.profile["blockxsize"] == 128
assert src.profile["blockysize"] == 128
assert src.overviews(1) == [2]
with pytest.warns(IncompatibleBlockRasterSize):
cog_translate(raster_path_toosmall,
"cogeo.tif",
default_profile,
quiet=True)
assert cog_validate("cogeo.tif")
with rasterio.open("cogeo.tif") as src:
assert src.height == 51
assert src.width == 51
assert not src.is_tiled
assert not src.profile.get("blockxsize")
assert not src.profile.get("blockysize")
assert not src.overviews(1)
def test_cog_translate_mask():
"""Should work as expected (copy mask from input)."""
runner = CliRunner()
with runner.isolated_filesystem():
cog_translate(raster_path_mask, "cogeo.tif", jpeg_profile, quiet=True)
with rasterio.open("cogeo.tif") as src:
assert has_mask_band(src)
def cogify(fin,
fout,
nodata=None,
web_optimized=False,
blocksize=256,
overview_blocksize=128,
overview_resampling='nearest'):
""" Turn a geospatial image into a COG """
output_profile = cog_profiles.get('deflate')
output_profile.update({
"BIGTIFF": getenv("BIGTIFF", "IF_SAFER"),
"blockxsize": blocksize,
"blockysize": blocksize,
"PREDICTOR": 2
})
config = {
"NUM_THREADS": "ALL_CPUS",
"GDAL_TIFF_INTERNAL_MASK": getenv("GDAL_TIFF_INTERNAL_MASK", True),
"GDAL_TIFF_OVR_BLOCKSIZE": str(overview_blocksize)
}
cog_translate(fin,
fout,
output_profile,
config=config,
nodata=nodata,
overview_resampling=overview_resampling,
add_mask=False,
web_optimized=web_optimized)
return fout
def create_cog(input, output, overview_resampling, bidx):
cogeo_profile = 'deflate'
nodata = -1
overview_level = 6
overview_resampling = overview_resampling
threads = 8
output_profile = cog_profiles.get(cogeo_profile)
output_profile.update(dict(BIGTIFF=os.environ.get("BIGTIFF", "IF_SAFER")))
block_size = min(int(output_profile["blockxsize"]),
int(output_profile["blockysize"]))
config = dict(
NUM_THREADS=threads,
GDAL_TIFF_INTERNAL_MASK=os.environ.get("GDAL_TIFF_INTERNAL_MASK",
True),
GDAL_TIFF_OVR_BLOCKSIZE=os.environ.get("GDAL_TIFF_OVR_BLOCKSIZE",
block_size),
)
cog_translate(src_path=input,
dst_path=output,
dst_kwargs=output_profile,
indexes=bidx,
nodata=nodata,
web_optimized=False,
add_mask=False,
overview_level=overview_level,
overview_resampling=overview_resampling,
config=config,
quiet=False)
def raster_to_cog(raster, transform, dst_path, block_size=None, nodata=None):
block_size = 256 if block_size is None else block_size
nrows, ncols = np.shape(raster)
# Source profile.
src_profile = dict(
driver='GTiff',
height=nrows,
width=ncols,
count=1,
dtype=raster.dtype, # if data_type is None else data_type,
crs='EPSG:3857',
transform=transform,
nodata=np.nan if nodata is None else nodata,
)
# Write data.
with MemoryFile() as memfile:
with memfile.open(**src_profile) as mem:
# Write raster to mem file.
mem.write(raster, 1)
# Copy to disk.
dst_profile = cog_profiles.get("raw")
dst_profile["blockxsize"] = block_size
dst_profile["blockysize"] = block_size
cog_translate(mem,
dst_path,
dst_profile,
in_memory=True,
quiet=True,
web_optimized=True)
def convert_to_cog(raster, validate=True):
out_path = str(raster.with_suffix(".tif")).replace(" ", "_")
assert raster != out_path, "Can't convert to files of the same name"
cog_translate(raster, out_path, output_profile, quiet=True)
if validate:
cog_validate(out_path)
return pathlib.Path(out_path)
def viz(src_paths, style, port, mapbox_token, no_check):
"""Rasterio Viz cli."""
# Check if cog
src_paths = list(src_paths)
with ExitStack() as ctx:
for ii, src_path in enumerate(src_paths):
if not no_check and not cog_validate(src_path):
# create tmp COG
click.echo("create temporaty COG")
tmp_path = ctx.enter_context(TemporaryRasterFile(src_path))
output_profile = cog_profiles.get("deflate")
output_profile.update(dict(blockxsize="256", blockysize="256"))
config = dict(
GDAL_TIFF_INTERNAL_MASK=os.environ.get(
"GDAL_TIFF_INTERNAL_MASK", True
),
GDAL_TIFF_OVR_BLOCKSIZE="128",
)
cog_translate(src_path, tmp_path.name, output_profile, config=config)
src_paths[ii] = tmp_path.name
src_dst = raster.RasterTiles(src_paths)
application = app.viz(src_dst, token=mapbox_token, port=port)
url = application.get_template_url()
click.echo(f"Viewer started at {url}", err=True)
click.launch(url)
application.start()
def test_cog_translate_validOverviews(runner):
"""Should work as expected (create cogeo file)."""
with runner.isolated_filesystem():
cog_translate(
raster_path_rgb, "cogeo.tif", jpeg_profile, overview_level=2, quiet=True
)
with rasterio.open("cogeo.tif") as src:
assert src.overviews(1) == [2, 4]
def assure_cogeo(src_path):
"""
Guarantee that the .tif passed as an argument is a Cloud Optimized GeoTIFF (cogeo)
If the path is not a cogeo, it is destructively converted into a cogeo.
If the file cannot be converted, the function does not change the file
:param src_path: path to GeoTIFF (cogeo or not)
:return: None
"""
if not os.path.isfile(src_path):
logger.warning("Cannot validate cogeo: %s (file does not exist)" %
src_path)
return
if valid_cogeo(src_path):
return
# Not a cogeo
logger.info("Optimizing %s as Cloud Optimized GeoTIFF" % src_path)
tmpfile = tempfile.mktemp('_cogeo.tif', dir=settings.MEDIA_TMP)
swapfile = tempfile.mktemp('_cogeo_swap.tif', dir=settings.MEDIA_TMP)
with rasterio.open(src_path) as dst:
output_profile = dict(blockxsize=256,
blockysize=256,
driver='GTiff',
tiled=True,
compress=dst.profile.get('compress', 'deflate'),
interleave='pixel')
# Dataset Open option (see gdalwarp `-oo` option)
config = dict(
GDAL_NUM_THREADS="ALL_CPUS",
GDAL_TIFF_INTERNAL_MASK=True,
GDAL_TIFF_OVR_BLOCKSIZE="128",
)
cog_translate(dst,
tmpfile,
output_profile,
config=config,
in_memory=False,
quiet=True,
web_optimized=True)
if os.path.isfile(tmpfile):
shutil.move(src_path, swapfile) # Move to swap location
try:
shutil.move(tmpfile, src_path)
except IOError as e:
logger.warning("Cannot move %s to %s: %s" %
(tmpfile, src_path, str(e)))
shutil.move(swapfile, src_path) # Attempt to restore
raise e
if os.path.isfile(swapfile):
os.remove(swapfile)
def create(
input,
output,
bidx,
cogeo_profile,
nodata,
dtype,
add_mask,
overview_level,
overview_resampling,
overview_blocksize,
web_optimized,
latitude_adjustment,
resampling,
in_memory,
allow_intermediate_compression,
forward_band_tags,
threads,
creation_options,
quiet,
):
"""Create Cloud Optimized Geotiff."""
if latitude_adjustment is not None and not web_optimized:
warnings.warn(
"'latitude_adjustment' option has to be used with --web-optimized options. "
"Will be ignored.")
output_profile = cog_profiles.get(cogeo_profile)
output_profile.update(dict(BIGTIFF=os.environ.get("BIGTIFF", "IF_SAFER")))
if creation_options:
output_profile.update(creation_options)
config = dict(
GDAL_NUM_THREADS=threads,
GDAL_TIFF_INTERNAL_MASK=os.environ.get("GDAL_TIFF_INTERNAL_MASK",
True),
GDAL_TIFF_OVR_BLOCKSIZE=str(overview_blocksize),
)
cog_translate(
input,
output,
output_profile,
indexes=bidx,
nodata=nodata,
dtype=dtype,
add_mask=add_mask,
overview_level=overview_level,
overview_resampling=overview_resampling,
web_optimized=web_optimized,
latitude_adjustment=latitude_adjustment,
resampling=resampling,
in_memory=in_memory,
config=config,
allow_intermediate_compression=allow_intermediate_compression,
forward_band_tags=forward_band_tags,
quiet=quiet,
)
def combine_cog(PATH, OUTPATH, TILE, YEAR):
logging.info("Combining GeoTIFFs")
if int(YEAR) > 2000:
bands = ['HH', 'HV', 'linci', 'date', 'mask']
else:
bands = ['HH', 'linci', 'date', 'mask']
output_cogs = []
gtiff_abs_path = os.path.abspath(PATH)
outtiff_abs_path = os.path.abspath(OUTPATH)
for band in bands:
# Find all the files
all_files = []
for path, subdirs, files in os.walk(gtiff_abs_path):
for fname in files:
if int(YEAR) > 2010:
if '_{}_'.format(band) in fname and not fname.endswith('.hdr'):
in_filename = os.path.join(path, fname)
all_files.append(in_filename)
else:
if '_{}'.format(band) in fname and not fname.endswith('.hdr'):
in_filename = os.path.join(path, fname)
all_files.append(in_filename)
# Create the VRT
logging.info("Building VRT for {} with {} files found".format(
band, len(all_files)))
vrt_path = os.path.join(gtiff_abs_path, '{}.vrt'.format(band))
if int(YEAR) > 2010:
cog_filename = os.path.join(outtiff_abs_path, '{}_{}_sl_{}_F02DAR.tif'.format(TILE, YEAR[-2:], band))
else:
cog_filename = os.path.join(outtiff_abs_path, '{}_{}_sl_{}.tif'.format(TILE, YEAR[-2:], band))
vrt_options = gdal.BuildVRTOptions()
gdal.BuildVRT(
vrt_path,
all_files,
options=vrt_options
)
# Default to nearest resampling
resampling = 'nearest'
if band in ['HH', 'HV', 'linci']:
resampling = 'average'
cog_translate(
vrt_path,
cog_filename,
cog_profile,
config={"GDAL_TIFF_OVR_BLOCKSIZE": "512"},
overview_level=5,
overview_resampling=resampling
)
output_cogs.append(cog_filename)
# Return the list of written files
return output_cogs
def process(
url: str,
out_bucket: str,
out_key: str,
profile: str = "webp",
profile_options: Dict = {},
allow_remote_read: bool = False,
copy_valid_cog: bool = False,
**options: Any,
) -> None:
"""Download, convert and upload."""
url_info = urlparse(url.strip())
if url_info.scheme not in ["http", "https", "s3"]:
raise Exception(f"Unsuported scheme {url_info.scheme}")
if allow_remote_read:
src_path = url
else:
src_path = "/tmp/" + os.path.basename(url_info.path)
if url_info.scheme.startswith("http"):
wget.download(url, src_path)
elif url_info.scheme == "s3":
in_bucket = url_info.netloc
in_key = url_info.path.strip("/")
_s3_download(in_bucket, in_key, src_path)
if copy_valid_cog and cog_validate(src_path):
with open(src_path, "rb") as f:
_upload_obj(f, out_bucket, out_key)
else:
config = dict(
GDAL_NUM_THREADS="ALL_CPUS",
GDAL_TIFF_INTERNAL_MASK=True,
GDAL_TIFF_OVR_BLOCKSIZE="128",
)
output_profile = cog_profiles.get(profile)
output_profile.update(dict(BIGTIFF="IF_SAFER"))
output_profile.update(profile_options)
with MemoryFile() as mem_dst:
cog_translate(
src_path,
mem_dst.name,
output_profile,
config=config,
in_memory=False, # Limit Memory usage
quiet=True,
allow_intermediate_compression=True, # Limit Disk usage
**options,
)
_upload_obj(mem_dst, out_bucket, out_key)
del mem_dst
if not allow_remote_read:
os.remove(src_path)
return
def test_cog_translate_dataset(runner):
"""Should work as expected (create cogeo from an open dataset)."""
with runner.isolated_filesystem():
with rasterio.open(raster_path_rgb) as src_dst:
cog_translate(src_dst, "cogeo.tif", jpeg_profile, quiet=True)
assert not src_dst.closed
with rasterio.open("cogeo.tif") as src:
_validate_translated_rgb_jpeg(src)
def test_cog_translate_warpedvrt(runner):
"""Should work as expected (create cogeo from an open memfile)."""
with runner.isolated_filesystem():
with rasterio.open(raster_path_rgb) as dataset:
with WarpedVRT(dataset) as vrt:
cog_translate(vrt, "cogeo.tif", jpeg_profile, quiet=True)
with rasterio.open("cogeo.tif") as src:
_validate_translated_rgb_jpeg(src)
def convert_to_cog(raster, out_path=None, validate=True, **kwargs):
output_profile = cog_profiles.get("deflate")
if out_path is None:
out_path = str(raster.with_suffix(".tif")).replace(" ", "_")
assert raster != out_path, "Can't convert to files of the same name"
cog_translate(raster, out_path, output_profile, quiet=True, **kwargs)
if validate:
cog_validate(out_path)
return pathlib.Path(out_path)
