def convert_to_cog(raster, validate=True):
out_path = str(raster.with_suffix(".tif")).replace(" ", "_")
assert raster != out_path, "Can't convert to files of the same name"
cog_translate(raster, out_path, output_profile, quiet=True)
if validate:
cog_validate(out_path)
return pathlib.Path(out_path)
def test_cog_validate_validCreatioValid(monkeypatch):
"""Should work as expected (validate cogeo file)."""
runner = CliRunner()
with runner.isolated_filesystem():
cog_translate(raster_rgb, "cogeo.tif", jpeg_profile, quiet=True)
assert cog_validate("cogeo.tif",
config={"GDAL_DISABLE_READDIR_ON_OPEN": "FALSE"})
cog_translate(raster_rgb,
"cogeo.tif",
jpeg_profile,
overview_level=0,
quiet=True)
assert cog_validate("cogeo.tif",
config={"GDAL_DISABLE_READDIR_ON_OPEN": "FALSE"})
# Change in rasterio 1.0.26
# https://github.com/mapbox/rasterio/blob/master/CHANGES.txt#L43
config = dict(GDAL_TIFF_OVR_BLOCKSIZE="1024")
cog_translate(
raster_big,
"cogeo.tif",
jpeg_profile,
overview_level=1,
config=config,
quiet=True,
)
assert cog_validate("cogeo.tif",
config={"GDAL_DISABLE_READDIR_ON_OPEN": "FALSE"})
def test_cog_translate_valid_blocksize(runner):
"""Should work as expected (create cogeo file)."""
with runner.isolated_filesystem():
with pytest.warns(IncompatibleBlockRasterSize):
cog_translate(raster_path_small,
"cogeo.tif",
default_profile,
quiet=True)
assert cog_validate("cogeo.tif")
with rasterio.open("cogeo.tif") as src:
assert src.height == 171
assert src.width == 171
assert src.is_tiled
assert src.profile["blockxsize"] == 128
assert src.profile["blockysize"] == 128
assert src.overviews(1) == [2]
with pytest.warns(IncompatibleBlockRasterSize):
cog_translate(raster_path_toosmall,
"cogeo.tif",
default_profile,
quiet=True)
assert cog_validate("cogeo.tif")
with rasterio.open("cogeo.tif") as src:
assert src.height == 51
assert src.width == 51
assert not src.is_tiled
assert not src.profile.get("blockxsize")
assert not src.profile.get("blockysize")
assert not src.overviews(1)
def convert_to_cog(raster, out_path=None, validate=True, **kwargs):
output_profile = cog_profiles.get("deflate")
if out_path is None:
out_path = str(raster.with_suffix(".tif")).replace(" ", "_")
assert raster != out_path, "Can't convert to files of the same name"
cog_translate(raster, out_path, output_profile, quiet=True, **kwargs)
if validate:
cog_validate(out_path)
return pathlib.Path(out_path)
def test_cog_validate_return():
valid, err, warn = cog_validate(raster_rgb)
assert valid
assert not err
assert not warn
valid, err, warn = cog_validate(raster_no_ovr)
assert valid
assert len(warn) == 1
assert not err
def test_cog_validate_return():
"""Checkout returned values."""
valid, err, warn = cog_validate(
raster_rgb, config={"GDAL_DISABLE_READDIR_ON_OPEN": "FALSE"})
assert valid
assert not err
assert not warn
valid, err, warn = cog_validate(
raster_no_ovr, config={"GDAL_DISABLE_READDIR_ON_OPEN": "FALSE"})
assert valid
assert len(warn) == 1
assert not err
def test_cog_validate_config(monkeypatch):
"""Should work as expected (validate cogeo file)."""
# No external overview found
assert cog_validate(raster_external,
config={"GDAL_DISABLE_READDIR_ON_OPEN":
"EMPTY_DIR"})[0]
# External overview found
assert not cog_validate(
raster_external, config={"GDAL_DISABLE_READDIR_ON_OPEN": "FALSE"})[0]
assert ("external"
in cog_validate(raster_external,
config={"GDAL_DISABLE_READDIR_ON_OPEN":
"FALSE"})[1][0])
def test_overview_valid():
"""Should work as expected (create cogeo file)."""
config = {
"GDAL_DISABLE_READDIR_ON_OPEN": "TRUE",
"GDAL_TIFF_INTERNAL_MASK": "TRUE",
"GDAL_TIFF_OVR_BLOCKSIZE": "128",
}
runner = CliRunner()
with runner.isolated_filesystem():
with open("mosaic.json", "w") as f:
f.write(json.dumps(mosaic_content))
create_overview_cogs("mosaic.json",
deflate_profile,
config=config,
threads=1)
assert cog_validate("mosaic_ovr_0.tif")
with rasterio.open("mosaic_ovr_0.tif") as src:
assert src.height == 512
assert src.width == 512
assert src.meta["dtype"] == "uint16"
assert src.is_tiled
assert src.profile["blockxsize"] == 256
assert src.profile["blockysize"] == 256
assert src.compression.value == "DEFLATE"
assert src.interleaving.value == "PIXEL"
assert src.overviews(1) == [2]
with rasterio.open("mosaic_ovr_0.tif", OVERVIEW_LEVEL=0) as src:
assert src.block_shapes[0] == (128, 128)
def mem_cog(raster, meta, filename):
config = dict(
GDAL_NUM_THREADS="ALL_CPUS",
GDAL_TIFF_INTERNAL_MASK=False,
GDAL_TIFF_OVR_BLOCKSIZE="128",
)
with MemoryFile() as memfile:
with memfile.open(**meta) as mem:
# Populate the input file with numpy array
mem.write(raster)
dst_profile = cog_profiles.get("deflate")
dst_profile.update(dict(BIGTIFF="IF_SAFER"))
cog_translate(mem,
filename,
dst_profile,
nodata=0,
add_mask=False,
config=config,
in_memory=True,
quiet=True)
if cog_validate(filename):
print('COGs created and validated')
with open('cog_list_' + time_frame + '.txt', 'a') as f:
print(f"{filename}", file=f)
def test_output_single_gtiff_cog_s3(output_single_gtiff_cog, mp_s3_tmpdir):
tile_id = (5, 3, 7)
with mapchete.open(
dict(output_single_gtiff_cog.dict,
output=dict(output_single_gtiff_cog.dict["output"],
path=os.path.join(mp_s3_tmpdir,
"cog.tif")))) as mp:
process_tile = mp.config.process_pyramid.tile(*tile_id)
# basic functions
assert mp.config.output.profile()
assert mp.config.output.empty(process_tile).mask.all()
assert mp.config.output.get_path(process_tile)
# check if tile exists
assert not mp.config.output.tiles_exist(process_tile)
# write
mp.batch_process(multi=2)
# check if tile exists
assert mp.config.output.tiles_exist(process_tile)
# read again, this time with data
data = mp.config.output.read(process_tile)
assert isinstance(data, np.ndarray)
assert not data[0].mask.all()
# write empty array
data = ma.masked_array(
data=np.ones(process_tile.shape),
mask=np.ones(process_tile.shape),
)
mp.config.output.write(process_tile, data)
assert path_exists(mp.config.output.path)
assert cog_validate(mp.config.output.path, strict=True)
def test_aoiclipped_fetcher_layers_cog(modis_instance):
"""
Unmocked ("live") test for fetching data. Tests cog conversion with image, with 7 bands.
"""
query = STACQuery.from_dict({
"zoom_level":
9,
"time":
"2019-01-01T16:40:49+00:00/2021-02-15T23:59:59+00:00",
"limit":
1,
"bbox": [
38.941807150840766,
21.288749561718983,
39.686130881309516,
21.808610762909364,
],
"imagery_layers": [
"MODIS_Terra_CorrectedReflectance_TrueColor",
"MODIS_Terra_EVI_8Day",
"MODIS_Terra_CorrectedReflectance_Bands721",
],
})
result = modis_instance.fetch(query, dry_run=False)
assert len(result.features) == 1
img_filename = "/tmp/output/%s" % result.features[0]["properties"][
"up42.data_path"]
with rio.open(img_filename) as dataset:
band2 = dataset.read(2)
assert np.sum(band2) == 28202042
assert dataset.count == 7
assert cog_validate(img_filename)[0]
def test_aoiclipped_fetcher_virs_fetch_live(modis_instance):
"""
Unmocked ("live") test for fetching VIIRS data in png
"""
query = STACQuery.from_dict({
"zoom_level":
9,
"time":
"2019-01-01T16:40:49+00:00/2019-01-25T16:41:49+00:00",
"limit":
2,
"bbox": [
38.941807150840766,
21.288749561718983,
39.686130881309516,
21.808610762909364,
],
"imagery_layers": ["VIIRS_SNPP_Brightness_Temp_BandI5_Night"],
})
result = modis_instance.fetch(query, dry_run=False)
assert len(result.features) == 2
img_filename = "/tmp/output/%s" % result.features[0]["properties"][
"up42.data_path"]
with rio.open(img_filename) as dataset:
band1 = dataset.read(1)
assert np.sum(band1) == 45232508
assert dataset.count == 1
assert os.path.isfile("/tmp/quicklooks/%s.jpg" % result.features[0]["id"])
assert cog_validate(img_filename)[0]
def valid_cogeo(src_path):
"""
Validate a Cloud Optimized GeoTIFF
:param src_path: path to GeoTIFF
:return: true if the GeoTIFF is a cogeo, false otherwise
"""
return cog_validate(src_path, strict=True)
def viz(src_paths, style, port, mapbox_token, no_check):
"""Rasterio Viz cli."""
# Check if cog
src_paths = list(src_paths)
with ExitStack() as ctx:
for ii, src_path in enumerate(src_paths):
if not no_check and not cog_validate(src_path):
# create tmp COG
click.echo("create temporaty COG")
tmp_path = ctx.enter_context(TemporaryRasterFile(src_path))
output_profile = cog_profiles.get("deflate")
output_profile.update(dict(blockxsize="256", blockysize="256"))
config = dict(
GDAL_TIFF_INTERNAL_MASK=os.environ.get(
"GDAL_TIFF_INTERNAL_MASK", True
),
GDAL_TIFF_OVR_BLOCKSIZE="128",
)
cog_translate(src_path, tmp_path.name, output_profile, config=config)
src_paths[ii] = tmp_path.name
src_dst = raster.RasterTiles(src_paths)
application = app.viz(src_dst, token=mapbox_token, port=port)
url = application.get_template_url()
click.echo(f"Viewer started at {url}", err=True)
click.launch(url)
application.start()
def validate(input, strict):
"""Validate Cloud Optimized Geotiff."""
is_valid, _, _ = cog_validate(input, strict=strict)
if is_valid:
click.echo("{} is a valid cloud optimized GeoTIFF".format(input))
else:
click.echo("{} is NOT a valid cloud optimized GeoTIFF".format(input))
def test_aoiclipped_fetcher_fetch(requests_mock, modis_instance):
"""
Mocked test for fetching data - quicker than the live one and therefore valuable for testing
purposes
"""
_location_ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
with open(os.path.join(_location_, "mock_data/tile.jpg"),
"rb") as tile_file:
mock_image: object = tile_file.read()
with open(
os.path.join(_location_, "mock_data/available_imagery_layers.xml"),
"rb") as xml_file:
mock_xml: object = xml_file.read()
matcher_wms = re.compile(
"https://gibs.earthdata.nasa.gov/wms/epsg4326/best/wms.cgi?")
matcher_wmts = re.compile(
"https://gibs.earthdata.nasa.gov/wmts/epsg3857/"
"best/MODIS_Terra_CorrectedReflectance_TrueColor/")
matcher_get_capabilities = re.compile("WMTSCapabilities.xml")
requests_mock.get(matcher_get_capabilities, content=mock_xml)
requests_mock.get(matcher_wms, content=mock_image)
requests_mock.get(matcher_wmts, content=mock_image)
query = STACQuery.from_dict({
"zoom_level":
9,
"time":
"2018-11-01T16:40:49+00:00/2018-11-20T16:41:49+00:00",
"limit":
1,
"bbox": [
123.59349578619005,
-10.188159969024264,
123.70257586240771,
-10.113232998848046,
],
"imagery_layers": ["MODIS_Terra_CorrectedReflectance_TrueColor"],
})
result = modis_instance.fetch(query, dry_run=False)
assert len(result.features) == 1
img_filename = "/tmp/output/%s" % result.features[0]["properties"][
"up42.data_path"]
assert cog_validate(img_filename)[0]
with rio.open(img_filename) as dataset:
band2 = dataset.read(2)
assert np.sum(band2) == 7954025
assert dataset.tags(
1)["layer"] == "MODIS_Terra_CorrectedReflectance_TrueColor"
assert dataset.tags(1)["band"] == str(1)
assert dataset.tags(2)["band"] == str(2)
assert os.path.isfile("/tmp/quicklooks/%s.jpg" % result.features[0]["id"])
def process(
url: str,
out_bucket: str,
out_key: str,
profile: str = "webp",
profile_options: Dict = {},
allow_remote_read: bool = False,
copy_valid_cog: bool = False,
**options: Any,
) -> None:
"""Download, convert and upload."""
url_info = urlparse(url.strip())
if url_info.scheme not in ["http", "https", "s3"]:
raise Exception(f"Unsuported scheme {url_info.scheme}")
if allow_remote_read:
src_path = url
else:
src_path = "/tmp/" + os.path.basename(url_info.path)
if url_info.scheme.startswith("http"):
wget.download(url, src_path)
elif url_info.scheme == "s3":
in_bucket = url_info.netloc
in_key = url_info.path.strip("/")
_s3_download(in_bucket, in_key, src_path)
if copy_valid_cog and cog_validate(src_path):
with open(src_path, "rb") as f:
_upload_obj(f, out_bucket, out_key)
else:
config = dict(
GDAL_NUM_THREADS="ALL_CPUS",
GDAL_TIFF_INTERNAL_MASK=True,
GDAL_TIFF_OVR_BLOCKSIZE="128",
)
output_profile = cog_profiles.get(profile)
output_profile.update(dict(BIGTIFF="IF_SAFER"))
output_profile.update(profile_options)
with MemoryFile() as mem_dst:
cog_translate(
src_path,
mem_dst.name,
output_profile,
config=config,
in_memory=False, # Limit Memory usage
quiet=True,
allow_intermediate_compression=True, # Limit Disk usage
**options,
)
_upload_obj(mem_dst, out_bucket, out_key)
del mem_dst
if not allow_remote_read:
os.remove(src_path)
return
def test_gdal_cog(src_path, runner):
"""Test GDAL COG."""
with runner.isolated_filesystem():
cog_translate(
src_path,
"cogeo.tif",
cog_profiles.get("raw"),
quiet=True,
use_cog_driver=True,
)
assert cog_validate("cogeo.tif")
def convert_to_cog_single(filename: str, validate: bool = True) -> str:
"""Convert a GeoTIFF to COG
Args:
filename (str): filename.
output_profile (dict): Dictionary with COG arguments
validate (bool, optional): Validate COG before to print the filename. Defaults to True.
Raises:
Exception: Filename needs to be a GeoTIFF file!
Returns:
str: The filename with the image converted to COG.
"""
tiff_inspector = re.compile('\\.tif$|\\.tiff$')
output_profile = cog_profiles["lzw"]
if not tiff_inspector.search(filename):
raise Exception("Filename needs to be a GeoTIFF file.")
cog_translate(filename, filename, output_profile, quiet=True)
if validate:
cog_validate(filename)
return filename
def main(input, output_dir, product, cogeo_profile, blocksize,
creation_options):
"""Translate a file to a COG."""
assert input.endswith(".hdf")
if not os.path.exists(output_dir):
os.makedirs(output_dir)
output_profile = cog_profiles.get(cogeo_profile)
output_profile.update(
dict(blockxsize=str(blocksize),
blockysize=str(blocksize),
predictor="2"))
if creation_options:
output_profile.update(creation_options)
config = dict(GDAL_NUM_THREADS="ALL_CPUS", GDAL_TIFF_OVR_BLOCKSIZE="128")
if product == "S30":
band_names = S30_BAND_NAMES
bname = os.path.splitext(os.path.basename(input))[0]
if product == "L30":
band_names = L30_BAND_NAMES
bname = os.path.splitext(os.path.basename(input))[0]
if product == "S30_ANGLES" or product == "L30_ANGLES":
band_names = ANGLE_BAND_NAMES
name = os.path.splitext(os.path.basename(input))[0]
# Remove ANGLE suffix from basename
bname = name.rsplit(".", 1)[0]
with rasterio.open(input) as src_dst:
for sds in src_dst.subdatasets:
band = sds.split(":")[-1]
if band in band_names:
try:
fname = "{}.{}.tif".format(bname, band_names[band])
except TypeError:
fname = "{}.{}.tif".format(bname, band)
output_name = os.path.join(output_dir, fname)
with rasterio.open(sds) as sub_dst:
cog_translate(
sub_dst,
output_name,
output_profile,
config=config,
forward_band_tags=True,
overview_resampling="nearest",
quiet=True,
)
assert cog_validate(output_name)
def test_convert_single_gtiff_cog(cleantopo_br_tif, mp_tmpdir):
"""Automatic geodetic tile pyramid creation of raster files."""
single_gtiff = os.path.join(mp_tmpdir, "single_out_cog.tif")
run_cli([
"convert", cleantopo_br_tif, single_gtiff, "--output-pyramid",
"geodetic", "-z", "3", "--cog"
])
with rasterio.open(single_gtiff, "r") as src:
assert src.meta["driver"] == "GTiff"
assert src.meta["dtype"] == "uint16"
data = src.read(masked=True)
assert data.mask.any()
assert cog_validate(single_gtiff, strict=True)
def process(
url: str,
out_bucket: str,
out_key: str,
profile: str = "webp",
profile_options: Dict = {},
allow_remote_read: bool = False,
copy_valid_cog: bool = False,
config: Dict[str, Any] = {},
**options: Any,
) -> bool:
"""Download, convert and upload."""
url_info = urlparse(url.strip())
if url_info.scheme not in ["http", "https", "s3"]:
raise Exception(f"Unsuported scheme {url_info.scheme}")
if copy_valid_cog and cog_validate(url):
_upload_obj(_get(url), out_bucket, out_key)
return True
src_path: Union[str, BytesIO] = url if allow_remote_read else _get(url)
config.update(
{
"GDAL_NUM_THREADS": "ALL_CPUS",
"GDAL_TIFF_INTERNAL_MASK": "TRUE",
"GDAL_TIFF_OVR_BLOCKSIZE": "128",
}
)
output_profile = cog_profiles.get(profile)
output_profile.update(dict(BIGTIFF="IF_SAFER"))
output_profile.update(profile_options)
with rasterio.open(src_path) as src:
with MemoryFile() as mem_dst:
cog_translate(
src,
mem_dst.name,
output_profile,
config=config,
in_memory=True,
quiet=True,
allow_intermediate_compression=True, # Limit Disk usage
**options,
)
_upload_obj(mem_dst, out_bucket, out_key)
del mem_dst, src_path, src
return True
def test_gdal_cog_web_mask(runner):
"""Raise a warning for specific mask/compression/web combination."""
with runner.isolated_filesystem():
with pytest.warns(UserWarning):
cog_translate(
raster_path_rgb,
"cogeo.tif",
cog_profiles.get("deflate"),
use_cog_driver=True,
web_optimized=True,
add_mask=True,
quiet=True,
)
assert cog_validate("cogeo.tif")
def valid_cogeo(src_path):
"""
Validate a Cloud Optimized GeoTIFF
:param src_path: path to GeoTIFF
:return: true if the GeoTIFF is a cogeo, false otherwise
"""
try:
from app.vendor.validate_cloud_optimized_geotiff import validate
warnings, errors, details = validate(src_path, full_check=True)
return not errors and not warnings
except ModuleNotFoundError:
logger.warning(
"Using legacy cog_validate (osgeo.gdal package not found)")
# Legacy
return cog_validate(src_path, strict=True)
def test_cog_validate_validCreatioValid(monkeypatch):
"""Should work as expected (validate cogeo file)."""
runner = CliRunner()
with runner.isolated_filesystem():
cog_translate(raster_rgb, "cogeo.tif", jpeg_profile, quiet=True)
assert cog_validate("cogeo.tif")
cog_translate(raster_rgb,
"cogeo.tif",
jpeg_profile,
overview_level=0,
quiet=True)
assert cog_validate("cogeo.tif")
config = dict(GDAL_TIFF_OVR_BLOCKSIZE="1024")
cog_translate(
raster_big,
"cogeo.tif",
jpeg_profile,
overview_level=1,
config=config,
quiet=True,
)
assert not cog_validate("cogeo.tif")
def test_cog_validate_valid(monkeypatch):
"""Should work as expected (validate cogeo file)."""
# not tiled but 512x512
assert cog_validate(raster_rgb)
# not tiled, no overview
assert not cog_validate(raster_big)
# external overview
assert not cog_validate(raster_external)
# non-sorted overview
assert not cog_validate(raster_ovrsorted)
# invalid decimation
assert not cog_validate(raster_decim)
# no overview
assert cog_validate(raster_no_ovr)
assert not cog_validate(raster_no_ovr, strict=True)
with pytest.raises(Exception):
cog_validate(raster_jpeg)
def process(
url,
out_bucket,
out_key,
profile="webp",
profile_options={},
copy_valid_cog=False,
**options,
):
"""Download, convert and upload."""
url_info = urlparse(url.strip())
src_path = "/tmp/" + os.path.basename(url_info.path)
if url_info.scheme not in ["http", "https", "s3"]:
raise Exception(f"Unsuported scheme {url_info.scheme}")
if url_info.scheme.startswith("http"):
wget.download(url, src_path)
elif url_info.scheme == "s3":
_s3_download(url, src_path)
uid = str(uuid.uuid4())
dst_path = f"/tmp/{uid}.tif"
if copy_valid_cog and cog_validate(src_path):
dst_path = src_path
else:
_translate(
src_path,
dst_path,
profile=profile,
profile_options=profile_options,
**options,
)
_upload(dst_path, out_bucket, out_key)
return True
def convert_to_cog(tif_path, local_dir, include_tiling_scheme: bool = True):
is_valid_cog, _, _ = cog_validate(tif_path, strict=True)
if is_valid_cog is True:
logger.info("Skipping conversion of %s to a cog", tif_path)
return tif_path
logger.info("Converting %s to a cog", tif_path)
with rasterio.open(tif_path) as src:
has_64_bit = rasterio.dtypes.float64 in src.dtypes
out_path = os.path.join(local_dir, "cog.tif")
cog_command = [
"gdal_translate",
tif_path,
*(["-co", "TILING_SCHEME=GoogleMapsCompatible"]
if include_tiling_scheme else []),
"-co",
"COMPRESS=DEFLATE",
"-co",
"BIGTIFF=IF_SAFER",
*(["-co", "PREDICTOR=2"] if not has_64_bit else []),
"-of",
"COG",
out_path,
]
try:
subprocess.check_output(cog_command, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
# This is fragile -- if the error message changes, we'll fail to detect the
# recoverable error. However, that can't happen unless we change the container, so we're
# Probably Fine™ for a while.
if b"ERROR 1: Could not find an appropriate zoom level" in e.output:
logger.warn(
"Couldn't process the tif with default command. Retrying without TILING_SCHEME=GoogleMapsCompatible"
)
return convert_to_cog(tif_path, local_dir, False)
else:
raise
return out_path
def test_aoiclipped_fetcher_multiple_fetch_live(modis_instance):
"""
Unmocked ("live") test for fetching data, multiple imagery_layers
"""
query = STACQuery.from_dict({
"zoom_level":
9,
"time":
"2019-01-01T16:40:49+00:00/2019-01-25T16:41:49+00:00",
"limit":
2,
"bbox": [
38.941807150840766,
21.288749561718983,
39.686130881309516,
21.808610762909364,
],
"imagery_layers": [
"MODIS_Terra_CorrectedReflectance_TrueColor",
"MODIS_Aqua_CorrectedReflectance_TrueColor",
],
})
result = modis_instance.fetch(query, dry_run=False)
assert len(result.features) == 2
img_filename = "/tmp/output/%s" % result.features[0]["properties"][
"up42.data_path"]
with rio.open(img_filename) as dataset:
band2 = dataset.read(2)
assert np.sum(band2) == 28351388
assert dataset.count == 6
assert os.path.isfile("/tmp/quicklooks/%s.jpg" % result.features[0]["id"])
assert cog_validate(img_filename)[0]
def validate(input):
"""Validate Cloud Optimized Geotiff."""
if cog_validate(input):
click.echo("{} is a valid cloud optimized GeoTIFF".format(input))
else:
click.echo("{} is NOT a valid cloud optimized GeoTIFF".format(input))