def test_update_mosaic():
"""Create mosaic and update it."""
mosaic = utils.create_mosaic([asset1], minzoom=9)
assert len(mosaic["tiles"]) == 36
mosaic = utils.create_mosaic([asset1], minzoom=9)
assert mosaic["version"] == "1.0.0"
utils.update_mosaic([asset2], mosaic)
assert len(mosaic["tiles"]) == 48
assert len(mosaic["tiles"]["030230132"]) == 2
assert mosaic["version"] == "1.0.1"
mosaic = utils.create_mosaic([asset1], minzoom=9)
utils.update_mosaic([asset2], mosaic, minimum_tile_cover=0.1)
assert len(mosaic["tiles"]) == 47
assert len(mosaic["tiles"]["030230132"]) == 1
def test_update_valid():
"""Should work as expected."""
runner = CliRunner()
with runner.isolated_filesystem():
with open("mosaic.json", "w") as f:
f.write(json.dumps(create_mosaic([asset1])))
with open("./list.txt", "w") as f:
f.write("\n".join([asset2]))
result = runner.invoke(cogeo_cli, ["update", "list.txt", "mosaic.json"])
assert not result.exception
assert result.exit_code == 0
updated_mosaic = json.loads(result.output)
updated_mosaic["version"] == "1.0.1"
assert mosaic_content["tiles"] == updated_mosaic["tiles"]
result = runner.invoke(
cogeo_cli, ["update", "list.txt", "mosaic.json", "-o", "mosaicU.json"]
)
assert not result.exception
assert result.exit_code == 0
with open("mosaicU.json", "r") as f:
updated_mosaic = json.load(f)
updated_mosaic["version"] == "1.0.1"
assert mosaic_content["tiles"] == updated_mosaic["tiles"]
def create(
input_files,
output,
minzoom,
maxzoom,
min_tile_cover,
tile_cover_sort,
threads,
quiet,
):
"""Create mosaic definition file."""
input_files = input_files.read().splitlines()
mosaic_spec = create_mosaic(
input_files,
minzoom=minzoom,
maxzoom=maxzoom,
minimum_tile_cover=min_tile_cover,
tile_cover_sort=tile_cover_sort,
max_threads=threads,
quiet=quiet,
)
if output:
with open(output, mode="w") as f:
f.write(json.dumps(mosaic_spec))
else:
click.echo(json.dumps(mosaic_spec))
def reduce(message):
"""Reduce Step: Create Mosaic."""
if isinstance(message, str):
message = json.loads(message)
mosaicid = message["metadata"]["mosaicid"]
jobid = message["jobid"]
logger.info(f"[REDUCE] Starting reduce step for job {jobid}")
bucket = os.environ["MOSAIC_BUCKET"]
list_cog = _aws_list_objects(bucket, f"cogs/{mosaicid}")
mosaic_definition = create_mosaic(list_cog)
key = f"mosaics/{mosaicid}.json.gz"
_aws_put_data(key, bucket, _compress_gz_json(mosaic_definition))
return True
import os
import json
from click.testing import CliRunner
from cogeo_mosaic.scripts.cli import cogeo_cli
from cogeo_mosaic.utils import create_mosaic
import rasterio
asset1 = os.path.join(os.path.dirname(__file__), "fixtures", "cog1.tif")
asset2 = os.path.join(os.path.dirname(__file__), "fixtures", "cog2.tif")
assets = [asset1, asset2]
mosaic_content = create_mosaic(assets)
def test_create_valid():
"""Should work as expected."""
runner = CliRunner()
with runner.isolated_filesystem():
with open("./list.txt", "w") as f:
f.write("\n".join(assets))
result = runner.invoke(cogeo_cli, ["create", "list.txt", "--quiet"])
assert not result.exception
assert result.exit_code == 0
assert mosaic_content == json.loads(result.output)
result = runner.invoke(cogeo_cli, ["create", "list.txt", "-o", "mosaic.json"])
def test_mosaic_create():
"""Fetch info from dataset and create the mosaicJSON definition."""
assets = [asset1, asset2]
mosaic = utils.create_mosaic(assets, quiet=False)
assert [round(b, 3) for b in mosaic["bounds"]
] == [round(b, 3) for b in mosaic_content["bounds"]]
assert mosaic["maxzoom"] == mosaic_content["maxzoom"]
assert mosaic["minzoom"] == mosaic_content["minzoom"]
assert list(mosaic["tiles"].keys()) == list(mosaic_content["tiles"].keys())
mosaic = utils.create_mosaic(assets, minzoom=7, maxzoom=9)
assert [round(b, 3) for b in mosaic["bounds"]
] == [round(b, 3) for b in mosaic_content["bounds"]]
assert mosaic["maxzoom"] == mosaic_content["maxzoom"]
assert mosaic["minzoom"] == mosaic_content["minzoom"]
assert list(mosaic["tiles"].keys()) == list(mosaic_content["tiles"].keys())
# 5% tile cover filter
mosaic = utils.create_mosaic(assets, minimum_tile_cover=0.05)
assert not list(mosaic["tiles"].keys()) == list(
mosaic_content["tiles"].keys())
# sort by tile cover
mosaic = utils.create_mosaic(assets, tile_cover_sort=True)
assert list(mosaic["tiles"].keys()) == list(mosaic_content["tiles"].keys())
assert not mosaic["tiles"] == mosaic_content["tiles"]
assets = [asset1, asset2]
mosaic = utils.create_mosaic(assets)
assert [round(b, 3) for b in mosaic["bounds"]
] == [round(b, 3) for b in mosaic_content["bounds"]]
assert mosaic["maxzoom"] == mosaic_content["maxzoom"]
# Wrong MosaicJSON version
with pytest.raises(Exception):
utils.create_mosaic(assets, version="0.0.3")
with pytest.warns(None) as record:
mosaic = utils.create_mosaic([asset1_small, asset2],
minzoom=7,
maxzoom=9)
assert not len(record)
# Multiple MaxZoom
with pytest.warns(UserWarning):
assets = [asset1_small, asset2]
utils.create_mosaic(assets)
# Mixed datatype
with pytest.raises(Exception):
asset1_uint32
assets = [asset1_uint32, asset2]
utils.create_mosaic(assets)
def _create(
body: str,
minzoom: Union[str, int] = None,
maxzoom: Union[str, int] = None,
min_tile_cover: Union[str, float] = None,
tile_cover_sort: Union[str, bool] = False,
tile_format: str = None,
tile_scale: Union[str, int] = 1,
**kwargs: Any,
) -> Tuple[str, str, str]:
minzoom = int(minzoom) if isinstance(minzoom, str) else minzoom
maxzoom = int(maxzoom) if isinstance(maxzoom, str) else maxzoom
min_tile_cover = (float(min_tile_cover) if isinstance(
min_tile_cover, float) else min_tile_cover)
mosaicid = get_hash(body=body, version=mosaic_version)
try:
mosaic_definition = fetch_mosaic_definition(_create_path(mosaicid))
except ClientError:
body = json.loads(body)
with rasterio.Env(aws_session):
mosaic_definition = create_mosaic(
body,
minzoom=minzoom,
maxzoom=maxzoom,
minimum_tile_cover=min_tile_cover,
tile_cover_sort=tile_cover_sort,
)
key = f"mosaics/{mosaicid}.json.gz"
bucket = os.environ["MOSAIC_DEF_BUCKET"]
_aws_put_data(key,
bucket,
_compress_gz_json(mosaic_definition),
client=s3_client)
if tile_format in ["pbf", "mvt"]:
tile_url = f"{app.host}/{mosaicid}/{{z}}/{{x}}/{{y}}.{tile_format}"
elif tile_format in ["png", "jpg", "webp", "tif", "npy"]:
tile_url = (
f"{app.host}/{mosaicid}/{{z}}/{{x}}/{{y}}@{tile_scale}x.{tile_format}"
)
else:
tile_url = f"{app.host}/{mosaicid}/{{z}}/{{x}}/{{y}}@{tile_scale}x"
qs = urllib.parse.urlencode(list(kwargs.items()))
if qs:
tile_url += f"?{qs}"
meta = {
"bounds": mosaic_definition["bounds"],
"center": mosaic_definition["center"],
"maxzoom": mosaic_definition["maxzoom"],
"minzoom": mosaic_definition["minzoom"],
"name": mosaicid,
"tilejson": "2.1.0",
"tiles": [tile_url],
}
return ("OK", "application/json", json.dumps(meta))
import re
import json
import base64
import urllib
import pytest
from mock import patch
from botocore.exceptions import ClientError
from cogeo_mosaic.utils import create_mosaic
from cogeo_mosaic import version
asset1 = os.path.join(os.path.dirname(__file__), "fixtures", "cog1.tif")
asset2 = os.path.join(os.path.dirname(__file__), "fixtures", "cog2.tif")
mosaic_content = create_mosaic([asset1, asset2])
request_json = os.path.join(os.path.dirname(__file__), "fixtures", "request.json")
@pytest.fixture(autouse=True)
def testing_env_var(monkeypatch):
"""Set fake env to make sure we don't hit AWS services."""
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "jqt")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "rde")
monkeypatch.setenv("AWS_DEFAULT_REGION", "us-west-2")
monkeypatch.setenv("AWS_REGION", "us-west-2")
monkeypatch.delenv("AWS_PROFILE", raising=False)
monkeypatch.setenv("AWS_CONFIG_FILE", "/tmp/noconfigheere")
monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", "/tmp/noconfighereeither")
monkeypatch.setenv("GDAL_DISABLE_READDIR_ON_OPEN", "EMPTY_DIR")
monkeypatch.setenv("MOSAIC_DEF_BUCKET", "my-bucket")