def test_update_valid():
"""Should work as expected."""
runner = CliRunner()
with runner.isolated_filesystem():
with open("mosaic.json", "w") as f:
f.write(
json.dumps(
MosaicJSON.from_urls([asset1]).dict(exclude_none=True)))
with open("./list.txt", "w") as f:
f.write("\n".join([asset2]))
result = runner.invoke(
cogeo_cli, ["update", "list.txt", "mosaic.json", "--quiet"])
assert not result.exception
assert result.exit_code == 0
with open("mosaic.json", "r") as f:
updated_mosaic = json.load(f)
updated_mosaic["version"] == "1.0.1"
assert not mosaic_content.tiles == updated_mosaic["tiles"]
with open("mosaic.json", "w") as f:
f.write(
json.dumps(
MosaicJSON.from_urls([asset1]).dict(exclude_none=True)))
result = runner.invoke(
cogeo_cli,
["update", "list.txt", "mosaic.json", "--add-last", "--quiet"])
assert not result.exception
assert result.exit_code == 0
with open("mosaic.json", "r") as f:
updated_mosaic = json.load(f)
updated_mosaic["version"] == "1.0.1"
assert mosaic_content.tiles == updated_mosaic["tiles"]
def test_read_mosaic(app):
"""test GET /mosaicjson endpoint"""
# TODO: Remove
response = app.get("/mosaicjson", params={"url": MOSAICJSON_FILE})
assert response.status_code == 200
MosaicJSON(**response.json())
response = app.get("/mosaicjson/", params={"url": MOSAICJSON_FILE})
assert response.status_code == 200
MosaicJSON(**response.json())
def __init__(
self,
path: str,
mosaic_def: Optional[Union[MosaicJSON, Dict]] = None,
**kwargs: Any,
):
"""Initialize FileBackend."""
self.path = path
if mosaic_def is not None:
self.mosaic_def = MosaicJSON(**dict(mosaic_def))
else:
self.mosaic_def = self._read(**kwargs)
def create(
input_files,
output,
minzoom,
maxzoom,
quadkey_zoom,
min_tile_cover,
tile_cover_sort,
threads,
quiet,
):
"""Create mosaic definition file."""
input_files = input_files.read().splitlines()
mosaicjson = MosaicJSON.from_urls(
input_files,
minzoom=minzoom,
maxzoom=maxzoom,
quadkey_zoom=quadkey_zoom,
minimum_tile_cover=min_tile_cover,
tile_cover_sort=tile_cover_sort,
max_threads=threads,
quiet=quiet,
)
if output:
with MosaicBackend(output, mosaic_def=mosaicjson) as mosaic:
mosaic.write()
else:
click.echo(json.dumps(mosaicjson.dict(exclude_none=True)))
def create_from_features(
features,
output,
minzoom,
maxzoom,
property,
quadkey_zoom,
min_tile_cover,
tile_cover_sort,
quiet,
):
"""Create mosaic definition file."""
mosaicjson = MosaicJSON.from_features(
list(features),
minzoom,
maxzoom,
quadkey_zoom=quadkey_zoom,
accessor=lambda feature: feature["properties"][property],
minimum_tile_cover=min_tile_cover,
tile_cover_sort=tile_cover_sort,
quiet=quiet,
)
if output:
with MosaicBackend(output, mosaic_def=mosaicjson) as mosaic:
mosaic.write()
else:
click.echo(json.dumps(mosaicjson.dict(exclude_none=True)))
def __init__(
self,
table_name: str,
mosaic_def: Optional[Union[MosaicJSON, Dict]] = None,
region: str = os.getenv("AWS_REGION", "us-east-1"),
client: Optional[Any] = None,
):
"""Initialize DynamoDBBackend."""
self.client = client or boto3.resource("dynamodb", region_name=region)
self.table = self.client.Table(table_name)
self.path = f"dynamodb://{region}/{table_name}"
if mosaic_def is not None:
self.mosaic_def = MosaicJSON(**dict(mosaic_def))
else:
self.mosaic_def = self._read()
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
body = _aws_get_data(self.key, self.bucket, client=self.client)
if gzip or (gzip is None and self.key.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def _read(self) -> MosaicJSON: # type: ignore
"""Get Mosaic definition info."""
meta = self._fetch_metadata()
if not meta:
raise MosaicNotFoundError(f"Mosaic not found in {self.path}")
meta["tiles"] = {}
return MosaicJSON(**meta)
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
body = requests.get(self.path).content
if gzip or (gzip is None and self.path.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def _read(*args: Any, **kwargs: Any) -> MosaicJSON:
"""Match signature of `cogeo_mosaic.backends.BaseBackend._read`"""
data = read_json_fixture(fname)
for qk in data["tiles"]:
data["tiles"][qk] = [
os.path.join(os.path.dirname(fname), f) for f in data["tiles"][qk]
]
return MosaicJSON(**data)
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
with open(self.path, "rb") as f:
body = f.read()
if gzip or (gzip is None and self.path.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def __init__(
self,
bucket: str,
key: str,
mosaic_def: Optional[Union[MosaicJSON, Dict]] = None,
client: Optional[boto3_session.client] = None,
**kwargs: Any,
):
"""Initialize S3Backend."""
self.client = client or boto3_session().client("s3")
self.key = key
self.bucket = bucket
self.path = f"s3://{bucket}/{key}"
if mosaic_def is not None:
self.mosaic_def = MosaicJSON(**dict(mosaic_def))
else:
self.mosaic_def = self._read(**kwargs)
def query(self, *args, **kwargs):
"""Mock Scan."""
mosaic = MosaicJSON(**mosaic_content)
return {
"Items": [{
"quadkey": qk,
"assets": assets
} for qk, assets in mosaic.tiles.items()]
}
def test_create_valid():
"""Should work as expected."""
runner = CliRunner()
with runner.isolated_filesystem():
with open("./list.txt", "w") as f:
f.write("\n".join(assets))
result = runner.invoke(cogeo_cli, ["create", "list.txt", "--quiet"])
assert not result.exception
assert result.exit_code == 0
assert mosaic_content == MosaicJSON(**json.loads(result.output))
result = runner.invoke(cogeo_cli,
["create", "list.txt", "-o", "mosaic.json"])
assert not result.exception
assert result.exit_code == 0
with open("mosaic.json", "r") as f:
assert mosaic_content == MosaicJSON(**json.load(f))
def _read(self) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
body = self._get_object(self.key, self.bucket)
self._file_byte_size = len(body)
if self.key.endswith(".gz"):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def update(
self,
features: Sequence[Dict],
add_first: bool = True,
quiet: bool = False,
**kwargs,
):
"""Update existing MosaicJSON on backend."""
logger.debug(f"Updating {self.mosaic_name}...")
new_mosaic = MosaicJSON.from_features(
features,
self.mosaic_def.minzoom,
self.mosaic_def.maxzoom,
quadkey_zoom=self.quadkey_zoom,
quiet=quiet,
**kwargs,
)
bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds)
self.mosaic_def._increase_version()
self.mosaic_def.bounds = bounds
self.mosaic_def.center = (
(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2,
self.mosaic_def.minzoom,
)
self.bounds = bounds
items: List[Dict[str, Any]] = []
# Create Metadata item
# Note: `parse_float=Decimal` is required because DynamoDB requires all numbers to be
# in Decimal type (ref: https://blog.ruanbekker.com/blog/2019/02/05/convert-float-to-decimal-data-types-for-boto3-dynamodb-using-python/)
meta = json.loads(self.mosaic_def.json(exclude={"tiles"}),
parse_float=Decimal)
items.append({
"quadkey": self._metadata_quadkey,
"mosaicId": self.mosaic_name,
**meta
})
# Create Tile items
for quadkey, new_assets in new_mosaic.tiles.items():
tile = mercantile.quadkey_to_tile(quadkey)
assets = self.assets_for_tile(*tile)
assets = [*new_assets, *assets
] if add_first else [*assets, *new_assets]
items.append({
"mosaicId": self.mosaic_name,
"quadkey": quadkey,
"assets": assets
})
self._write_items(items)
def mosaic(preference, check_exists, file):
features = load_features(file)
mosaic = MosaicJSON.from_features(features,
minzoom=11,
maxzoom=16,
asset_filter=asset_filter,
accessor=path_accessor,
check_exists=check_exists,
preference=preference)
print(json.dumps(mosaic.dict(), separators=(',', ':')))
def _read( # type: ignore
self,
query: Dict,
minzoom: int,
maxzoom: int,
accessor: Callable = default_stac_accessor,
max_items: Optional[int] = None,
stac_query_limit: int = 500,
stac_next_link_key: Optional[str] = None,
**kwargs: Any,
) -> MosaicJSON:
"""
Fetch STAC API and construct the mosaicjson.
Attributes
----------
query : Dict, required
STAC API POST request query.
minzoom: int, required
mosaic min-zoom.
maxzoom: int, required
mosaic max-zoom.
accessor: callable, required
Function called on each feature to get its identifier.
max_items: int, optional
Limit the maximum of items returned by the API
stac_query_limit: int, optional
Add "limit" option to the POST Query, default is set to 500.
stac_next_link_key: str, optional
link's 'next' key.
kwargs: any
Options forwarded to `MosaicJSON.from_features`
Returns
-------
mosaic_definition : MosaicJSON
Mosaic definition.
"""
logger.debug(f"Using STAC backend: {self.path}")
features = _fetch(
self.path,
query,
max_items=max_items,
limit=stac_query_limit,
next_link_key=stac_next_link_key,
)
logger.debug(f"Creating mosaic from {len(features)} features")
return MosaicJSON.from_features(
features, minzoom, maxzoom, accessor=accessor, **kwargs
)
def tmpmosaic():
"""Create a Temporary MosaicJSON file."""
fileobj = tempfile.NamedTemporaryFile(suffix=".json.gz", delete=False)
fileobj.close()
mosaic_def = MosaicJSON.from_urls(assets)
with FileBackend(fileobj.name, mosaic_def=mosaic_def) as mosaic:
mosaic.write(overwrite=True)
try:
yield fileobj.name
finally:
os.remove(fileobj.name)
def __attrs_post_init__(self):
"""Post Init."""
# Construct a FAKE mosaicJSON
# mosaic_def has to be defined. As we do for the DynamoDB and SQLite backend
# we set `tiles` to an empty list.
self.mosaic_def = MosaicJSON(
mosaicjson="0.0.2",
name="it's fake but it's ok",
minzoom=self.minzoom,
maxzoom=self.maxzoom,
tiles=[],
)
def test_create_valid():
"""Should work as expected."""
runner = CliRunner()
with runner.isolated_filesystem():
with open("./list.txt", "w") as f:
f.write("\n")
f.write("\n".join(assets))
f.write("\n")
result = runner.invoke(cogeo_cli, ["create", "list.txt", "--quiet"])
assert not result.exception
assert result.exit_code == 0
assert mosaic_content == MosaicJSON(**json.loads(result.output))
result = runner.invoke(cogeo_cli, ["create", "list.txt", "-o", "mosaic.json"])
assert not result.exception
assert result.exit_code == 0
assert mosaic_content == MosaicJSON.parse_file("mosaic.json")
result = runner.invoke(
cogeo_cli,
[
"create",
"list.txt",
"-o",
"mosaic.json",
"--name",
"my_mosaic",
"--description",
"A mosaic",
"--attribution",
"someone",
],
)
assert not result.exception
assert result.exit_code == 0
mosaic = MosaicJSON.parse_file("mosaic.json")
assert mosaic.name == "my_mosaic"
assert mosaic.description == "A mosaic"
assert mosaic.attribution == "someone"
class S3Backend(BaseBackend):
"""S3 Backend Adapter"""
def __init__(
self,
bucket: str,
key: str,
mosaic_def: Optional[Union[MosaicJSON, Dict]] = None,
client: Optional[boto3_session.client] = None,
**kwargs: Any,
):
"""Initialize S3Backend."""
self.client = client or boto3_session().client("s3")
self.key = key
self.bucket = bucket
self.path = f"s3://{bucket}/{key}"
if mosaic_def is not None:
self.mosaic_def = MosaicJSON(**dict(mosaic_def))
else:
self.mosaic_def = self._read(**kwargs)
def tile(self, x: int, y: int, z: int) -> List[str]:
"""Retrieve assets for tile."""
return get_assets_from_json(self.mosaic_def.tiles, self.quadkey_zoom, x, y, z)
def point(self, lng: float, lat: float) -> List[str]:
"""Retrieve assets for point."""
tile = mercantile.tile(lng, lat, self.quadkey_zoom)
return get_assets_from_json(
self.mosaic_def.tiles, self.quadkey_zoom, tile.x, tile.y, tile.z
)
def write(self, gzip: bool = None, **kwargs: Any):
"""Write mosaicjson document to AWS S3."""
mosaic_doc = self.mosaic_def.dict(exclude_none=True)
if gzip or (gzip is None and self.key.endswith(".gz")):
body = _compress_gz_json(mosaic_doc)
else:
body = json.dumps(mosaic_doc).encode("utf-8")
_aws_put_data(self.key, self.bucket, body, client=self.client, **kwargs)
@functools.lru_cache(maxsize=512)
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
body = _aws_get_data(self.key, self.bucket, client=self.client)
if gzip or (gzip is None and self.key.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def _add(body: str, url: str) -> Tuple:
mosaic_definition = MosaicJSON(**json.loads(body))
with MosaicBackend(url, mosaic_def=mosaic_definition) as mosaic:
mosaic.write()
return (
"OK",
"application/json",
json.dumps({
"id": url,
"status": "READY"
}, separators=(",", ":")),
)
def _add(body: str, mosaicid: str) -> Tuple:
if _aws_head_object(_create_mosaic_path(mosaicid), client=s3_client):
return ("NOK", "text/plain", f"Mosaic: {mosaicid} already exist.")
mosaic_definition = MosaicJSON(**json.loads(body))
with MosaicBackend(
_create_mosaic_path(mosaicid), mosaic_def=mosaic_definition
) as mosaic:
mosaic.write()
return (
"OK",
"application/json",
json.dumps({"id": mosaicid, "status": "READY"}, separators=(",", ":")),
)
def _read(self) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
try:
with open(self.path, "rb") as f:
body = f.read()
except Exception as e:
exc = _FILE_EXCEPTIONS.get(e, MosaicError) # type: ignore
raise exc(str(e)) from e
self._file_byte_size = len(body)
if self.path.endswith(".gz"):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
class FileBackend(BaseBackend):
"""Local File Backend Adapter"""
def __init__(
self,
path: str,
mosaic_def: Optional[Union[MosaicJSON, Dict]] = None,
**kwargs: Any,
):
"""Initialize FileBackend."""
self.path = path
if mosaic_def is not None:
self.mosaic_def = MosaicJSON(**dict(mosaic_def))
else:
self.mosaic_def = self._read(**kwargs)
def tile(self, x: int, y: int, z: int) -> List[str]:
"""Retrieve assets for tile."""
return get_assets_from_json(self.mosaic_def.tiles, self.quadkey_zoom, x, y, z)
def point(self, lng: float, lat: float) -> List[str]:
"""Retrieve assets for point."""
tile = mercantile.tile(lng, lat, self.quadkey_zoom)
return get_assets_from_json(
self.mosaic_def.tiles, self.quadkey_zoom, tile.x, tile.y, tile.z
)
def write(self, gzip: bool = None):
"""Write mosaicjson document to a file."""
body = self.mosaic_def.dict(exclude_none=True)
with open(self.path, "wb") as f:
if gzip or (gzip is None and self.path.endswith(".gz")):
f.write(_compress_gz_json(body))
else:
f.write(json.dumps(body).encode("utf-8"))
@functools.lru_cache(maxsize=512)
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
with open(self.path, "rb") as f:
body = f.read()
if gzip or (gzip is None and self.path.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def create_mosaicjson(body: CreateMosaicJSON):
"""Create a MosaicJSON"""
mosaic = MosaicJSON.from_urls(
body.files,
minzoom=body.minzoom,
maxzoom=body.maxzoom,
max_threads=body.max_threads,
)
mosaic_path = MosaicPath(body.url)
with MosaicBackend(mosaic_path, mosaic_def=mosaic) as mosaic:
try:
mosaic.write()
except NotImplementedError:
raise BadRequestError(
f"{mosaic.__class__.__name__} does not support write operations"
)
return mosaic.mosaic_def
def subset_mosaic(mosaic, overview_qk, overview_zoom):
"""Create subset of mosaic within a single overview quadkey
Args:
- overview_qk: zoom 6 quadkey
"""
qk_tiles = {
k: v
for k, v in mosaic['tiles'].items() if k[:overview_zoom] == overview_qk
}
bounds = mercantile.bounds(mercantile.quadkey_to_tile(overview_qk))
# The new mosaic needs to be the same minzoom, quadkey zoom as
new_mosaic = deepcopy(mosaic)
new_mosaic['tiles'] = qk_tiles
new_mosaic['bounds'] = bounds
return MosaicJSON(**new_mosaic)
def create_mosaic(urls=None, res=None):
allowed_res = ['1', '13', '1m', '2']
if res and res not in allowed_res:
raise ValueError(f'res must be in {allowed_res}')
urls = find_urls_for_res(res)
features = []
for url in urls:
key = url.split('/')[4]
geom = box(*parse_grid(key))
full_s3_path = f's3://{BUCKET}/{url}'
feature = Feature(geometry=geom, properties={'path': full_s3_path})
features.append(dict(feature))
minzoom, maxzoom = DEFAULT_ZOOMS[res]
mosaic = MosaicJSON.from_features(
features, minzoom, maxzoom, accessor=lambda x: x['properties']['path'])
def _read(self) -> MosaicJSON: # type: ignore
"""Get Mosaic definition info."""
meta = self._fetch_dynamodb("-1")
# Numeric values are loaded from DynamoDB as Decimal types
# Convert maxzoom, minzoom, quadkey_zoom to float/int
for key in ["minzoom", "maxzoom", "quadkey_zoom"]:
if meta.get(key):
meta[key] = int(meta[key])
# Convert bounds, center to float/int
for key in ["bounds", "center"]:
if meta.get(key):
meta[key] = list(map(float, meta[key]))
# Create pydantic class
# For now, a tiles key must exist
meta["tiles"] = {}
return MosaicJSON(**meta)
