def update(
self,
features: Sequence[Dict],
add_first: bool = True,
quiet: bool = False,
**kwargs,
):
"""Update existing MosaicJSON on backend."""
logger.debug(f"Updating {self.mosaic_name}...")
new_mosaic = MosaicJSON.from_features(
features,
self.mosaic_def.minzoom,
self.mosaic_def.maxzoom,
quadkey_zoom=self.quadkey_zoom,
quiet=quiet,
**kwargs,
)
bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds)
self.mosaic_def._increase_version()
self.mosaic_def.bounds = bounds
self.mosaic_def.center = (
(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2,
self.mosaic_def.minzoom,
)
self.bounds = bounds
items: List[Dict[str, Any]] = []
# Create Metadata item
# Note: `parse_float=Decimal` is required because DynamoDB requires all numbers to be
# in Decimal type (ref: https://blog.ruanbekker.com/blog/2019/02/05/convert-float-to-decimal-data-types-for-boto3-dynamodb-using-python/)
meta = json.loads(self.mosaic_def.json(exclude={"tiles"}),
parse_float=Decimal)
items.append({
"quadkey": self._metadata_quadkey,
"mosaicId": self.mosaic_name,
**meta
})
# Create Tile items
for quadkey, new_assets in new_mosaic.tiles.items():
tile = mercantile.quadkey_to_tile(quadkey)
assets = self.assets_for_tile(*tile)
assets = [*new_assets, *assets
] if add_first else [*assets, *new_assets]
items.append({
"mosaicId": self.mosaic_name,
"quadkey": quadkey,
"assets": assets
})
self._write_items(items)
def delete(self):
"""Delete a mosaic."""
logger.debug(
f"Deleting all items for '{self.mosaic_name}' mosaic in {self.db_path}..."
)
with self.db:
self.db.execute(
f"DELETE FROM {self._metadata_table} WHERE name=?;",
(self.mosaic_name, ))
self.db.execute(f'DROP TABLE IF EXISTS "{self.mosaic_name}";')
def delete(self):
"""Delete all items for a specific mosaic in the dynamoDB Table."""
logger.debug(f"Deleting all items for mosaic {self.mosaic_name}...")
quadkey_list = self._quadkeys + [self._metadata_quadkey]
with self.table.batch_writer() as batch_writer:
for item in quadkey_list:
batch_writer.delete_item(
Key={"mosaicId": self.mosaic_name, "quadkey": item}
)
def _read( # type: ignore
self,
query: Dict,
minzoom: int,
maxzoom: int,
accessor: Callable = default_stac_accessor,
max_items: Optional[int] = None,
stac_query_limit: int = 500,
stac_next_link_key: Optional[str] = None,
**kwargs: Any,
) -> MosaicJSON:
"""
Fetch STAC API and construct the mosaicjson.
Attributes
----------
query : Dict, required
STAC API POST request query.
minzoom: int, required
mosaic min-zoom.
maxzoom: int, required
mosaic max-zoom.
accessor: callable, required
Function called on each feature to get its identifier.
max_items: int, optional
Limit the maximum of items returned by the API
stac_query_limit: int, optional
Add "limit" option to the POST Query, default is set to 500.
stac_next_link_key: str, optional
link's 'next' key.
kwargs: any
Options forwarded to `MosaicJSON.from_features`
Returns
-------
mosaic_definition : MosaicJSON
Mosaic definition.
"""
logger.debug(f"Using STAC backend: {self.path}")
features = _fetch(
self.path,
query,
max_items=max_items,
limit=stac_query_limit,
next_link_key=stac_next_link_key,
)
logger.debug(f"Creating mosaic from {len(features)} features")
return MosaicJSON.from_features(
features, minzoom, maxzoom, accessor=accessor, **kwargs
)
def _create_table(self,
billing_mode: str = "PAY_PER_REQUEST",
**kwargs: Any):
"""Create DynamoDB Table.
Args:
billing_mode (str): DynamoDB billing mode (default set to PER_REQUEST).
**kwargs (any): Options forwarded to `dynamodb.create_table`
"""
logger.debug(f"Creating {self.table_name} Table.")
# Define schema for primary key
# Non-keys don't need a schema
attr_defs = [
{
"AttributeName": "mosaicId",
"AttributeType": "S"
},
{
"AttributeName": "quadkey",
"AttributeType": "S"
},
]
key_schema = [
{
"AttributeName": "mosaicId",
"KeyType": "HASH"
},
{
"AttributeName": "quadkey",
"KeyType": "RANGE"
},
]
# Note: errors if table already exists
try:
self.client.create_table(
AttributeDefinitions=attr_defs,
TableName=self.table.table_name,
KeySchema=key_schema,
BillingMode=billing_mode,
**kwargs,
)
# If outside try/except block, could wait forever if unable to
# create table
self.table.wait_until_exists()
except self.table.meta.client.exceptions.ResourceNotFoundException:
warnings.warn("Unable to create table.")
return
def update(
self,
features: Sequence[Dict],
add_first: bool = True,
quiet: bool = False,
**kwargs,
):
"""Update existing MosaicJSON on backend."""
logger.debug(f"Updating {self.mosaic_name}...")
new_mosaic = self.mosaic_def.from_features(
features,
self.mosaic_def.minzoom,
self.mosaic_def.maxzoom,
quadkey_zoom=self.quadkey_zoom,
quiet=quiet,
**kwargs,
)
fout = os.devnull if quiet else sys.stderr
with click.progressbar( # type: ignore
new_mosaic.tiles.items(),
file=fout,
show_percent=True,
label=f"Updating mosaic {self.table_name}:{self.mosaic_name}",
) as items:
for quadkey, new_assets in items:
tile = mercantile.quadkey_to_tile(quadkey)
assets = self.assets_for_tile(*tile)
assets = [*new_assets, *assets
] if add_first else [*assets, *new_assets]
# add custom sorting algorithm (e.g based on path name)
self._update_quadkey(quadkey, assets)
bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds)
self.mosaic_def._increase_version()
self.mosaic_def.bounds = bounds
self.mosaic_def.center = (
(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2,
self.mosaic_def.minzoom,
)
self._update_metadata()
def __attrs_post_init__(self):
"""Post Init: parse path, create client and connect to Table.
A path looks like
dynamodb://{region}/{table_name}:{mosaic_name}
dynamodb:///{table_name}:{mosaic_name}
"""
assert boto3 is not None, "'boto3' must be installed to use DynamoDBBackend"
logger.debug(f"Using DynamoDB backend: {self.path}")
if not re.match(
r"^dynamodb://([a-z]{2}\-[a-z]+\-[0-9])?\/[a-zA-Z0-9\_\-\.]+\:[a-zA-Z0-9\_\-\.]+$",
self.path,
):
raise ValueError(f"Invalid DynamoDB path: {self.path}")
parsed = urlparse(self.path)
mosaic_info = parsed.path.lstrip("/").split(":")
self.table_name = mosaic_info[0]
self.mosaic_name = mosaic_info[1]
logger.debug(f"Table: {self.table_name}")
logger.debug(f"Mosaic: {self.mosaic_name}")
self.region = parsed.netloc or self.region
self.client = self.client or boto3.resource("dynamodb",
region_name=self.region)
self.table = self.client.Table(self.table_name)
super().__attrs_post_init__()
def __attrs_post_init__(self):
"""Post Init: parse path connect to Table.
A path looks like
sqlite:///{db_path}:{mosaic_name}
"""
if not re.match(
r"^sqlite:///.+\:[a-zA-Z0-9\_\-\.]+$",
self.path,
):
raise ValueError(f"Invalid SQLite path: {self.path}")
parsed = urlparse(self.path)
uri_path = parsed.path[1:] # remove `/` on the left
self.mosaic_name = uri_path.split(":")[-1]
assert (not self.mosaic_name == self._metadata_table
), f"'{self._metadata_table}' is a reserved table name."
self.db_path = uri_path.replace(f":{self.mosaic_name}", "")
# When mosaic_def is not passed, we have to make sure the db exists
if not self.mosaic_def and not Path(self.db_path).exists():
raise MosaicNotFoundError(
f"SQLite database not found at path {self.db_path}.")
self.db = sqlite3.connect(self.db_path,
detect_types=sqlite3.PARSE_DECLTYPES)
self.db.row_factory = sqlite3.Row
# Here we make sure the mosaicJSON.name is the same
if self.mosaic_def and self.mosaic_def.name != self.mosaic_name:
warnings.warn("Updating 'mosaic.name' to match table name.")
self.mosaic_def.name = self.mosaic_name
logger.debug(f"Using SQLite backend: {self.db_path}")
super().__attrs_post_init__()
def _read(self) -> MosaicJSON:
"""
Fetch STAC API and construct the mosaicjson.
Returns:
MosaicJSON: Mosaic definition.
"""
logger.debug(f"Using STAC backend: {self.path}")
features = _fetch(
self.path,
self.query,
**self.stac_api_options,
)
logger.debug(f"Creating mosaic from {len(features)} features")
# We need a specific accessor for STAC
options = self.mosaic_options.copy()
if "accessor" not in options:
options["accessor"] = default_stac_accessor
return MosaicJSON.from_features(features, self.minzoom, self.maxzoom,
**options)
def _fetch(
stac_url: str,
query: Dict,
max_items: Optional[int] = None,
next_link_key: Optional[str] = None,
limit: int = 500,
) -> List[Dict]:
"""Call STAC API."""
features: List[Dict] = []
stac_query = query.copy()
headers = {
"Content-Type": "application/json",
"Accept-Encoding": "gzip",
"Accept": "application/geo+json",
}
if "limit" not in stac_query:
stac_query.update({"limit": limit})
def _stac_search(url: str, q: Dict):
try:
r = requests.post(url, headers=headers, json=q)
r.raise_for_status()
except requests.exceptions.HTTPError as e:
# post-flight errors
status_code = e.response.status_code
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response.content) from e
except requests.exceptions.RequestException as e:
# pre-flight errors
raise MosaicError(e.args[0].reason) from e
return r.json()
page = 1
while True:
logger.debug(f"Fetching page {page}")
logger.debug("query: " + json.dumps(stac_query))
results = _stac_search(stac_url, stac_query)
if not results.get("features"):
break
features.extend(results["features"])
if max_items and len(features) >= max_items:
features = features[:max_items]
break
# new STAC context spec
# {"page": 1, "limit": 1000, "matched": 5671, "returned": 1000}
# SAT-API META
# {"page": 4, "limit": 100, "found": 350, "returned": 50}
ctx = results.get("context", results.get("meta"))
matched = ctx.get("matched", ctx.get("found"))
logger.debug(json.dumps(ctx))
# Check if there is more data to fetch
if matched <= ctx["returned"]:
break
# We shouldn't fetch more item than matched
if len(features) == matched:
break
if len(features) > matched:
raise MosaicError(
"Something weird is going on, please open an issue in https://github.com/developmentseed/cogeo-mosaic"
)
page += 1
# https://github.com/radiantearth/stac-api-spec/blob/master/api-spec.md#paging-extension
if next_link_key:
links = list(
filter(lambda link: link["rel"] == next_link_key,
results["links"]))
if not links:
break
stac_query = query_from_link(links[0], stac_query)
else:
stac_query.update({"page": page})
return features
def update(
self,
features: Sequence[Dict],
add_first: bool = True,
quiet: bool = False,
**kwargs,
):
"""Update existing MosaicJSON on backend."""
logger.debug(f"Updating {self.mosaic_name}...")
new_mosaic = MosaicJSON.from_features(
features,
self.mosaic_def.minzoom,
self.mosaic_def.maxzoom,
quadkey_zoom=self.quadkey_zoom,
quiet=quiet,
**kwargs,
)
bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds)
self.mosaic_def._increase_version()
self.mosaic_def.bounds = bounds
self.mosaic_def.center = (
(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2,
self.mosaic_def.minzoom,
)
self.bounds = bounds
with self.db:
self.db.execute(
f"""
UPDATE {self._metadata_table}
SET mosaicjson = :mosaicjson,
name = :name,
description = :description,
version = :version,
attribution = :attribution,
minzoom = :minzoom,
maxzoom = :maxzoom,
quadkey_zoom = :quadkey_zoom,
bounds = :bounds,
center = :center
WHERE name=:name
""",
self.mosaic_def.dict(),
)
if add_first:
self.db.executemany(
f"""
UPDATE "{self.mosaic_name}"
SET assets = (
SELECT json_group_array(value)
FROM (
SELECT value FROM json_each(?)
UNION ALL
SELECT value FROM json_each(assets)
)
)
WHERE quadkey=?;
""",
[(assets, qk) for qk, assets in new_mosaic.tiles.items()],
)
else:
self.db.executemany(
f"""
UPDATE "{self.mosaic_name}"
SET assets = (
SELECT json_group_array(value)
FROM (
SELECT value FROM json_each(assets)
UNION ALL
SELECT value FROM json_each(?)
)
)
WHERE quadkey=?;
""",
[(assets, qk) for qk, assets in new_mosaic.tiles.items()],
)
def write(self, overwrite: bool = False):
"""Write mosaicjson document to an SQLite database.
Args:
overwrite (bool): delete old mosaic items in the Table.
Raises:
MosaicExistsError: If mosaic already exists in the Table.
"""
if self._mosaic_exists():
if not overwrite:
raise MosaicExistsError(
f"'{self.mosaic_name}' Table already exists in {self.db_path}, use `overwrite=True`."
)
self.delete()
with self.db:
logger.debug(
f"Creating '{self.mosaic_name}' Table in {self.db_path}.")
self.db.execute(f"""
CREATE TABLE IF NOT EXISTS {self._metadata_table}
(
mosaicjson TEXT NOT NULL,
name TEXT NOT NULL,
description TEXT,
version TEXT NOT NULL,
attribution TEXT,
minzoom INTEGER NOT NULL,
maxzoom INTEGER NOT NULL,
quadkey_zoom INTEGER,
bounds JSON NOT NULL,
center JSON
);
""")
self.db.execute(f"""
CREATE TABLE "{self.mosaic_name}"
(
quadkey TEXT NOT NULL,
assets JSON NOT NULL
);
""")
logger.debug(f"Adding items in '{self.mosaic_name}' Table.")
self.db.execute(
f"""
INSERT INTO {self._metadata_table}
(
mosaicjson,
name,
description,
version,
attribution,
minzoom,
maxzoom,
quadkey_zoom,
bounds,
center
)
VALUES
(
:mosaicjson,
:name,
:description,
:version,
:attribution,
:minzoom,
:maxzoom,
:quadkey_zoom,
:bounds,
:center
);
""",
self.mosaic_def.dict(),
)
self.db.executemany(
f'INSERT INTO "{self.mosaic_name}" (quadkey, assets) VALUES (?, ?);',
self.mosaic_def.tiles.items(),
)
