def update( self, features: Sequence[Dict], add_first: bool = True, quiet: bool = False, **kwargs, ): """Update existing MosaicJSON on backend.""" logger.debug(f"Updating {self.mosaic_name}...") new_mosaic = MosaicJSON.from_features( features, self.mosaic_def.minzoom, self.mosaic_def.maxzoom, quadkey_zoom=self.quadkey_zoom, quiet=quiet, **kwargs, ) bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds) self.mosaic_def._increase_version() self.mosaic_def.bounds = bounds self.mosaic_def.center = ( (bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, self.mosaic_def.minzoom, ) self.bounds = bounds items: List[Dict[str, Any]] = [] # Create Metadata item # Note: `parse_float=Decimal` is required because DynamoDB requires all numbers to be # in Decimal type (ref: https://blog.ruanbekker.com/blog/2019/02/05/convert-float-to-decimal-data-types-for-boto3-dynamodb-using-python/) meta = json.loads(self.mosaic_def.json(exclude={"tiles"}), parse_float=Decimal) items.append({ "quadkey": self._metadata_quadkey, "mosaicId": self.mosaic_name, **meta }) # Create Tile items for quadkey, new_assets in new_mosaic.tiles.items(): tile = mercantile.quadkey_to_tile(quadkey) assets = self.assets_for_tile(*tile) assets = [*new_assets, *assets ] if add_first else [*assets, *new_assets] items.append({ "mosaicId": self.mosaic_name, "quadkey": quadkey, "assets": assets }) self._write_items(items)
def delete(self): """Delete a mosaic.""" logger.debug( f"Deleting all items for '{self.mosaic_name}' mosaic in {self.db_path}..." ) with self.db: self.db.execute( f"DELETE FROM {self._metadata_table} WHERE name=?;", (self.mosaic_name, )) self.db.execute(f'DROP TABLE IF EXISTS "{self.mosaic_name}";')
def delete(self): """Delete all items for a specific mosaic in the dynamoDB Table.""" logger.debug(f"Deleting all items for mosaic {self.mosaic_name}...") quadkey_list = self._quadkeys + [self._metadata_quadkey] with self.table.batch_writer() as batch_writer: for item in quadkey_list: batch_writer.delete_item( Key={"mosaicId": self.mosaic_name, "quadkey": item} )
def _read( # type: ignore self, query: Dict, minzoom: int, maxzoom: int, accessor: Callable = default_stac_accessor, max_items: Optional[int] = None, stac_query_limit: int = 500, stac_next_link_key: Optional[str] = None, **kwargs: Any, ) -> MosaicJSON: """ Fetch STAC API and construct the mosaicjson. Attributes ---------- query : Dict, required STAC API POST request query. minzoom: int, required mosaic min-zoom. maxzoom: int, required mosaic max-zoom. accessor: callable, required Function called on each feature to get its identifier. max_items: int, optional Limit the maximum of items returned by the API stac_query_limit: int, optional Add "limit" option to the POST Query, default is set to 500. stac_next_link_key: str, optional link's 'next' key. kwargs: any Options forwarded to `MosaicJSON.from_features` Returns ------- mosaic_definition : MosaicJSON Mosaic definition. """ logger.debug(f"Using STAC backend: {self.path}") features = _fetch( self.path, query, max_items=max_items, limit=stac_query_limit, next_link_key=stac_next_link_key, ) logger.debug(f"Creating mosaic from {len(features)} features") return MosaicJSON.from_features( features, minzoom, maxzoom, accessor=accessor, **kwargs )
def _create_table(self, billing_mode: str = "PAY_PER_REQUEST", **kwargs: Any): """Create DynamoDB Table. Args: billing_mode (str): DynamoDB billing mode (default set to PER_REQUEST). **kwargs (any): Options forwarded to `dynamodb.create_table` """ logger.debug(f"Creating {self.table_name} Table.") # Define schema for primary key # Non-keys don't need a schema attr_defs = [ { "AttributeName": "mosaicId", "AttributeType": "S" }, { "AttributeName": "quadkey", "AttributeType": "S" }, ] key_schema = [ { "AttributeName": "mosaicId", "KeyType": "HASH" }, { "AttributeName": "quadkey", "KeyType": "RANGE" }, ] # Note: errors if table already exists try: self.client.create_table( AttributeDefinitions=attr_defs, TableName=self.table.table_name, KeySchema=key_schema, BillingMode=billing_mode, **kwargs, ) # If outside try/except block, could wait forever if unable to # create table self.table.wait_until_exists() except self.table.meta.client.exceptions.ResourceNotFoundException: warnings.warn("Unable to create table.") return
def update( self, features: Sequence[Dict], add_first: bool = True, quiet: bool = False, **kwargs, ): """Update existing MosaicJSON on backend.""" logger.debug(f"Updating {self.mosaic_name}...") new_mosaic = self.mosaic_def.from_features( features, self.mosaic_def.minzoom, self.mosaic_def.maxzoom, quadkey_zoom=self.quadkey_zoom, quiet=quiet, **kwargs, ) fout = os.devnull if quiet else sys.stderr with click.progressbar( # type: ignore new_mosaic.tiles.items(), file=fout, show_percent=True, label=f"Updating mosaic {self.table_name}:{self.mosaic_name}", ) as items: for quadkey, new_assets in items: tile = mercantile.quadkey_to_tile(quadkey) assets = self.assets_for_tile(*tile) assets = [*new_assets, *assets ] if add_first else [*assets, *new_assets] # add custom sorting algorithm (e.g based on path name) self._update_quadkey(quadkey, assets) bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds) self.mosaic_def._increase_version() self.mosaic_def.bounds = bounds self.mosaic_def.center = ( (bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, self.mosaic_def.minzoom, ) self._update_metadata()
def __attrs_post_init__(self): """Post Init: parse path, create client and connect to Table. A path looks like dynamodb://{region}/{table_name}:{mosaic_name} dynamodb:///{table_name}:{mosaic_name} """ assert boto3 is not None, "'boto3' must be installed to use DynamoDBBackend" logger.debug(f"Using DynamoDB backend: {self.path}") if not re.match( r"^dynamodb://([a-z]{2}\-[a-z]+\-[0-9])?\/[a-zA-Z0-9\_\-\.]+\:[a-zA-Z0-9\_\-\.]+$", self.path, ): raise ValueError(f"Invalid DynamoDB path: {self.path}") parsed = urlparse(self.path) mosaic_info = parsed.path.lstrip("/").split(":") self.table_name = mosaic_info[0] self.mosaic_name = mosaic_info[1] logger.debug(f"Table: {self.table_name}") logger.debug(f"Mosaic: {self.mosaic_name}") self.region = parsed.netloc or self.region self.client = self.client or boto3.resource("dynamodb", region_name=self.region) self.table = self.client.Table(self.table_name) super().__attrs_post_init__()
def __attrs_post_init__(self): """Post Init: parse path connect to Table. A path looks like sqlite:///{db_path}:{mosaic_name} """ if not re.match( r"^sqlite:///.+\:[a-zA-Z0-9\_\-\.]+$", self.path, ): raise ValueError(f"Invalid SQLite path: {self.path}") parsed = urlparse(self.path) uri_path = parsed.path[1:] # remove `/` on the left self.mosaic_name = uri_path.split(":")[-1] assert (not self.mosaic_name == self._metadata_table ), f"'{self._metadata_table}' is a reserved table name." self.db_path = uri_path.replace(f":{self.mosaic_name}", "") # When mosaic_def is not passed, we have to make sure the db exists if not self.mosaic_def and not Path(self.db_path).exists(): raise MosaicNotFoundError( f"SQLite database not found at path {self.db_path}.") self.db = sqlite3.connect(self.db_path, detect_types=sqlite3.PARSE_DECLTYPES) self.db.row_factory = sqlite3.Row # Here we make sure the mosaicJSON.name is the same if self.mosaic_def and self.mosaic_def.name != self.mosaic_name: warnings.warn("Updating 'mosaic.name' to match table name.") self.mosaic_def.name = self.mosaic_name logger.debug(f"Using SQLite backend: {self.db_path}") super().__attrs_post_init__()
def _read(self) -> MosaicJSON: """ Fetch STAC API and construct the mosaicjson. Returns: MosaicJSON: Mosaic definition. """ logger.debug(f"Using STAC backend: {self.path}") features = _fetch( self.path, self.query, **self.stac_api_options, ) logger.debug(f"Creating mosaic from {len(features)} features") # We need a specific accessor for STAC options = self.mosaic_options.copy() if "accessor" not in options: options["accessor"] = default_stac_accessor return MosaicJSON.from_features(features, self.minzoom, self.maxzoom, **options)
def _fetch( stac_url: str, query: Dict, max_items: Optional[int] = None, next_link_key: Optional[str] = None, limit: int = 500, ) -> List[Dict]: """Call STAC API.""" features: List[Dict] = [] stac_query = query.copy() headers = { "Content-Type": "application/json", "Accept-Encoding": "gzip", "Accept": "application/geo+json", } if "limit" not in stac_query: stac_query.update({"limit": limit}) def _stac_search(url: str, q: Dict): try: r = requests.post(url, headers=headers, json=q) r.raise_for_status() except requests.exceptions.HTTPError as e: # post-flight errors status_code = e.response.status_code exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError) raise exc(e.response.content) from e except requests.exceptions.RequestException as e: # pre-flight errors raise MosaicError(e.args[0].reason) from e return r.json() page = 1 while True: logger.debug(f"Fetching page {page}") logger.debug("query: " + json.dumps(stac_query)) results = _stac_search(stac_url, stac_query) if not results.get("features"): break features.extend(results["features"]) if max_items and len(features) >= max_items: features = features[:max_items] break # new STAC context spec # {"page": 1, "limit": 1000, "matched": 5671, "returned": 1000} # SAT-API META # {"page": 4, "limit": 100, "found": 350, "returned": 50} ctx = results.get("context", results.get("meta")) matched = ctx.get("matched", ctx.get("found")) logger.debug(json.dumps(ctx)) # Check if there is more data to fetch if matched <= ctx["returned"]: break # We shouldn't fetch more item than matched if len(features) == matched: break if len(features) > matched: raise MosaicError( "Something weird is going on, please open an issue in https://github.com/developmentseed/cogeo-mosaic" ) page += 1 # https://github.com/radiantearth/stac-api-spec/blob/master/api-spec.md#paging-extension if next_link_key: links = list( filter(lambda link: link["rel"] == next_link_key, results["links"])) if not links: break stac_query = query_from_link(links[0], stac_query) else: stac_query.update({"page": page}) return features
def update( self, features: Sequence[Dict], add_first: bool = True, quiet: bool = False, **kwargs, ): """Update existing MosaicJSON on backend.""" logger.debug(f"Updating {self.mosaic_name}...") new_mosaic = MosaicJSON.from_features( features, self.mosaic_def.minzoom, self.mosaic_def.maxzoom, quadkey_zoom=self.quadkey_zoom, quiet=quiet, **kwargs, ) bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds) self.mosaic_def._increase_version() self.mosaic_def.bounds = bounds self.mosaic_def.center = ( (bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, self.mosaic_def.minzoom, ) self.bounds = bounds with self.db: self.db.execute( f""" UPDATE {self._metadata_table} SET mosaicjson = :mosaicjson, name = :name, description = :description, version = :version, attribution = :attribution, minzoom = :minzoom, maxzoom = :maxzoom, quadkey_zoom = :quadkey_zoom, bounds = :bounds, center = :center WHERE name=:name """, self.mosaic_def.dict(), ) if add_first: self.db.executemany( f""" UPDATE "{self.mosaic_name}" SET assets = ( SELECT json_group_array(value) FROM ( SELECT value FROM json_each(?) UNION ALL SELECT value FROM json_each(assets) ) ) WHERE quadkey=?; """, [(assets, qk) for qk, assets in new_mosaic.tiles.items()], ) else: self.db.executemany( f""" UPDATE "{self.mosaic_name}" SET assets = ( SELECT json_group_array(value) FROM ( SELECT value FROM json_each(assets) UNION ALL SELECT value FROM json_each(?) ) ) WHERE quadkey=?; """, [(assets, qk) for qk, assets in new_mosaic.tiles.items()], )
def write(self, overwrite: bool = False): """Write mosaicjson document to an SQLite database. Args: overwrite (bool): delete old mosaic items in the Table. Raises: MosaicExistsError: If mosaic already exists in the Table. """ if self._mosaic_exists(): if not overwrite: raise MosaicExistsError( f"'{self.mosaic_name}' Table already exists in {self.db_path}, use `overwrite=True`." ) self.delete() with self.db: logger.debug( f"Creating '{self.mosaic_name}' Table in {self.db_path}.") self.db.execute(f""" CREATE TABLE IF NOT EXISTS {self._metadata_table} ( mosaicjson TEXT NOT NULL, name TEXT NOT NULL, description TEXT, version TEXT NOT NULL, attribution TEXT, minzoom INTEGER NOT NULL, maxzoom INTEGER NOT NULL, quadkey_zoom INTEGER, bounds JSON NOT NULL, center JSON ); """) self.db.execute(f""" CREATE TABLE "{self.mosaic_name}" ( quadkey TEXT NOT NULL, assets JSON NOT NULL ); """) logger.debug(f"Adding items in '{self.mosaic_name}' Table.") self.db.execute( f""" INSERT INTO {self._metadata_table} ( mosaicjson, name, description, version, attribution, minzoom, maxzoom, quadkey_zoom, bounds, center ) VALUES ( :mosaicjson, :name, :description, :version, :attribution, :minzoom, :maxzoom, :quadkey_zoom, :bounds, :center ); """, self.mosaic_def.dict(), ) self.db.executemany( f'INSERT INTO "{self.mosaic_name}" (quadkey, assets) VALUES (?, ?);', self.mosaic_def.tiles.items(), )