def test_update_valid(): """Should work as expected.""" runner = CliRunner() with runner.isolated_filesystem(): with open("mosaic.json", "w") as f: f.write( json.dumps( MosaicJSON.from_urls([asset1]).dict(exclude_none=True))) with open("./list.txt", "w") as f: f.write("\n".join([asset2])) result = runner.invoke( cogeo_cli, ["update", "list.txt", "mosaic.json", "--quiet"]) assert not result.exception assert result.exit_code == 0 with open("mosaic.json", "r") as f: updated_mosaic = json.load(f) updated_mosaic["version"] == "1.0.1" assert not mosaic_content.tiles == updated_mosaic["tiles"] with open("mosaic.json", "w") as f: f.write( json.dumps( MosaicJSON.from_urls([asset1]).dict(exclude_none=True))) result = runner.invoke( cogeo_cli, ["update", "list.txt", "mosaic.json", "--add-last", "--quiet"]) assert not result.exception assert result.exit_code == 0 with open("mosaic.json", "r") as f: updated_mosaic = json.load(f) updated_mosaic["version"] == "1.0.1" assert mosaic_content.tiles == updated_mosaic["tiles"]
def test_read_mosaic(app): """test GET /mosaicjson endpoint""" # TODO: Remove response = app.get("/mosaicjson", params={"url": MOSAICJSON_FILE}) assert response.status_code == 200 MosaicJSON(**response.json()) response = app.get("/mosaicjson/", params={"url": MOSAICJSON_FILE}) assert response.status_code == 200 MosaicJSON(**response.json())
def __init__( self, path: str, mosaic_def: Optional[Union[MosaicJSON, Dict]] = None, **kwargs: Any, ): """Initialize FileBackend.""" self.path = path if mosaic_def is not None: self.mosaic_def = MosaicJSON(**dict(mosaic_def)) else: self.mosaic_def = self._read(**kwargs)
def create( input_files, output, minzoom, maxzoom, quadkey_zoom, min_tile_cover, tile_cover_sort, threads, quiet, ): """Create mosaic definition file.""" input_files = input_files.read().splitlines() mosaicjson = MosaicJSON.from_urls( input_files, minzoom=minzoom, maxzoom=maxzoom, quadkey_zoom=quadkey_zoom, minimum_tile_cover=min_tile_cover, tile_cover_sort=tile_cover_sort, max_threads=threads, quiet=quiet, ) if output: with MosaicBackend(output, mosaic_def=mosaicjson) as mosaic: mosaic.write() else: click.echo(json.dumps(mosaicjson.dict(exclude_none=True)))
def create_from_features( features, output, minzoom, maxzoom, property, quadkey_zoom, min_tile_cover, tile_cover_sort, quiet, ): """Create mosaic definition file.""" mosaicjson = MosaicJSON.from_features( list(features), minzoom, maxzoom, quadkey_zoom=quadkey_zoom, accessor=lambda feature: feature["properties"][property], minimum_tile_cover=min_tile_cover, tile_cover_sort=tile_cover_sort, quiet=quiet, ) if output: with MosaicBackend(output, mosaic_def=mosaicjson) as mosaic: mosaic.write() else: click.echo(json.dumps(mosaicjson.dict(exclude_none=True)))
def __init__( self, table_name: str, mosaic_def: Optional[Union[MosaicJSON, Dict]] = None, region: str = os.getenv("AWS_REGION", "us-east-1"), client: Optional[Any] = None, ): """Initialize DynamoDBBackend.""" self.client = client or boto3.resource("dynamodb", region_name=region) self.table = self.client.Table(table_name) self.path = f"dynamodb://{region}/{table_name}" if mosaic_def is not None: self.mosaic_def = MosaicJSON(**dict(mosaic_def)) else: self.mosaic_def = self._read()
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" body = _aws_get_data(self.key, self.bucket, client=self.client) if gzip or (gzip is None and self.key.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def _read(self) -> MosaicJSON: # type: ignore """Get Mosaic definition info.""" meta = self._fetch_metadata() if not meta: raise MosaicNotFoundError(f"Mosaic not found in {self.path}") meta["tiles"] = {} return MosaicJSON(**meta)
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" body = requests.get(self.path).content if gzip or (gzip is None and self.path.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def _read(*args: Any, **kwargs: Any) -> MosaicJSON: """Match signature of `cogeo_mosaic.backends.BaseBackend._read`""" data = read_json_fixture(fname) for qk in data["tiles"]: data["tiles"][qk] = [ os.path.join(os.path.dirname(fname), f) for f in data["tiles"][qk] ] return MosaicJSON(**data)
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" with open(self.path, "rb") as f: body = f.read() if gzip or (gzip is None and self.path.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def __init__( self, bucket: str, key: str, mosaic_def: Optional[Union[MosaicJSON, Dict]] = None, client: Optional[boto3_session.client] = None, **kwargs: Any, ): """Initialize S3Backend.""" self.client = client or boto3_session().client("s3") self.key = key self.bucket = bucket self.path = f"s3://{bucket}/{key}" if mosaic_def is not None: self.mosaic_def = MosaicJSON(**dict(mosaic_def)) else: self.mosaic_def = self._read(**kwargs)
def query(self, *args, **kwargs): """Mock Scan.""" mosaic = MosaicJSON(**mosaic_content) return { "Items": [{ "quadkey": qk, "assets": assets } for qk, assets in mosaic.tiles.items()] }
def test_create_valid(): """Should work as expected.""" runner = CliRunner() with runner.isolated_filesystem(): with open("./list.txt", "w") as f: f.write("\n".join(assets)) result = runner.invoke(cogeo_cli, ["create", "list.txt", "--quiet"]) assert not result.exception assert result.exit_code == 0 assert mosaic_content == MosaicJSON(**json.loads(result.output)) result = runner.invoke(cogeo_cli, ["create", "list.txt", "-o", "mosaic.json"]) assert not result.exception assert result.exit_code == 0 with open("mosaic.json", "r") as f: assert mosaic_content == MosaicJSON(**json.load(f))
def _read(self) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" body = self._get_object(self.key, self.bucket) self._file_byte_size = len(body) if self.key.endswith(".gz"): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def update( self, features: Sequence[Dict], add_first: bool = True, quiet: bool = False, **kwargs, ): """Update existing MosaicJSON on backend.""" logger.debug(f"Updating {self.mosaic_name}...") new_mosaic = MosaicJSON.from_features( features, self.mosaic_def.minzoom, self.mosaic_def.maxzoom, quadkey_zoom=self.quadkey_zoom, quiet=quiet, **kwargs, ) bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds) self.mosaic_def._increase_version() self.mosaic_def.bounds = bounds self.mosaic_def.center = ( (bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, self.mosaic_def.minzoom, ) self.bounds = bounds items: List[Dict[str, Any]] = [] # Create Metadata item # Note: `parse_float=Decimal` is required because DynamoDB requires all numbers to be # in Decimal type (ref: https://blog.ruanbekker.com/blog/2019/02/05/convert-float-to-decimal-data-types-for-boto3-dynamodb-using-python/) meta = json.loads(self.mosaic_def.json(exclude={"tiles"}), parse_float=Decimal) items.append({ "quadkey": self._metadata_quadkey, "mosaicId": self.mosaic_name, **meta }) # Create Tile items for quadkey, new_assets in new_mosaic.tiles.items(): tile = mercantile.quadkey_to_tile(quadkey) assets = self.assets_for_tile(*tile) assets = [*new_assets, *assets ] if add_first else [*assets, *new_assets] items.append({ "mosaicId": self.mosaic_name, "quadkey": quadkey, "assets": assets }) self._write_items(items)
def mosaic(preference, check_exists, file): features = load_features(file) mosaic = MosaicJSON.from_features(features, minzoom=11, maxzoom=16, asset_filter=asset_filter, accessor=path_accessor, check_exists=check_exists, preference=preference) print(json.dumps(mosaic.dict(), separators=(',', ':')))
def _read( # type: ignore self, query: Dict, minzoom: int, maxzoom: int, accessor: Callable = default_stac_accessor, max_items: Optional[int] = None, stac_query_limit: int = 500, stac_next_link_key: Optional[str] = None, **kwargs: Any, ) -> MosaicJSON: """ Fetch STAC API and construct the mosaicjson. Attributes ---------- query : Dict, required STAC API POST request query. minzoom: int, required mosaic min-zoom. maxzoom: int, required mosaic max-zoom. accessor: callable, required Function called on each feature to get its identifier. max_items: int, optional Limit the maximum of items returned by the API stac_query_limit: int, optional Add "limit" option to the POST Query, default is set to 500. stac_next_link_key: str, optional link's 'next' key. kwargs: any Options forwarded to `MosaicJSON.from_features` Returns ------- mosaic_definition : MosaicJSON Mosaic definition. """ logger.debug(f"Using STAC backend: {self.path}") features = _fetch( self.path, query, max_items=max_items, limit=stac_query_limit, next_link_key=stac_next_link_key, ) logger.debug(f"Creating mosaic from {len(features)} features") return MosaicJSON.from_features( features, minzoom, maxzoom, accessor=accessor, **kwargs )
def tmpmosaic(): """Create a Temporary MosaicJSON file.""" fileobj = tempfile.NamedTemporaryFile(suffix=".json.gz", delete=False) fileobj.close() mosaic_def = MosaicJSON.from_urls(assets) with FileBackend(fileobj.name, mosaic_def=mosaic_def) as mosaic: mosaic.write(overwrite=True) try: yield fileobj.name finally: os.remove(fileobj.name)
def __attrs_post_init__(self): """Post Init.""" # Construct a FAKE mosaicJSON # mosaic_def has to be defined. As we do for the DynamoDB and SQLite backend # we set `tiles` to an empty list. self.mosaic_def = MosaicJSON( mosaicjson="0.0.2", name="it's fake but it's ok", minzoom=self.minzoom, maxzoom=self.maxzoom, tiles=[], )
def test_create_valid(): """Should work as expected.""" runner = CliRunner() with runner.isolated_filesystem(): with open("./list.txt", "w") as f: f.write("\n") f.write("\n".join(assets)) f.write("\n") result = runner.invoke(cogeo_cli, ["create", "list.txt", "--quiet"]) assert not result.exception assert result.exit_code == 0 assert mosaic_content == MosaicJSON(**json.loads(result.output)) result = runner.invoke(cogeo_cli, ["create", "list.txt", "-o", "mosaic.json"]) assert not result.exception assert result.exit_code == 0 assert mosaic_content == MosaicJSON.parse_file("mosaic.json") result = runner.invoke( cogeo_cli, [ "create", "list.txt", "-o", "mosaic.json", "--name", "my_mosaic", "--description", "A mosaic", "--attribution", "someone", ], ) assert not result.exception assert result.exit_code == 0 mosaic = MosaicJSON.parse_file("mosaic.json") assert mosaic.name == "my_mosaic" assert mosaic.description == "A mosaic" assert mosaic.attribution == "someone"
class S3Backend(BaseBackend): """S3 Backend Adapter""" def __init__( self, bucket: str, key: str, mosaic_def: Optional[Union[MosaicJSON, Dict]] = None, client: Optional[boto3_session.client] = None, **kwargs: Any, ): """Initialize S3Backend.""" self.client = client or boto3_session().client("s3") self.key = key self.bucket = bucket self.path = f"s3://{bucket}/{key}" if mosaic_def is not None: self.mosaic_def = MosaicJSON(**dict(mosaic_def)) else: self.mosaic_def = self._read(**kwargs) def tile(self, x: int, y: int, z: int) -> List[str]: """Retrieve assets for tile.""" return get_assets_from_json(self.mosaic_def.tiles, self.quadkey_zoom, x, y, z) def point(self, lng: float, lat: float) -> List[str]: """Retrieve assets for point.""" tile = mercantile.tile(lng, lat, self.quadkey_zoom) return get_assets_from_json( self.mosaic_def.tiles, self.quadkey_zoom, tile.x, tile.y, tile.z ) def write(self, gzip: bool = None, **kwargs: Any): """Write mosaicjson document to AWS S3.""" mosaic_doc = self.mosaic_def.dict(exclude_none=True) if gzip or (gzip is None and self.key.endswith(".gz")): body = _compress_gz_json(mosaic_doc) else: body = json.dumps(mosaic_doc).encode("utf-8") _aws_put_data(self.key, self.bucket, body, client=self.client, **kwargs) @functools.lru_cache(maxsize=512) def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" body = _aws_get_data(self.key, self.bucket, client=self.client) if gzip or (gzip is None and self.key.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def _add(body: str, url: str) -> Tuple: mosaic_definition = MosaicJSON(**json.loads(body)) with MosaicBackend(url, mosaic_def=mosaic_definition) as mosaic: mosaic.write() return ( "OK", "application/json", json.dumps({ "id": url, "status": "READY" }, separators=(",", ":")), )
def _add(body: str, mosaicid: str) -> Tuple: if _aws_head_object(_create_mosaic_path(mosaicid), client=s3_client): return ("NOK", "text/plain", f"Mosaic: {mosaicid} already exist.") mosaic_definition = MosaicJSON(**json.loads(body)) with MosaicBackend( _create_mosaic_path(mosaicid), mosaic_def=mosaic_definition ) as mosaic: mosaic.write() return ( "OK", "application/json", json.dumps({"id": mosaicid, "status": "READY"}, separators=(",", ":")), )
def _read(self) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" try: with open(self.path, "rb") as f: body = f.read() except Exception as e: exc = _FILE_EXCEPTIONS.get(e, MosaicError) # type: ignore raise exc(str(e)) from e self._file_byte_size = len(body) if self.path.endswith(".gz"): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
class FileBackend(BaseBackend): """Local File Backend Adapter""" def __init__( self, path: str, mosaic_def: Optional[Union[MosaicJSON, Dict]] = None, **kwargs: Any, ): """Initialize FileBackend.""" self.path = path if mosaic_def is not None: self.mosaic_def = MosaicJSON(**dict(mosaic_def)) else: self.mosaic_def = self._read(**kwargs) def tile(self, x: int, y: int, z: int) -> List[str]: """Retrieve assets for tile.""" return get_assets_from_json(self.mosaic_def.tiles, self.quadkey_zoom, x, y, z) def point(self, lng: float, lat: float) -> List[str]: """Retrieve assets for point.""" tile = mercantile.tile(lng, lat, self.quadkey_zoom) return get_assets_from_json( self.mosaic_def.tiles, self.quadkey_zoom, tile.x, tile.y, tile.z ) def write(self, gzip: bool = None): """Write mosaicjson document to a file.""" body = self.mosaic_def.dict(exclude_none=True) with open(self.path, "wb") as f: if gzip or (gzip is None and self.path.endswith(".gz")): f.write(_compress_gz_json(body)) else: f.write(json.dumps(body).encode("utf-8")) @functools.lru_cache(maxsize=512) def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" with open(self.path, "rb") as f: body = f.read() if gzip or (gzip is None and self.path.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
def create_mosaicjson(body: CreateMosaicJSON): """Create a MosaicJSON""" mosaic = MosaicJSON.from_urls( body.files, minzoom=body.minzoom, maxzoom=body.maxzoom, max_threads=body.max_threads, ) mosaic_path = MosaicPath(body.url) with MosaicBackend(mosaic_path, mosaic_def=mosaic) as mosaic: try: mosaic.write() except NotImplementedError: raise BadRequestError( f"{mosaic.__class__.__name__} does not support write operations" ) return mosaic.mosaic_def
def subset_mosaic(mosaic, overview_qk, overview_zoom): """Create subset of mosaic within a single overview quadkey Args: - overview_qk: zoom 6 quadkey """ qk_tiles = { k: v for k, v in mosaic['tiles'].items() if k[:overview_zoom] == overview_qk } bounds = mercantile.bounds(mercantile.quadkey_to_tile(overview_qk)) # The new mosaic needs to be the same minzoom, quadkey zoom as new_mosaic = deepcopy(mosaic) new_mosaic['tiles'] = qk_tiles new_mosaic['bounds'] = bounds return MosaicJSON(**new_mosaic)
def create_mosaic(urls=None, res=None): allowed_res = ['1', '13', '1m', '2'] if res and res not in allowed_res: raise ValueError(f'res must be in {allowed_res}') urls = find_urls_for_res(res) features = [] for url in urls: key = url.split('/')[4] geom = box(*parse_grid(key)) full_s3_path = f's3://{BUCKET}/{url}' feature = Feature(geometry=geom, properties={'path': full_s3_path}) features.append(dict(feature)) minzoom, maxzoom = DEFAULT_ZOOMS[res] mosaic = MosaicJSON.from_features( features, minzoom, maxzoom, accessor=lambda x: x['properties']['path'])
def _read(self) -> MosaicJSON: # type: ignore """Get Mosaic definition info.""" meta = self._fetch_dynamodb("-1") # Numeric values are loaded from DynamoDB as Decimal types # Convert maxzoom, minzoom, quadkey_zoom to float/int for key in ["minzoom", "maxzoom", "quadkey_zoom"]: if meta.get(key): meta[key] = int(meta[key]) # Convert bounds, center to float/int for key in ["bounds", "center"]: if meta.get(key): meta[key] = list(map(float, meta[key])) # Create pydantic class # For now, a tiles key must exist meta["tiles"] = {} return MosaicJSON(**meta)