class S3Backend(BaseBackend): """S3 Backend Adapter""" def __init__( self, bucket: str, key: str, mosaic_def: Optional[Union[MosaicJSON, Dict]] = None, client: Optional[boto3_session.client] = None, **kwargs: Any, ): """Initialize S3Backend.""" self.client = client or boto3_session().client("s3") self.key = key self.bucket = bucket self.path = f"s3://{bucket}/{key}" if mosaic_def is not None: self.mosaic_def = MosaicJSON(**dict(mosaic_def)) else: self.mosaic_def = self._read(**kwargs) def tile(self, x: int, y: int, z: int) -> List[str]: """Retrieve assets for tile.""" return get_assets_from_json(self.mosaic_def.tiles, self.quadkey_zoom, x, y, z) def point(self, lng: float, lat: float) -> List[str]: """Retrieve assets for point.""" tile = mercantile.tile(lng, lat, self.quadkey_zoom) return get_assets_from_json( self.mosaic_def.tiles, self.quadkey_zoom, tile.x, tile.y, tile.z ) def write(self, gzip: bool = None, **kwargs: Any): """Write mosaicjson document to AWS S3.""" mosaic_doc = self.mosaic_def.dict(exclude_none=True) if gzip or (gzip is None and self.key.endswith(".gz")): body = _compress_gz_json(mosaic_doc) else: body = json.dumps(mosaic_doc).encode("utf-8") _aws_put_data(self.key, self.bucket, body, client=self.client, **kwargs) @functools.lru_cache(maxsize=512) def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" body = _aws_get_data(self.key, self.bucket, client=self.client) if gzip or (gzip is None and self.key.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
class FileBackend(BaseBackend): """Local File Backend Adapter""" def __init__( self, path: str, mosaic_def: Optional[Union[MosaicJSON, Dict]] = None, **kwargs: Any, ): """Initialize FileBackend.""" self.path = path if mosaic_def is not None: self.mosaic_def = MosaicJSON(**dict(mosaic_def)) else: self.mosaic_def = self._read(**kwargs) def tile(self, x: int, y: int, z: int) -> List[str]: """Retrieve assets for tile.""" return get_assets_from_json(self.mosaic_def.tiles, self.quadkey_zoom, x, y, z) def point(self, lng: float, lat: float) -> List[str]: """Retrieve assets for point.""" tile = mercantile.tile(lng, lat, self.quadkey_zoom) return get_assets_from_json( self.mosaic_def.tiles, self.quadkey_zoom, tile.x, tile.y, tile.z ) def write(self, gzip: bool = None): """Write mosaicjson document to a file.""" body = self.mosaic_def.dict(exclude_none=True) with open(self.path, "wb") as f: if gzip or (gzip is None and self.path.endswith(".gz")): f.write(_compress_gz_json(body)) else: f.write(json.dumps(body).encode("utf-8")) @functools.lru_cache(maxsize=512) def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore """Get mosaicjson document.""" with open(self.path, "rb") as f: body = f.read() if gzip or (gzip is None and self.path.endswith(".gz")): body = _decompress_gz(body) return MosaicJSON(**json.loads(body))
class BaseBackend(BaseReader): """Base Class for cogeo-mosaic backend storage.""" path: str = attr.ib() mosaic_def: MosaicJSON = attr.ib(default=None) reader: Type[BaseReader] = attr.ib(default=COGReader) reader_options: Dict = attr.ib(factory=dict) backend_options: Dict = attr.ib(factory=dict) # TMS is outside the init because mosaicJSON and cogeo-mosaic only # works with WebMercator (mercantile) for now. tms: TileMatrixSet = attr.ib(init=False, default=WEB_MERCATOR_TMS) _backend_name: str _file_byte_size: Optional[int] = 0 @mosaic_def.validator def _check_mosaic_def(self, attribute, value): if value is not None: self.mosaic_def = MosaicJSON(**dict(value)) def __attrs_post_init__(self): """Post Init: if not passed in init, try to read from self.path.""" self.mosaic_def = self.mosaic_def or self._read(**self.backend_options) self.minzoom = self.mosaic_def.minzoom self.maxzoom = self.mosaic_def.maxzoom self.bounds = self.mosaic_def.bounds @property def center(self): """Return center from the mosaic definition.""" return self.mosaic_def.center def info(self, quadkeys: bool = False) -> Info: # type: ignore """Mosaic info.""" return Info( bounds=self.mosaic_def.bounds, center=self.mosaic_def.center, maxzoom=self.mosaic_def.maxzoom, minzoom=self.mosaic_def.minzoom, name=self.mosaic_def.name if self.mosaic_def.name else "mosaic", quadkeys=[] if not quadkeys else self._quadkeys, ) @property def _quadkeys(self) -> List[str]: """Return the list of quadkey tiles.""" return list(self.mosaic_def.tiles) def stats(self): """PlaceHolder for BaseReader.stats.""" raise NotImplementedError @property def metadata(self) -> Metadata: # type: ignore """Retrieve Mosaic metadata Returns ------- MosaicJSON as dict without `tiles` key. """ return Metadata(**self.mosaic_def.dict()) def assets_for_tile(self, x: int, y: int, z: int) -> List[str]: """Retrieve assets for tile.""" return self.get_assets(x, y, z) def assets_for_point(self, lng: float, lat: float) -> List[str]: """Retrieve assets for point.""" tile = mercantile.tile(lng, lat, self.quadkey_zoom) return self.get_assets(tile.x, tile.y, tile.z) @cached( TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl), key=lambda self, x, y, z: hashkey(self.path, x, y, z), ) def get_assets(self, x: int, y: int, z: int) -> List[str]: """Find assets.""" mercator_tile = mercantile.Tile(x=x, y=y, z=z) quadkeys = find_quadkeys(mercator_tile, self.quadkey_zoom) return list( itertools.chain.from_iterable( [self.mosaic_def.tiles.get(qk, []) for qk in quadkeys])) def tile( # type: ignore self, x: int, y: int, z: int, reverse: bool = False, **kwargs: Any, ) -> Tuple[ImageData, List[str]]: """Get Tile from multiple observation.""" assets = self.assets_for_tile(x, y, z) if not assets: raise NoAssetFoundError(f"No assets found for tile {z}-{x}-{y}") if reverse: assets = list(reversed(assets)) def _reader(asset: str, x: int, y: int, z: int, **kwargs: Any) -> ImageData: with self.reader(asset, **self.reader_options) as src_dst: return src_dst.tile(x, y, z, **kwargs) return mosaic_reader(assets, _reader, x, y, z, **kwargs) def point( self, lon: float, lat: float, threads=MAX_THREADS, reverse: bool = False, **kwargs: Any, ) -> List[Dict]: """Get Point value from multiple observation.""" assets = self.assets_for_point(lon, lat) if not assets: raise NoAssetFoundError(f"No assets found for point ({lon},{lat})") if reverse: assets = list(reversed(assets)) def _reader(asset: str, lon: float, lat: float, **kwargs) -> Dict: with self.reader(asset, **self.reader_options) as src_dst: return src_dst.point(lon, lat, **kwargs) tasks = create_tasks(_reader, assets, threads, lon, lat, **kwargs) return [{ "asset": asset, "values": pt } for pt, asset in filter_tasks( tasks, allowed_exceptions=(PointOutsideBounds, ))] def preview(self): """PlaceHolder for BaseReader.preview.""" raise NotImplementedError def part(self): """PlaceHolder for BaseReader.part.""" raise NotImplementedError def feature(self): """PlaceHolder for BaseReader.feature.""" raise NotImplementedError @abc.abstractmethod def _read(self) -> MosaicJSON: """Fetch mosaic definition""" @property def mosaicid(self) -> str: """Return sha224 id of the mosaicjson document.""" return get_hash(**self.mosaic_def.dict(exclude_none=True)) @property def quadkey_zoom(self) -> int: """Return Quadkey zoom property.""" return self.mosaic_def.quadkey_zoom or self.mosaic_def.minzoom @abc.abstractmethod def write(self, overwrite: bool = True): """Upload new MosaicJSON to backend.""" def update( self, features: Sequence[Dict], add_first: bool = True, quiet: bool = False, **kwargs, ): """Update existing MosaicJSON on backend.""" new_mosaic = self.mosaic_def.from_features( features, self.mosaic_def.minzoom, self.mosaic_def.maxzoom, quadkey_zoom=self.quadkey_zoom, quiet=quiet, **kwargs, ) for quadkey, new_assets in new_mosaic.tiles.items(): tile = mercantile.quadkey_to_tile(quadkey) assets = self.assets_for_tile(*tile) assets = [*new_assets, *assets ] if add_first else [*assets, *new_assets] # add custom sorting algorithm (e.g based on path name) self.mosaic_def.tiles[quadkey] = assets bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds) self.mosaic_def._increase_version() self.mosaic_def.bounds = bounds self.mosaic_def.center = ( (bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, self.mosaic_def.minzoom, ) # We only write if path is set if self.path: self.write(overwrite=True) return
def features_to_mosaicJSON(features: List[Dict], quadkey_zoom: int = None, minzoom: int = 7, maxzoom: int = 12, index: Union[bool, Dict] = True, sort='min-cloud') -> Dict: """ Create a mosaicJSON from stac features. Attributes ---------- features : list sat-api features. minzoom : int, optional, (default: 7) Mosaic Min Zoom. maxzoom : int, optional (default: 12) Mosaic Max Zoom. Returns ------- out : dict MosaicJSON definition. """ if not index: mosaic = MosaicJSON.from_features(features=features, minzoom=minzoom, maxzoom=maxzoom, quadkey_zoom=quadkey_zoom, accessor=landsat_accessor) return mosaic.dict(exclude_none=True) if not isinstance(index, dict): path = index_data_path() with gzip.open(path, 'rt') as f: index = json.load(f) # Define quadkey zoom from index quadkey_zoom = len(list(index.values())[0][0]) pr_keys = set(index.keys()) sorted_features = {} for feature in features: pathrow = feature['properties']['eo:column'].zfill( 3) + feature['properties']['eo:row'].zfill(3) if pathrow not in pr_keys: continue sorted_features[pathrow] = sorted_features.get(pathrow, []) sorted_features[pathrow].append(feature) tiles = {} for pathrow, feats in sorted_features.items(): if sort == 'min-cloud': selected = min(feats, key=lambda x: x['properties']['eo:cloud_cover']) elif sort == 'max-cloud': selected = max(feats, key=lambda x: x['properties']['eo:cloud_cover']) else: selected = feats[0] product_id = landsat_accessor(selected) quadkeys = index[pathrow] for qk in quadkeys: tiles[qk] = tiles.get(qk, set()) tiles[qk].add(product_id) bounds = quadkeys_to_bounds(tiles.keys()) mosaic = MosaicJSON(mosaicjson="0.0.2", minzoom=minzoom, maxzoom=maxzoom, quadkey_zoom=quadkey_zoom, bounds=bounds, tiles=tiles) return mosaic.dict(exclude_none=True)