class S3Backend(BaseBackend):
"""S3 Backend Adapter"""
def __init__(
self,
bucket: str,
key: str,
mosaic_def: Optional[Union[MosaicJSON, Dict]] = None,
client: Optional[boto3_session.client] = None,
**kwargs: Any,
):
"""Initialize S3Backend."""
self.client = client or boto3_session().client("s3")
self.key = key
self.bucket = bucket
self.path = f"s3://{bucket}/{key}"
if mosaic_def is not None:
self.mosaic_def = MosaicJSON(**dict(mosaic_def))
else:
self.mosaic_def = self._read(**kwargs)
def tile(self, x: int, y: int, z: int) -> List[str]:
"""Retrieve assets for tile."""
return get_assets_from_json(self.mosaic_def.tiles, self.quadkey_zoom, x, y, z)
def point(self, lng: float, lat: float) -> List[str]:
"""Retrieve assets for point."""
tile = mercantile.tile(lng, lat, self.quadkey_zoom)
return get_assets_from_json(
self.mosaic_def.tiles, self.quadkey_zoom, tile.x, tile.y, tile.z
)
def write(self, gzip: bool = None, **kwargs: Any):
"""Write mosaicjson document to AWS S3."""
mosaic_doc = self.mosaic_def.dict(exclude_none=True)
if gzip or (gzip is None and self.key.endswith(".gz")):
body = _compress_gz_json(mosaic_doc)
else:
body = json.dumps(mosaic_doc).encode("utf-8")
_aws_put_data(self.key, self.bucket, body, client=self.client, **kwargs)
@functools.lru_cache(maxsize=512)
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
body = _aws_get_data(self.key, self.bucket, client=self.client)
if gzip or (gzip is None and self.key.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
class FileBackend(BaseBackend):
"""Local File Backend Adapter"""
def __init__(
self,
path: str,
mosaic_def: Optional[Union[MosaicJSON, Dict]] = None,
**kwargs: Any,
):
"""Initialize FileBackend."""
self.path = path
if mosaic_def is not None:
self.mosaic_def = MosaicJSON(**dict(mosaic_def))
else:
self.mosaic_def = self._read(**kwargs)
def tile(self, x: int, y: int, z: int) -> List[str]:
"""Retrieve assets for tile."""
return get_assets_from_json(self.mosaic_def.tiles, self.quadkey_zoom, x, y, z)
def point(self, lng: float, lat: float) -> List[str]:
"""Retrieve assets for point."""
tile = mercantile.tile(lng, lat, self.quadkey_zoom)
return get_assets_from_json(
self.mosaic_def.tiles, self.quadkey_zoom, tile.x, tile.y, tile.z
)
def write(self, gzip: bool = None):
"""Write mosaicjson document to a file."""
body = self.mosaic_def.dict(exclude_none=True)
with open(self.path, "wb") as f:
if gzip or (gzip is None and self.path.endswith(".gz")):
f.write(_compress_gz_json(body))
else:
f.write(json.dumps(body).encode("utf-8"))
@functools.lru_cache(maxsize=512)
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
with open(self.path, "rb") as f:
body = f.read()
if gzip or (gzip is None and self.path.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
class BaseBackend(BaseReader):
"""Base Class for cogeo-mosaic backend storage."""
path: str = attr.ib()
mosaic_def: MosaicJSON = attr.ib(default=None)
reader: Type[BaseReader] = attr.ib(default=COGReader)
reader_options: Dict = attr.ib(factory=dict)
backend_options: Dict = attr.ib(factory=dict)
# TMS is outside the init because mosaicJSON and cogeo-mosaic only
# works with WebMercator (mercantile) for now.
tms: TileMatrixSet = attr.ib(init=False, default=WEB_MERCATOR_TMS)
_backend_name: str
_file_byte_size: Optional[int] = 0
@mosaic_def.validator
def _check_mosaic_def(self, attribute, value):
if value is not None:
self.mosaic_def = MosaicJSON(**dict(value))
def __attrs_post_init__(self):
"""Post Init: if not passed in init, try to read from self.path."""
self.mosaic_def = self.mosaic_def or self._read(**self.backend_options)
self.minzoom = self.mosaic_def.minzoom
self.maxzoom = self.mosaic_def.maxzoom
self.bounds = self.mosaic_def.bounds
@property
def center(self):
"""Return center from the mosaic definition."""
return self.mosaic_def.center
def info(self, quadkeys: bool = False) -> Info: # type: ignore
"""Mosaic info."""
return Info(
bounds=self.mosaic_def.bounds,
center=self.mosaic_def.center,
maxzoom=self.mosaic_def.maxzoom,
minzoom=self.mosaic_def.minzoom,
name=self.mosaic_def.name if self.mosaic_def.name else "mosaic",
quadkeys=[] if not quadkeys else self._quadkeys,
)
@property
def _quadkeys(self) -> List[str]:
"""Return the list of quadkey tiles."""
return list(self.mosaic_def.tiles)
def stats(self):
"""PlaceHolder for BaseReader.stats."""
raise NotImplementedError
@property
def metadata(self) -> Metadata: # type: ignore
"""Retrieve Mosaic metadata
Returns
-------
MosaicJSON as dict without `tiles` key.
"""
return Metadata(**self.mosaic_def.dict())
def assets_for_tile(self, x: int, y: int, z: int) -> List[str]:
"""Retrieve assets for tile."""
return self.get_assets(x, y, z)
def assets_for_point(self, lng: float, lat: float) -> List[str]:
"""Retrieve assets for point."""
tile = mercantile.tile(lng, lat, self.quadkey_zoom)
return self.get_assets(tile.x, tile.y, tile.z)
@cached(
TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl),
key=lambda self, x, y, z: hashkey(self.path, x, y, z),
)
def get_assets(self, x: int, y: int, z: int) -> List[str]:
"""Find assets."""
mercator_tile = mercantile.Tile(x=x, y=y, z=z)
quadkeys = find_quadkeys(mercator_tile, self.quadkey_zoom)
return list(
itertools.chain.from_iterable(
[self.mosaic_def.tiles.get(qk, []) for qk in quadkeys]))
def tile( # type: ignore
self,
x: int,
y: int,
z: int,
reverse: bool = False,
**kwargs: Any,
) -> Tuple[ImageData, List[str]]:
"""Get Tile from multiple observation."""
assets = self.assets_for_tile(x, y, z)
if not assets:
raise NoAssetFoundError(f"No assets found for tile {z}-{x}-{y}")
if reverse:
assets = list(reversed(assets))
def _reader(asset: str, x: int, y: int, z: int,
**kwargs: Any) -> ImageData:
with self.reader(asset, **self.reader_options) as src_dst:
return src_dst.tile(x, y, z, **kwargs)
return mosaic_reader(assets, _reader, x, y, z, **kwargs)
def point(
self,
lon: float,
lat: float,
threads=MAX_THREADS,
reverse: bool = False,
**kwargs: Any,
) -> List[Dict]:
"""Get Point value from multiple observation."""
assets = self.assets_for_point(lon, lat)
if not assets:
raise NoAssetFoundError(f"No assets found for point ({lon},{lat})")
if reverse:
assets = list(reversed(assets))
def _reader(asset: str, lon: float, lat: float, **kwargs) -> Dict:
with self.reader(asset, **self.reader_options) as src_dst:
return src_dst.point(lon, lat, **kwargs)
tasks = create_tasks(_reader, assets, threads, lon, lat, **kwargs)
return [{
"asset": asset,
"values": pt
} for pt, asset in filter_tasks(
tasks, allowed_exceptions=(PointOutsideBounds, ))]
def preview(self):
"""PlaceHolder for BaseReader.preview."""
raise NotImplementedError
def part(self):
"""PlaceHolder for BaseReader.part."""
raise NotImplementedError
def feature(self):
"""PlaceHolder for BaseReader.feature."""
raise NotImplementedError
@abc.abstractmethod
def _read(self) -> MosaicJSON:
"""Fetch mosaic definition"""
@property
def mosaicid(self) -> str:
"""Return sha224 id of the mosaicjson document."""
return get_hash(**self.mosaic_def.dict(exclude_none=True))
@property
def quadkey_zoom(self) -> int:
"""Return Quadkey zoom property."""
return self.mosaic_def.quadkey_zoom or self.mosaic_def.minzoom
@abc.abstractmethod
def write(self, overwrite: bool = True):
"""Upload new MosaicJSON to backend."""
def update(
self,
features: Sequence[Dict],
add_first: bool = True,
quiet: bool = False,
**kwargs,
):
"""Update existing MosaicJSON on backend."""
new_mosaic = self.mosaic_def.from_features(
features,
self.mosaic_def.minzoom,
self.mosaic_def.maxzoom,
quadkey_zoom=self.quadkey_zoom,
quiet=quiet,
**kwargs,
)
for quadkey, new_assets in new_mosaic.tiles.items():
tile = mercantile.quadkey_to_tile(quadkey)
assets = self.assets_for_tile(*tile)
assets = [*new_assets, *assets
] if add_first else [*assets, *new_assets]
# add custom sorting algorithm (e.g based on path name)
self.mosaic_def.tiles[quadkey] = assets
bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds)
self.mosaic_def._increase_version()
self.mosaic_def.bounds = bounds
self.mosaic_def.center = (
(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2,
self.mosaic_def.minzoom,
)
# We only write if path is set
if self.path:
self.write(overwrite=True)
return
def features_to_mosaicJSON(features: List[Dict],
quadkey_zoom: int = None,
minzoom: int = 7,
maxzoom: int = 12,
index: Union[bool, Dict] = True,
sort='min-cloud') -> Dict:
"""
Create a mosaicJSON from stac features.
Attributes
----------
features : list
sat-api features.
minzoom : int, optional, (default: 7)
Mosaic Min Zoom.
maxzoom : int, optional (default: 12)
Mosaic Max Zoom.
Returns
-------
out : dict
MosaicJSON definition.
"""
if not index:
mosaic = MosaicJSON.from_features(features=features,
minzoom=minzoom,
maxzoom=maxzoom,
quadkey_zoom=quadkey_zoom,
accessor=landsat_accessor)
return mosaic.dict(exclude_none=True)
if not isinstance(index, dict):
path = index_data_path()
with gzip.open(path, 'rt') as f:
index = json.load(f)
# Define quadkey zoom from index
quadkey_zoom = len(list(index.values())[0][0])
pr_keys = set(index.keys())
sorted_features = {}
for feature in features:
pathrow = feature['properties']['eo:column'].zfill(
3) + feature['properties']['eo:row'].zfill(3)
if pathrow not in pr_keys:
continue
sorted_features[pathrow] = sorted_features.get(pathrow, [])
sorted_features[pathrow].append(feature)
tiles = {}
for pathrow, feats in sorted_features.items():
if sort == 'min-cloud':
selected = min(feats,
key=lambda x: x['properties']['eo:cloud_cover'])
elif sort == 'max-cloud':
selected = max(feats,
key=lambda x: x['properties']['eo:cloud_cover'])
else:
selected = feats[0]
product_id = landsat_accessor(selected)
quadkeys = index[pathrow]
for qk in quadkeys:
tiles[qk] = tiles.get(qk, set())
tiles[qk].add(product_id)
bounds = quadkeys_to_bounds(tiles.keys())
mosaic = MosaicJSON(mosaicjson="0.0.2",
minzoom=minzoom,
maxzoom=maxzoom,
quadkey_zoom=quadkey_zoom,
bounds=bounds,
tiles=tiles)
return mosaic.dict(exclude_none=True)
