def default_filter(
tile: mercantile.Tile,
dataset: Sequence[Dict],
geoms: Sequence[polygons],
minimum_tile_cover: float = None,
tile_cover_sort: bool = False,
maximum_items_per_tile: Optional[int] = None,
) -> List:
"""Filter and/or sort dataset per intersection coverage."""
indices = list(range(len(dataset)))
if minimum_tile_cover or tile_cover_sort:
tile_geom = polygons(mercantile.feature(tile)["geometry"]["coordinates"][0])
int_pcts = _intersect_percent(tile_geom, geoms)
if minimum_tile_cover:
if minimum_tile_cover > 1.0:
raise MosaicError("`minimum_tile_cover` HAS TO be between 0 and 1.")
indices = [ind for ind in indices if int_pcts[ind] > minimum_tile_cover]
if tile_cover_sort:
# https://stackoverflow.com/a/9764364
_, indices = zip(*sorted(zip(int_pcts, indices), reverse=True))
if maximum_items_per_tile:
indices = indices[:maximum_items_per_tile]
return [dataset[ind] for ind in indices]
def query_from_link(link: Dict, query: Dict):
"""Handle Next Link."""
q = query.copy()
if link["method"] != "POST":
raise MosaicError("Fetch doesn't support GET for next request.")
if link.get("merge", False):
q.update(link.get("body", {}))
else:
q = link.get("body", {})
return q
def _stac_search(url: str, q: Dict):
try:
r = requests.post(url, headers=headers, json=q)
r.raise_for_status()
except requests.exceptions.HTTPError as e:
# post-flight errors
status_code = e.response.status_code
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response.content) from e
except requests.exceptions.RequestException as e:
# pre-flight errors
raise MosaicError(e.args[0].reason) from e
return r.json()
def _read(self, gzip: bool = None) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
try:
r = requests.get(self.path)
r.raise_for_status()
except requests.exceptions.HTTPError as e:
# post-flight errors
status_code = e.response.status_code
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response.content) from e
except requests.exceptions.RequestException as e:
# pre-flight errors
raise MosaicError(e.args[0].reason) from e
body = r.content
self._file_byte_size = len(body)
if gzip or (gzip is None and self.path.endswith(".gz")):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def _read(self) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
try:
r = httpx.get(self.input)
r.raise_for_status()
except httpx.HTTPStatusError as e:
# post-flight errors
status_code = e.response.status_code
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response.content) from e
except httpx.RequestError as e:
# pre-flight errors
raise MosaicError(e.args[0].reason) from e
body = r.content
self._file_byte_size = len(body)
if self.input.endswith(".gz"):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def _fetch(
stac_url: str,
query: Dict,
max_items: Optional[int] = None,
next_link_key: Optional[str] = None,
limit: int = 500,
) -> List[Dict]:
"""Call STAC API."""
features: List[Dict] = []
stac_query = query.copy()
headers = {
"Content-Type": "application/json",
"Accept-Encoding": "gzip",
"Accept": "application/geo+json",
}
if "limit" not in stac_query:
stac_query.update({"limit": limit})
def _stac_search(url: str, q: Dict):
try:
r = requests.post(url, headers=headers, json=q)
r.raise_for_status()
except requests.exceptions.HTTPError as e:
# post-flight errors
status_code = e.response.status_code
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response.content) from e
except requests.exceptions.RequestException as e:
# pre-flight errors
raise MosaicError(e.args[0].reason) from e
return r.json()
page = 1
while True:
logger.debug(f"Fetching page {page}")
logger.debug("query: " + json.dumps(stac_query))
results = _stac_search(stac_url, stac_query)
if not results.get("features"):
break
features.extend(results["features"])
if max_items and len(features) >= max_items:
features = features[:max_items]
break
# new STAC context spec
# {"page": 1, "limit": 1000, "matched": 5671, "returned": 1000}
# SAT-API META
# {"page": 4, "limit": 100, "found": 350, "returned": 50}
ctx = results.get("context", results.get("meta"))
matched = ctx.get("matched", ctx.get("found"))
logger.debug(json.dumps(ctx))
# Check if there is more data to fetch
if matched <= ctx["returned"]:
break
# We shouldn't fetch more item than matched
if len(features) == matched:
break
if len(features) > matched:
raise MosaicError(
"Something weird is going on, please open an issue in https://github.com/developmentseed/cogeo-mosaic"
)
page += 1
# https://github.com/radiantearth/stac-api-spec/blob/master/api-spec.md#paging-extension
if next_link_key:
links = list(
filter(lambda link: link["rel"] == next_link_key,
results["links"]))
if not links:
break
stac_query = query_from_link(links[0], stac_query)
else:
stac_query.update({"page": page})
return features
