def __update_data_from_catalog_config(self, catalog_config):
"""Updates configuration and data using given input catalog config
:param catalog_config: catalog config, from yml stac_config[catalogs]
:type catalog_config: dict
"""
model = catalog_config["model"]
self.catalog_config = update_nested_dict(self.catalog_config,
catalog_config)
# parse f-strings
# defaultdict usage will return "" for missing keys in format_args
format_args = copy.deepcopy(self.stac_config)
format_args["catalog"] = defaultdict(
str, dict(model, **{
"root": self.root,
"url": self.url
}))
parsed_model = format_dict_items(self.catalog_config["model"],
**format_args)
self.update_data(parsed_model)
return True
def set_stac_product_type_by_id(self, product_type, **kwargs):
"""Updates catalog with given product_type
:param product_type: product type
:type product_type: str
"""
collection = StacCollection(
url=self.url,
stac_config=self.stac_config,
provider=self.provider,
eodag_api=self.eodag_api,
root=self.root,
).get_collection_by_id(product_type)
cat_model = copy.deepcopy(
self.stac_config["catalogs"]["product_type"]["model"])
# parse f-strings
format_args = copy.deepcopy(self.stac_config)
format_args["catalog"] = defaultdict(str, **self.data)
format_args["collection"] = collection
try:
parsed_dict = format_dict_items(cat_model, **format_args)
except Exception:
logger.error("Could not format product_type catalog")
raise
self.update_data(parsed_dict)
# update search args
self.search_args.update({"product_type": product_type})
return parsed_dict
def __get_item_list(self, search_results, catalog):
"""Build STAC items list from EODAG search results
:param search_results: EODAG search results
:type search_results: :class:`eodag.api.search_result.SearchResult`
:param catalog: STAC catalog dict used for parsing item metadata
:type catalog: dict
:returns: STAC item dicts list
:rtype: list
"""
if len(search_results) <= 0:
return []
item_model = self.__filter_item_model_properties(
self.stac_config["item"], search_results[0].product_type)
item_list = []
for product in search_results:
# parse jsonpath
product_item = jsonpath_parse_dict_items(
item_model, {"product": product.__dict__})
# parse f-strings
format_args = copy.deepcopy(self.stac_config)
format_args["catalog"] = catalog
format_args["item"] = product_item
product_item = format_dict_items(product_item, **format_args)
product_item["bbox"] = [float(i) for i in product_item["bbox"]]
# remove empty properties
product_item = self.__filter_item_properties_values(product_item)
item_list.append(product_item)
return item_list
def get_stac_extension(url, stac_config, extension, **kwargs):
"""Parse STAC extension from config and return as dict
:param url: requested URL
:type url: str
:param stac_config: STAC configuration from stac.yml conf file
:type stac_config: dict
:param stac_config: STAC configuration from stac.yml conf file
:type stac_config: dic
:param extension: extension name
:type extension: str
:param dict kwargs: Additional variables needed for parsing extension
:returns: STAC extension as dictionnary
:rtype: dict
"""
extension_model = (copy.deepcopy(stac_config).get("extensions",
{}).get(
extension, {}))
# parse f-strings
format_args = copy.deepcopy(stac_config)
format_args["extension"] = {
"url": url,
"properties": kwargs.get("properties", {}),
}
extension = format_dict_items(extension_model, **format_args)
return extension
def set_stac_location_by_id(self, location, catalog_name):
"""Updates and returns catalog with given location
:param location: feature attribute value for shp filtering
:type location: str
:param catalog_name: catalog/location name
:type catalog_name: str
:returns: updated catalog
:rtype: dict
"""
location_list_cat_key = catalog_name + "_list"
if location_list_cat_key not in self.stac_config["catalogs"]:
logger.warning(
"no entry found for {}'s list in location_config".format(
catalog_name))
return {}
location_config = self.stac_config["catalogs"][location_list_cat_key]
for k in ["path", "attr"]:
if k not in location_config.keys():
logger.warning(
"no {} key found for {}'s list in location_config".format(
k, catalog_name))
return {}
path = location_config["path"]
attr = location_config["attr"]
with shapefile.Reader(path) as shp:
geom_hits = [
shape(shaperec.shape) for shaperec in shp.shapeRecords()
if shaperec.record.as_dict().get(attr, None) == location
]
if len(geom_hits) == 0:
logger.warning("no feature found in %s matching %s=%s" %
(path, attr, location))
return {}
geom = unary_union(geom_hits)
cat_model = copy.deepcopy(
self.stac_config["catalogs"]["country"]["model"])
# parse f-strings
format_args = copy.deepcopy(self.stac_config)
format_args["catalog"] = defaultdict(str, **self.data)
format_args["feature"] = defaultdict(str, {
"geometry": geom,
"id": location
})
parsed_dict = format_dict_items(cat_model, **format_args)
self.update_data(parsed_dict)
# update search args
self.search_args.update({"geom": geom})
return parsed_dict
def get_collections(self, *args, **kwargs):
"""Get the collection to which the product belongs"""
# See https://earth.esa.int/web/sentinel/missions/sentinel-2/news/-
# /asset_publisher/Ac0d/content/change-of
# -format-for-new-sentinel-2-level-1c-products-starting-on-6-december
product_type = kwargs.get("productType")
if product_type is None and not self.product_type_def_params:
collections = set()
collection = getattr(self.config, "collection", None)
if collection is None:
try:
for product_type, product_config in self.config.products.items():
if product_type != GENERIC_PRODUCT_TYPE:
collections.add(product_config["collection"])
else:
collections.add(
format_dict_items(product_config, **kwargs).get(
"collection", ""
)
)
except KeyError:
collections.add("")
else:
collections.add(collection)
return tuple(collections)
if self.provider == "peps":
if product_type == "S2_MSI_L1C":
date = kwargs.get("startTimeFromAscendingNode")
# If there is no criteria on date, we want to query all the collections
# known for providing L1C products
if date is None:
collections = ("S2", "S2ST")
else:
match = re.match(
r"(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})", date
).groupdict()
year, month, day = (
int(match["year"]),
int(match["month"]),
int(match["day"]),
)
if year > 2016 or (year == 2016 and month == 12 and day > 5):
collections = ("S2ST",)
else:
collections = ("S2", "S2ST")
else:
collections = (self.product_type_def_params.get("collection", ""),)
else:
collection = getattr(self.config, "collection", None)
if collection is None:
collection = (
self.product_type_def_params.get("collection", None) or product_type
)
collections = (
(collection,) if not isinstance(collection, list) else tuple(collection)
)
return collections
def __get_collection_list(self, filters=None):
"""Build STAC collections list
:param filters: additional filters for collections search
:type filters: dict
:returns: STAC collection dicts list
:rtype: list
"""
collection_model = copy.deepcopy(self.stac_config["collection"])
product_types = self.__get_product_types(filters)
collection_list = []
for product_type in product_types:
# get default provider for each product_type
product_type_provider = (self.provider or next(
self.eodag_api._plugins_manager.get_search_plugins(
product_type=product_type["ID"])).provider)
# parse jsonpath
product_type_collection = jsonpath_parse_dict_items(
collection_model,
{
"product_type":
product_type,
"provider":
self.eodag_api.providers_config[product_type_provider].
__dict__,
},
)
# parse f-strings
format_args = copy.deepcopy(self.stac_config)
format_args["collection"] = dict(
product_type_collection, **{
"url": self.url,
"root": self.root
})
product_type_collection = format_dict_items(
product_type_collection, **format_args)
collection_list.append(product_type_collection)
return collection_list
def get_stac_item_from_product(self, product):
"""Build STAC item from EODAG product
:param product: EODAG product
:type product: :class:`eodag.api.product._product.EOProduct`
:returns: STAC item
:rtype: list
"""
product_type = product.product_type
item_model = self.__filter_item_model_properties(
self.stac_config["item"], product_type)
catalog = StacCatalog(
url=self.url.split("/items")[0],
stac_config=self.stac_config,
root=self.root,
provider=self.provider,
eodag_api=self.eodag_api,
catalogs=[product_type],
)
# parse jsonpath
product_item = jsonpath_parse_dict_items(item_model,
{"product": product.__dict__})
# parse f-strings
format_args = copy.deepcopy(self.stac_config)
# format_args["collection"] = dict(catalog.as_dict(), **{"url": catalog.url})
format_args["catalog"] = dict(
catalog.as_dict(), **{
"url": catalog.url,
"root": catalog.root
})
format_args["item"] = product_item
product_item = format_dict_items(product_item, **format_args)
product_item["bbox"] = [float(i) for i in product_item["bbox"]]
# remove empty properties
product_item = self.__filter_item_properties_values(product_item)
self.update_data(product_item)
return self.as_dict()
def set_stac_cloud_cover_by_id(self, cloud_cover, **kwargs):
"""Updates and returns catalog with given max cloud_cover
:param cloud_cover: cloud_cover number
:type cloud_cover: str
:returns: updated catalog
:rtype: dict
"""
cat_model = copy.deepcopy(
self.stac_config["catalogs"]["cloud_cover"]["model"])
# parse f-strings
format_args = copy.deepcopy(self.stac_config)
format_args["catalog"] = defaultdict(str, **self.data)
format_args["cloud_cover"] = cloud_cover
parsed_dict = format_dict_items(cat_model, **format_args)
self.update_data(parsed_dict)
# update search args
self.search_args.update({"cloudCover": cloud_cover})
return parsed_dict
def get_product_type_def_params(self, product_type, **kwargs):
"""Get the provider product type definition parameters"""
if product_type in self.config.products.keys():
logger.debug(
"Getting provider product type definition parameters for %s",
product_type,
)
return self.config.products[product_type]
elif GENERIC_PRODUCT_TYPE in self.config.products.keys():
logger.debug(
"Getting genric provider product type definition parameters for %s",
product_type,
)
return {
k: v
for k, v in format_dict_items(
self.config.products[GENERIC_PRODUCT_TYPE], **
kwargs).items() if v
}
else:
return {}
def set_stac_date(self, datetime_min, datetime_max, catalog_model):
"""Updates catalog data using given dates
:param datetime_min: date min of interval
:type datetime_min: :class:`datetime.datetime`
:param datetime_max: date max of interval
:type datetime_max: :class:`datetime.datetime`
:param catalog_model: catalog model to use, from yml stac_config[catalogs]
:type catalog_model: dict
:returns: updated catalog
:rtype: dict
"""
# parse f-strings
format_args = copy.deepcopy(self.stac_config)
format_args["catalog"] = defaultdict(str, **self.data)
format_args["date"] = defaultdict(
str,
{
"year": datetime_min.year,
"month": datetime_min.month,
"day": datetime_min.day,
"min": datetime_min.isoformat().replace("+00:00", "") + "Z",
"max": datetime_max.isoformat().replace("+00:00", "") + "Z",
},
)
parsed_dict = format_dict_items(catalog_model, **format_args)
self.update_data(parsed_dict)
# update search args
self.search_args.update({
"dtstart":
datetime_min.isoformat().split("T")[0],
"dtend":
datetime_max.isoformat().split("T")[0],
})
return parsed_dict
def query(self,
product_type=None,
items_per_page=None,
page=None,
count=True,
**kwargs):
"""Search for data on USGS catalogues
.. versionchanged::
2.2.0
* Based on usgs library v0.3.0 which now uses M2M API. The library
is used for both search & download
.. versionchanged::
1.0
* ``product_type`` is no longer mandatory
"""
product_type = kwargs.get("productType")
if product_type is None:
return [], 0
try:
api.login(
self.config.credentials["username"],
self.config.credentials["password"],
save=True,
)
except USGSError:
raise AuthenticationError(
"Please check your USGS credentials.") from None
product_type_def_params = self.config.products.get(
product_type, self.config.products[GENERIC_PRODUCT_TYPE])
usgs_dataset = format_dict_items(product_type_def_params,
**kwargs)["dataset"]
start_date = kwargs.pop("startTimeFromAscendingNode", None)
end_date = kwargs.pop("completionTimeFromAscendingNode", None)
geom = kwargs.pop("geometry", None)
footprint = {}
if hasattr(geom, "bounds"):
(
footprint["lonmin"],
footprint["latmin"],
footprint["lonmax"],
footprint["latmax"],
) = geom.bounds
else:
footprint = geom
final = []
if footprint and len(footprint.keys()) == 4: # a rectangle (or bbox)
lower_left = {
"longitude": footprint["lonmin"],
"latitude": footprint["latmin"],
}
upper_right = {
"longitude": footprint["lonmax"],
"latitude": footprint["latmax"],
}
else:
lower_left, upper_right = None, None
try:
results = api.scene_search(
usgs_dataset,
start_date=start_date,
end_date=end_date,
ll=lower_left,
ur=upper_right,
max_results=items_per_page,
starting_number=(1 + (page - 1) * items_per_page),
)
# Same method as in base.py, Search.__init__()
# Prepare the metadata mapping
# Do a shallow copy, the structure is flat enough for this to be sufficient
metas = DEFAULT_METADATA_MAPPING.copy()
# Update the defaults with the mapping value. This will add any new key
# added by the provider mapping that is not in the default metadata.
# A deepcopy is done to prevent self.config.metadata_mapping from being modified when metas[metadata]
# is a list and is modified
metas.update(copy.deepcopy(self.config.metadata_mapping))
metas = mtd_cfg_as_jsonpath(metas)
for result in results["data"]["results"]:
result["productType"] = usgs_dataset
product_properties = properties_from_json(result, metas)
final.append(
EOProduct(
productType=product_type,
provider=self.provider,
properties=product_properties,
geometry=footprint,
))
except USGSError as e:
logger.warning(
"Product type %s does not exist on USGS EE catalog",
usgs_dataset,
)
logger.warning("Skipping error: %s", e)
api.logout()
if final:
# parse total_results
path_parsed = parse(
self.config.pagination["total_items_nb_key_path"])
total_results = path_parsed.find(results["data"])[0].value
else:
total_results = 0
return final, total_results
def get_stac_items(self, search_results, catalog):
"""Build STAC items from EODAG search results
:param search_results: EODAG search results
:type search_results: :class:`eodag.api.search_result.SearchResult`
:param catalog: STAC catalog dict used for parsing item metadata
:type catalog: dict
:returns: STAC item dicts list
:rtype: list
:returns: items dictionnary
:rtype: dict
"""
items_model = copy.deepcopy(self.stac_config["items"])
search_results.numberMatched = search_results.properties[
"totalResults"]
search_results.numberReturned = len(search_results)
# next page link
if "?" in self.url:
# search endpoint: use page url as self link
for i, _ in enumerate(items_model["links"]):
if items_model["links"][i]["rel"] == "self":
items_model["links"][i]["href"] = catalog["url"]
if "page=" not in self.url:
search_results.next = "%s&page=%s" % (
self.url,
search_results.properties["page"] + 1,
)
else:
search_results.next = re.sub(
r"^(.*)(page=[0-9]+)(.*)$",
r"\1page=%s\3" % (search_results.properties["page"] + 1),
self.url,
)
else:
search_results.next = "%s?page=%s" % (
self.url,
search_results.properties["page"] + 1,
)
search_results.timeStamp = (datetime.datetime.now(
datetime.timezone.utc).isoformat().replace("+00:00", "") + "Z")
# parse jsonpath
items = jsonpath_parse_dict_items(
items_model, {"search_results": search_results.__dict__})
# parse f-strings
format_args = copy.deepcopy(self.stac_config)
format_args["catalog"] = catalog
items = format_dict_items(items, **format_args)
# last page: remove next page link
if (search_results.properties["itemsPerPage"] *
search_results.properties["page"] >=
search_results.properties["totalResults"]):
items["links"] = [
link for link in items["links"] if link["rel"] != "next"
]
# provide static catalog to build features
if "search?" in catalog["url"]:
catalog["url"] = os.path.join(
catalog["url"].split("search?")[0],
"collections",
catalog["id"],
)
else:
catalog["url"] = catalog["url"].split("?")[0]
items["features"] = self.__get_item_list(search_results, catalog)
self.update_data(items)
return self.as_dict()
