def normalize_results(self, results, *args, **kwargs):
"""Build EOProducts from provider results"""
logger.debug("Adapting plugin results to eodag product representation")
products = []
for result in results:
product = EOProduct(
self.provider,
QueryStringSearch.extract_properties[self.config.result_type](
result,
self.config.metadata_mapping,
discovery_pattern=getattr(self.config, "discover_metadata", {}).get(
"metadata_pattern", None
),
discovery_path=getattr(self.config, "discover_metadata", {}).get(
"metadata_path", "null"
),
),
*args,
**kwargs
)
# use product_type_config as default properties
product.properties = dict(
getattr(self.config, "product_type_config", {}), **product.properties
)
products.append(product)
return products
def from_geojson(feature_collection):
"""Builds an :class:`~eodag.api.search_result.SearchResult` object from its representation as geojson
:param feature_collection: A collection representing a search result.
:type feature_collection: dict
:returns: An eodag representation of a search result
:rtype: :class:`~eodag.api.search_result.SearchResult`
"""
return SearchResult(
EOProduct.from_geojson(feature)
for feature in feature_collection["features"])
def normalize_results(self, results, **kwargs):
"""Transform metadata from provider representation to eodag representation"""
normalized = []
logger.debug("Adapting plugin results to eodag product representation")
for result in results:
ref = result["properties"]["title"].split("_")[5]
year = result["properties"]["completionDate"][0:4]
month = str(int(result["properties"]["completionDate"][5:7]))
day = str(int(result["properties"]["completionDate"][8:10]))
properties = QueryStringSearch.extract_properties[
self.config.result_type](result, self.config.metadata_mapping)
properties["downloadLink"] = (
"s3://tiles/{ref[1]}{ref[2]}/{ref[3]}/{ref[4]}{ref[5]}/{year}/"
"{month}/{day}/0/").format(**locals())
normalized.append(EOProduct(self.provider, properties, **kwargs))
return normalized
def __build_product(self, rec, product_type, **kwargs):
"""Enable search results to be handled by http download plugin"""
download_url = ""
resource_filter = re.compile(
self.config.search_definition.get("resource_location_filter", ""))
for ref in rec.references:
if ref["scheme"] in SUPPORTED_REFERENCE_SCHEMES:
if resource_filter.pattern and resource_filter.search(
ref["url"]):
download_url = ref["url"]
else:
download_url = ref["url"] # noqa
break
properties = properties_from_xml(rec.xml, self.config.metadata_mapping)
if not properties["geometry"]:
bbox = rec.bbox_wgs84
if not bbox:
code = "EPSG:4326"
if rec.bbox.crs and rec.bbox.crs.code and rec.bbox.crs.code > 0:
code = ":".join(
(str(rec.bbox.crs.id), str(rec.bbox.crs.code)))
rec_proj = pyproj.Proj(init=code)
default_proj_as_pyproj = pyproj.Proj(DEFAULT_PROJ)
maxx, maxy = pyproj.transform(rec_proj, default_proj_as_pyproj,
rec.bbox.maxx, rec.bbox.maxy)
minx, miny = pyproj.transform(rec_proj, default_proj_as_pyproj,
rec.bbox.minx, rec.bbox.miny)
bbox = (minx, miny, maxx, maxy)
properties["geometry"] = geometry.box(*bbox)
# Ensure the geometry property is shapely-compatible (the geometry is assumed
# to be a wkt)
else:
properties["geometry"] = wkt.loads(properties["geometry"])
return EOProduct(
product_type,
self.provider,
# TODO: EOProduct has no more *args in its __init__ (search_args attribute removed)
# Not sure why download_url was here in the first place, needs to be updated,
# possibly by having instead 'downloadLink' in the properties
# download_url,
properties,
searched_bbox=kwargs.get("footprints"),
)
def query(self, product_type=None, **kwargs):
"""Search for data on USGS catalogues
.. versionchanged::
1.0
* ``product_type`` is no longer mandatory
"""
product_type = kwargs.get("productType")
if product_type is None:
return [], 0
api.login(
self.config.credentials["username"],
self.config.credentials["password"],
save=True,
)
usgs_dataset = self.config.products[product_type]["dataset"]
usgs_catalog_node = self.config.products[product_type]["catalog_node"]
start_date = kwargs.pop("startTimeFromAscendingNode", None)
end_date = kwargs.pop("completionTimeFromAscendingNode", None)
footprint = kwargs.pop("geometry", None)
# Configuration to generate the download url of search results
result_summary_pattern = re.compile(
r"^ID: .+, Acquisition Date: .+, Path: (?P<path>\d+), Row: (?P<row>\d+)$" # noqa
)
# See https://pyformat.info/, on section "Padding and aligning strings" to
# understand {path:0>3} and {row:0>3}.
# It roughly means: 'if the string that will be passed as "path" has length < 3,
# prepend as much "0"s as needed to reach length 3' and same for "row"
dl_url_pattern = "{base_url}/L8/{path:0>3}/{row:0>3}/{entity}.tar.bz"
final = []
if footprint and len(footprint.keys()) == 4: # a rectangle (or bbox)
lower_left = {
"longitude": footprint["lonmin"],
"latitude": footprint["latmin"],
}
upper_right = {
"longitude": footprint["lonmax"],
"latitude": footprint["latmax"],
}
else:
lower_left, upper_right = None, None
try:
results = api.search(
usgs_dataset,
usgs_catalog_node,
start_date=start_date,
end_date=end_date,
ll=lower_left,
ur=upper_right,
)
for result in results["data"]["results"]:
r_lower_left = result["spatialFootprint"]["coordinates"][0][0]
r_upper_right = result["spatialFootprint"]["coordinates"][0][2]
summary_match = result_summary_pattern.match(
result["summary"]).groupdict()
result["geometry"] = geometry.box(r_lower_left[0],
r_lower_left[1],
r_upper_right[0],
r_upper_right[1])
# Same method as in base.py, Search.__init__()
# Prepare the metadata mapping
# Do a shallow copy, the structure is flat enough for this to be sufficient
metas = DEFAULT_METADATA_MAPPING.copy()
# Update the defaults with the mapping value. This will add any new key
# added by the provider mapping that is not in the default metadata.
# A deepcopy is done to prevent self.config.metadata_mapping from being modified when metas[metadata]
# is a list and is modified
metas.update(copy.deepcopy(self.config.metadata_mapping))
metas = mtd_cfg_as_jsonpath(metas)
result["productType"] = usgs_dataset
product_properties = properties_from_json(result, metas)
if getattr(self.config, "product_location_scheme",
"https") == "file":
product_properties["downloadLink"] = dl_url_pattern.format(
base_url="file://")
else:
product_properties["downloadLink"] = dl_url_pattern.format(
base_url=self.config.google_base_url.rstrip("/"),
entity=result["entityId"],
**summary_match)
final.append(
EOProduct(
productType=product_type,
provider=self.provider,
properties=product_properties,
geometry=footprint,
))
except USGSError as e:
logger.debug(
"Product type %s does not exist on catalogue %s",
usgs_dataset,
usgs_catalog_node,
)
logger.debug("Skipping error: %s", e)
api.logout()
return final, len(final)
def query(self,
product_type=None,
items_per_page=None,
page=None,
count=True,
**kwargs):
"""Search for data on USGS catalogues
.. versionchanged::
2.2.0
* Based on usgs library v0.3.0 which now uses M2M API. The library
is used for both search & download
.. versionchanged::
1.0
* ``product_type`` is no longer mandatory
"""
product_type = kwargs.get("productType")
if product_type is None:
return [], 0
try:
api.login(
self.config.credentials["username"],
self.config.credentials["password"],
save=True,
)
except USGSError:
raise AuthenticationError(
"Please check your USGS credentials.") from None
product_type_def_params = self.config.products.get(
product_type, self.config.products[GENERIC_PRODUCT_TYPE])
usgs_dataset = format_dict_items(product_type_def_params,
**kwargs)["dataset"]
start_date = kwargs.pop("startTimeFromAscendingNode", None)
end_date = kwargs.pop("completionTimeFromAscendingNode", None)
geom = kwargs.pop("geometry", None)
footprint = {}
if hasattr(geom, "bounds"):
(
footprint["lonmin"],
footprint["latmin"],
footprint["lonmax"],
footprint["latmax"],
) = geom.bounds
else:
footprint = geom
final = []
if footprint and len(footprint.keys()) == 4: # a rectangle (or bbox)
lower_left = {
"longitude": footprint["lonmin"],
"latitude": footprint["latmin"],
}
upper_right = {
"longitude": footprint["lonmax"],
"latitude": footprint["latmax"],
}
else:
lower_left, upper_right = None, None
try:
results = api.scene_search(
usgs_dataset,
start_date=start_date,
end_date=end_date,
ll=lower_left,
ur=upper_right,
max_results=items_per_page,
starting_number=(1 + (page - 1) * items_per_page),
)
# Same method as in base.py, Search.__init__()
# Prepare the metadata mapping
# Do a shallow copy, the structure is flat enough for this to be sufficient
metas = DEFAULT_METADATA_MAPPING.copy()
# Update the defaults with the mapping value. This will add any new key
# added by the provider mapping that is not in the default metadata.
# A deepcopy is done to prevent self.config.metadata_mapping from being modified when metas[metadata]
# is a list and is modified
metas.update(copy.deepcopy(self.config.metadata_mapping))
metas = mtd_cfg_as_jsonpath(metas)
for result in results["data"]["results"]:
result["productType"] = usgs_dataset
product_properties = properties_from_json(result, metas)
final.append(
EOProduct(
productType=product_type,
provider=self.provider,
properties=product_properties,
geometry=footprint,
))
except USGSError as e:
logger.warning(
"Product type %s does not exist on USGS EE catalog",
usgs_dataset,
)
logger.warning("Skipping error: %s", e)
api.logout()
if final:
# parse total_results
path_parsed = parse(
self.config.pagination["total_items_nb_key_path"])
total_results = path_parsed.find(results["data"])[0].value
else:
total_results = 0
return final, total_results