def group_by_extent(searches):
"""Combines multiple SearchResults and return a list of SearchResults grouped
by extent (i.e. bounding box).
:param searches: List of eodag SearchResult
:type searches: list
:return: list of :class:`~eodag.api.search_result.SearchResult`
.. versionchanged::
2.0
* Renamed from sort_by_extent to group_by_extent to reflect its
actual behaviour.
"""
# Dict with extents as keys, each extent being defined by a str
# "{minx}{miny}{maxx}{maxy}" (each float rounded to 2 dec).
products_grouped_by_extent = {}
for search in searches:
for product in search:
same_geom = products_grouped_by_extent.setdefault(
"".join(
[str(round(p, 2)) for p in product.geometry.bounds]),
[])
same_geom.append(product)
return [
SearchResult(products_grouped_by_extent[extent_as_str])
for extent_as_str in products_grouped_by_extent
]
def _search_by_id(self, uid, provider=None):
"""Internal method that enables searching a product by its id.
Keeps requesting providers until a result matching the id is supplied. The
search plugins should be developed in the way that enable them to handle the
support of a search by id by the providers. The providers are requested one by
one, in the order defined by their priorities. Be aware that because of that,
the search can be slow, if the priority order is such that the provider that
contains the requested product has the lowest priority. However, you can always
speed up a little the search by passing the name of the provider on which to
perform the search, if this information is available
:param uid: The uid of the EO product
:type uid: str
:param provider: (optional) The provider on which to search the product.
This may be useful for performance reasons when the user
knows this product is available on the given provider
:type provider: str
:returns: A search result with one EO product or None at all, and the number
of EO products retrieved (0 or 1)
:rtype: tuple(:class:`~eodag.api.search_result.SearchResult`, int)
.. versionadded:: 1.0
"""
for plugin in self._plugins_manager.get_search_plugins(
provider=provider):
logger.info("Searching product with id '%s' on provider: %s", uid,
plugin.provider)
logger.debug("Using plugin class for search: %s",
plugin.__class__.__name__)
auth = self._plugins_manager.get_auth_plugin(plugin.provider)
results, _ = self._do_search(plugin, auth=auth, id=uid)
if len(results) == 1:
return results, 1
return SearchResult([]), 0
def deserialize(filename):
"""Loads results of a search from a geojson file.
:param filename: A filename containing a search result encoded as a geojson
:type filename: str
:returns: The search results encoded in `filename`
:rtype: :class:`~eodag.api.search_result.SearchResult`
"""
with open(filename, "r") as fh:
return SearchResult.from_geojson(geojson.load(fh))
def download(self,
product,
auth=None,
progress_callback=None,
**kwargs) -> str:
"""
Download product.
:param product: (EOProduct) EOProduct
:param auth: Not used, just here for compatibility reasons
:param progress_callback: Not used, just here for compatibility reasons
:param kwargs: Not used, just here for compatibility reasons
:return: Downloaded product path
"""
prods = self.download_all(SearchResult([
product,
]), auth, progress_callback, **kwargs)
# Manage the case if nothing has been downloaded
return prods[0] if len(prods) > 0 else ""
def sort_by_extent(searches):
"""Combines multiple SearchResults and return a list of SearchResults sorted
by extent.
:param searches: List of eodag SearchResult
:type searches: list
:return: list of :class:`~eodag.api.search_result.SearchResult`
"""
products_grouped_by_extent = {}
for search in searches:
for product in search:
same_geom = products_grouped_by_extent.setdefault(
"".join([str(round(p, 2)) for p in product.geometry.bounds]), []
)
same_geom.append(product)
return [
SearchResult(products_grouped_by_extent[extent_as_wkb_hex])
for extent_as_wkb_hex in products_grouped_by_extent
]
def _do_search(self, search_plugin, **kwargs):
"""Internal method that performs a search on a given provider.
.. versionadded:: 1.0
"""
results = SearchResult([])
total_results = 0
try:
res, nb_res = search_plugin.query(**kwargs)
# Only do the pagination computations when it makes sense. For example,
# for a search by id, we can reasonably guess that the provider will return
# At most 1 product, so we don't need such a thing as pagination
page = kwargs.get("page")
items_per_page = kwargs.get("items_per_page")
if page and items_per_page:
# Take into account the fact that a provider may not return the count of
# products (in that case, fallback to using the length of the results it
# returned and the page requested. As an example, check the result of
# the following request (look for the value of properties.totalResults)
# https://theia-landsat.cnes.fr/resto/api/collections/Landsat/search.json?
# maxRecords=1&page=1
if nb_res == 0:
nb_res = len(res) * page
# Attempt to ensure a little bit more coherence. Some providers return
# a fuzzy number of total results, meaning that you have to keep
# requesting it until it has returned everything it has to know exactly
# how many EO products they have in their stock. In that case, we need
# to replace the returned number of results with the sum of the number
# of items that were skipped so far and the length of the currently
# retrieved items. We know there is an incoherence when the number of
# skipped items is greater than the total number of items returned by
# the plugin
nb_skipped_items = items_per_page * (page - 1)
nb_current_items = len(res)
if nb_skipped_items > nb_res:
if nb_res != 0:
nb_res = nb_skipped_items + nb_current_items
# This is for when the returned results is an empty list and the
# number of results returned is incoherent with the observations.
# In that case, we assume the total number of results is the number
# of skipped results. By requesting a lower page than the current
# one, a user can iteratively reach the last page of results for
# these criteria on the provider.
else:
nb_res = nb_skipped_items
if not isinstance(res, list):
raise PluginImplementationError(
"The query function of a Search plugin must return a list of "
"results, got {} instead".format(type(res)))
# Filter and attach to each eoproduct in the result the plugin capable of
# downloading it (this is done to enable the eo_product to download itself
# doing: eo_product.download()). The filtering is done by keeping only
# those eo_products that intersects the search extent (if there was no
# search extent, search_intersection contains the geometry of the
# eo_product)
# WARNING: this means an eo_product that has an invalid geometry can still
# be returned as a search result if there was no search extent (because we
# will not try to do an intersection)
for eo_product in res:
if eo_product.search_intersection is not None:
download_plugin = self._plugins_manager.get_download_plugin(
eo_product)
eo_product.register_downloader(download_plugin,
kwargs.get("auth", None))
results.extend(res)
total_results += nb_res
logger.info("Found %s result(s) on provider '%s'", nb_res,
search_plugin.provider)
# Hitting for instance
# https://theia.cnes.fr/atdistrib/resto2/api/collections/SENTINEL2/
# search.json?startDate=2019-03-01&completionDate=2019-06-15
# &processingLevel=LEVEL2A&maxRecords=1&page=1
# returns a number (properties.totalResults) that is the number of
# products in the collection (here SENTINEL2) instead of the estimated
# total number of products matching the search criteria (start/end date).
# Remove this warning when this is fixed upstream by THEIA.
if search_plugin.provider == "theia":
logger.warning(
"Results found on provider 'theia' is the total number of products "
"available in the searched collection (e.g. SENTINEL2) instead of "
"the total number of products matching the search criteria"
)
except Exception:
logger.info(
"No result from provider '%s' due to an error during search. Raise "
"verbosity of log messages for details",
search_plugin.provider,
)
if kwargs.get("raise_errors"):
# Raise the error, letting the application wrapping eodag know that
# something went bad. This way it will be able to decide what to do next
raise
else:
logger.exception(
"Error while searching on provider %s (ignored):",
search_plugin.provider,
)
return SearchResult(results), total_results
def query(self, items_per_page=None, page=None, count=True, **kwargs):
"""Perform a search on a static STAC Catalog"""
features = fetch_stac_items(
self.config.api_endpoint,
recursive=True,
max_connections=self.config.max_connections,
timeout=self.config.timeout,
)
nb_features = len(features)
feature_collection = geojson.FeatureCollection(features)
feature_collection["context"] = {
"limit": nb_features,
"matched": nb_features,
"returned": nb_features,
}
# save StaticStacSearch._request and mock it to make return loaded static results
stacapi_request = self._request
self._request = lambda url, info_message=None, exception_message=None: MockResponse(
feature_collection, 200
)
# query on mocked StacSearch
eo_products, _ = super(StaticStacSearch, self).query(
items_per_page=nb_features, page=1, count=True, **kwargs
)
# filter using query params
search_result = SearchResult(eo_products)
# Filter by date
if "startTimeFromAscendingNode" in kwargs:
kwargs["start"] = kwargs.pop("startTimeFromAscendingNode")
if "completionTimeFromAscendingNode" in kwargs:
kwargs["end"] = kwargs.pop("completionTimeFromAscendingNode")
if any(k in ["start", "end"] for k in kwargs.keys()):
search_result = search_result.crunch(
FilterDate({k: kwargs[k] for k in ["start", "end"] if k in kwargs})
)
# Filter by geometry
if "geometry" in kwargs.keys():
search_result = search_result.crunch(
FilterOverlap({"intersects": True}), geometry=kwargs.pop("geometry")
)
# Filter by cloudCover
if "cloudCover" in kwargs.keys():
search_result = search_result.crunch(
FilterProperty(
{"cloudCover": kwargs.pop("cloudCover"), "operator": "lt"}
)
)
# Filter by other properties
skip_eodag_internal_parameters = [
"auth",
"raise_errors",
"productType",
"locations",
"start",
"end",
"geom",
]
for property_key, property_value in kwargs.items():
if property_key not in skip_eodag_internal_parameters:
search_result = search_result.crunch(
FilterProperty({property_key: property_value, "operator": "eq"})
)
# restore plugin._request
self._request = stacapi_request
return search_result.data, len(search_result)
def search(
self,
page=DEFAULT_PAGE,
items_per_page=DEFAULT_ITEMS_PER_PAGE,
raise_errors=False,
start=None,
end=None,
box=None,
**kwargs
):
"""Look for products matching criteria on known providers.
The default behaviour is to look for products on the provider with the
highest priority supporting the requested product type. These priorities
are configurable through user configuration file or individual
environment variable.
:param page: The page number to return (default: 1)
:type page: int
:param items_per_page: The number of results that must appear in one single
page (default: 20)
:type items_per_page: int
:param raise_errors: When an error occurs when searching, if this is set to
True, the error is raised (default: False)
:type raise_errors: bool
:param start: Start sensing time in iso format
:type start: str or unicode
:param end: End sensing time in iso format
:type end: str or unicode
:param box: A bounding box delimiting the AOI (as a dict with keys: "lonmin",
"latmin", "lonmax", "latmax")
:type box: dict
:param dict kwargs: some other criteria that will be used to do the search,
using paramaters compatibles with the provider
:returns: A collection of EO products matching the criteria and the total
number of results found
:rtype: tuple(:class:`~eodag.api.search_result.SearchResult`, int)
.. versionchanged::
1.6
* Any search parameter supported by the provider can be passed as
kwargs. Each provider has a 'discover_metadata' configuration
with a metadata_pattern (to which the parameter must match) and a
search_param setting, defining the way the query will be built.
.. versionchanged::
1.0
* The ``product_type`` parameter is no longer mandatory
* Support new search parameters compliant with OpenSearch
* Fails if a suitable product type could not be guessed (returns an
empty search result) and if the user is not querying a specific
product (by providing ``id`` as search parameter)
* A search by ID is now performed if a product type can not be guessed
from the user input, and if the user has provided an ID as a search
criteria (in the keyword arguments)
* It is now possible to pass in the name of the provider on which to
perform the request when searching a product by its ID,
for performance reasons. In that case, the search with the product ID
will be done directly on that provider
.. versionchanged::
0.7.1
* The search now stops at the first provider that supports the
requested product type (removal of the partial mechanism)
* The method now returns a tuple of 2 elements, the first one
being the result and the second one being the total number of
results satisfying the criteria on the provider. This is useful
for applications wrapping eodag and wishing to implement
pagination. An example of this kind of application is the
embedded eodag HTTP server. Also useful for informational purposes
in the CLI: the user is informed about the total number of results
available, and can ask for retrieving a given number of these
results (See the help message of the CLI for more information).
.. note::
The search interfaces, which are implemented as plugins, are required to
return a list as a result of their processing. This requirement is
enforced here.
"""
product_type = kwargs.get("productType", None)
if product_type is None:
try:
guesses = self.guess_product_type(**kwargs)
# By now, only use the best bet
product_type = guesses[0]
except NoMatchingProductType:
queried_id = kwargs.get("id", None)
if queried_id is None:
logger.error("Unable to satisfy search query: %s", kwargs)
logger.error(
"No product type could be guessed with provided arguments"
)
else:
provider = kwargs.get("provider", None)
return self._search_by_id(kwargs["id"], provider=provider)
return SearchResult([]), 0
kwargs["productType"] = product_type
if start is not None:
kwargs["startTimeFromAscendingNode"] = start
if end is not None:
kwargs["completionTimeFromAscendingNode"] = end
if box is not None:
kwargs["geometry"] = box
plugin = next(
self._plugins_manager.get_search_plugins(product_type=product_type)
)
logger.info(
"Searching product type '%s' on provider: %s", product_type, plugin.provider
)
# add product_types_config to plugin config
plugin.config.product_type_config = dict(
[
p
for p in self.list_product_types(plugin.provider)
if p["ID"] == product_type
][0],
**{"productType": product_type}
)
plugin.config.product_type_config.pop("ID", None)
logger.debug("Using plugin class for search: %s", plugin.__class__.__name__)
auth = self._plugins_manager.get_auth_plugin(plugin.provider)
return self._do_search(
plugin,
auth=auth,
page=page,
items_per_page=items_per_page,
raise_errors=raise_errors,
**kwargs
)
def search_products(product_type, arguments):
"""Returns product search results
:param product_type: the product type criteria
:type product_type: str
:param arguments: filter criteria
:type arguments: dict
:return: search result
:rtype serialized GeoJSON response"""
try:
page, items_per_page = get_pagination_info(arguments)
geom = get_geometry(arguments)
criteria = {
"geom":
geom,
"startTimeFromAscendingNode":
get_date(arguments.pop("dtstart", None)),
"completionTimeFromAscendingNode":
get_date(arguments.pop("dtend", None)),
"cloudCover":
get_int(arguments.pop("cloudCover", None)),
"productType":
product_type,
}
if "id" in arguments.keys():
criteria["id"] = arguments["id"]
if items_per_page is None:
items_per_page = DEFAULT_ITEMS_PER_PAGE
if page is None:
page = DEFAULT_PAGE
unserialized = arguments.pop("unserialized", None)
arguments.pop("product_type", None)
products, total = eodag_api.search(
page=page,
items_per_page=items_per_page,
raise_errors=True,
**dict(criteria, **arguments),
)
products = filter_products(products, arguments, **criteria)
if not unserialized:
response = SearchResult(products).as_geojson_object()
response.update({
"properties": {
"page": page,
"itemsPerPage": items_per_page,
"totalResults": total,
}
})
else:
response = SearchResult(products)
response.properties = {
"page": page,
"itemsPerPage": items_per_page,
"totalResults": total,
}
except ValidationError as e:
raise e
except RuntimeError as e:
raise e
except UnsupportedProductType as e:
raise e
return response