def __init__(self, provider, config): super(Search, self).__init__(provider, config) # Prepare the metadata mapping # Do a shallow copy, the structure is flat enough for this to be sufficient metas = DEFAULT_METADATA_MAPPING.copy() # Update the defaults with the mapping value. This will add any new key # added by the provider mapping that is not in the default metadata metas.update(self.config.metadata_mapping) self.config.metadata_mapping = mtd_cfg_as_jsonpath( metas, self.config.metadata_mapping)
def __filter_item_model_properties(self, item_model, product_type): """Filter item model depending on product type metadata and its extensions. Removes not needed parameters, and adds supplementary ones as part of eodag extension. :param item_model: item model from stac_config :type item_model: dict :param product_type: product type :type product_type: str :returns: filtered item model :rtype: dict """ try: product_type_dict = [ pt for pt in self.eodag_api.list_product_types( provider=self.provider) if pt["ID"] == product_type ][0] except IndexError: raise MisconfiguredError( "Product type {} not available for {}".format( product_type, self.provider)) result_item_model = copy.deepcopy(item_model) if product_type_dict["sensorType"] != "RADAR": result_item_model["stac_extensions"].remove("sar") extensions_names = self.get_stac_extensions_dict( result_item_model["stac_extensions"]).keys() for k, v in item_model["properties"].items(): # remove key if extension not in stac_extensions if ":" in k and k.split(":")[0] not in extensions_names: result_item_model["properties"].pop(k, None) # build jsonpath for eodag product properties and adapt path eodag_properties_dict = { k: string_to_jsonpath(k, v.replace("$.", "$.product.")) for k, v in DEFAULT_METADATA_MAPPING.items() if "$.properties." in v } # add missing properties as oseo:missingProperty for k, v in eodag_properties_dict.items(): if (v not in result_item_model["properties"].values() and k not in self.stac_config["metadata_ignore"]): result_item_model["properties"]["oseo:" + k] = string_to_jsonpath(k, v) return result_item_model
def query(self, product_type=None, **kwargs): """Search for data on USGS catalogues .. versionchanged:: 1.0 * ``product_type`` is no longer mandatory """ product_type = kwargs.get("productType") if product_type is None: return [], 0 api.login( self.config.credentials["username"], self.config.credentials["password"], save=True, ) usgs_dataset = self.config.products[product_type]["dataset"] usgs_catalog_node = self.config.products[product_type]["catalog_node"] start_date = kwargs.pop("startTimeFromAscendingNode", None) end_date = kwargs.pop("completionTimeFromAscendingNode", None) footprint = kwargs.pop("geometry", None) # Configuration to generate the download url of search results result_summary_pattern = re.compile( r"^ID: .+, Acquisition Date: .+, Path: (?P<path>\d+), Row: (?P<row>\d+)$" # noqa ) # See https://pyformat.info/, on section "Padding and aligning strings" to # understand {path:0>3} and {row:0>3}. # It roughly means: 'if the string that will be passed as "path" has length < 3, # prepend as much "0"s as needed to reach length 3' and same for "row" dl_url_pattern = "{base_url}/L8/{path:0>3}/{row:0>3}/{entity}.tar.bz" final = [] if footprint and len(footprint.keys()) == 4: # a rectangle (or bbox) lower_left = { "longitude": footprint["lonmin"], "latitude": footprint["latmin"], } upper_right = { "longitude": footprint["lonmax"], "latitude": footprint["latmax"], } else: lower_left, upper_right = None, None try: results = api.search( usgs_dataset, usgs_catalog_node, start_date=start_date, end_date=end_date, ll=lower_left, ur=upper_right, ) for result in results["data"]["results"]: r_lower_left = result["spatialFootprint"]["coordinates"][0][0] r_upper_right = result["spatialFootprint"]["coordinates"][0][2] summary_match = result_summary_pattern.match( result["summary"]).groupdict() result["geometry"] = geometry.box(r_lower_left[0], r_lower_left[1], r_upper_right[0], r_upper_right[1]) # Same method as in base.py, Search.__init__() # Prepare the metadata mapping # Do a shallow copy, the structure is flat enough for this to be sufficient metas = DEFAULT_METADATA_MAPPING.copy() # Update the defaults with the mapping value. This will add any new key # added by the provider mapping that is not in the default metadata. # A deepcopy is done to prevent self.config.metadata_mapping from being modified when metas[metadata] # is a list and is modified metas.update(copy.deepcopy(self.config.metadata_mapping)) metas = mtd_cfg_as_jsonpath(metas) result["productType"] = usgs_dataset product_properties = properties_from_json(result, metas) if getattr(self.config, "product_location_scheme", "https") == "file": product_properties["downloadLink"] = dl_url_pattern.format( base_url="file://") else: product_properties["downloadLink"] = dl_url_pattern.format( base_url=self.config.google_base_url.rstrip("/"), entity=result["entityId"], **summary_match) final.append( EOProduct( productType=product_type, provider=self.provider, properties=product_properties, geometry=footprint, )) except USGSError as e: logger.debug( "Product type %s does not exist on catalogue %s", usgs_dataset, usgs_catalog_node, ) logger.debug("Skipping error: %s", e) api.logout() return final, len(final)
def query(self, product_type=None, items_per_page=None, page=None, count=True, **kwargs): """Search for data on USGS catalogues .. versionchanged:: 2.2.0 * Based on usgs library v0.3.0 which now uses M2M API. The library is used for both search & download .. versionchanged:: 1.0 * ``product_type`` is no longer mandatory """ product_type = kwargs.get("productType") if product_type is None: return [], 0 try: api.login( self.config.credentials["username"], self.config.credentials["password"], save=True, ) except USGSError: raise AuthenticationError( "Please check your USGS credentials.") from None product_type_def_params = self.config.products.get( product_type, self.config.products[GENERIC_PRODUCT_TYPE]) usgs_dataset = format_dict_items(product_type_def_params, **kwargs)["dataset"] start_date = kwargs.pop("startTimeFromAscendingNode", None) end_date = kwargs.pop("completionTimeFromAscendingNode", None) geom = kwargs.pop("geometry", None) footprint = {} if hasattr(geom, "bounds"): ( footprint["lonmin"], footprint["latmin"], footprint["lonmax"], footprint["latmax"], ) = geom.bounds else: footprint = geom final = [] if footprint and len(footprint.keys()) == 4: # a rectangle (or bbox) lower_left = { "longitude": footprint["lonmin"], "latitude": footprint["latmin"], } upper_right = { "longitude": footprint["lonmax"], "latitude": footprint["latmax"], } else: lower_left, upper_right = None, None try: results = api.scene_search( usgs_dataset, start_date=start_date, end_date=end_date, ll=lower_left, ur=upper_right, max_results=items_per_page, starting_number=(1 + (page - 1) * items_per_page), ) # Same method as in base.py, Search.__init__() # Prepare the metadata mapping # Do a shallow copy, the structure is flat enough for this to be sufficient metas = DEFAULT_METADATA_MAPPING.copy() # Update the defaults with the mapping value. This will add any new key # added by the provider mapping that is not in the default metadata. # A deepcopy is done to prevent self.config.metadata_mapping from being modified when metas[metadata] # is a list and is modified metas.update(copy.deepcopy(self.config.metadata_mapping)) metas = mtd_cfg_as_jsonpath(metas) for result in results["data"]["results"]: result["productType"] = usgs_dataset product_properties = properties_from_json(result, metas) final.append( EOProduct( productType=product_type, provider=self.provider, properties=product_properties, geometry=footprint, )) except USGSError as e: logger.warning( "Product type %s does not exist on USGS EE catalog", usgs_dataset, ) logger.warning("Skipping error: %s", e) api.logout() if final: # parse total_results path_parsed = parse( self.config.pagination["total_items_nb_key_path"]) total_results = path_parsed.find(results["data"])[0].value else: total_results = 0 return final, total_results