def _get_oidc_provider(self, provider_id: Union[str, None] = None) -> Tuple[str, OidcProviderInfo]:
"""
Get OpenID Connect discovery URL for given provider_id
:param provider_id: id of OIDC provider as specified by backend (/credentials/oidc).
Can be None if there is just one provider.
:return: updated provider_id and provider info object
"""
if self._api_version.at_least("1.0.0"):
oidc_info = self.get("/credentials/oidc", expected_status=200).json()
providers = {p["id"]: p for p in oidc_info["providers"]}
_log.info("Found OIDC providers: {p}".format(p=list(providers.keys())))
if provider_id:
if provider_id not in providers:
raise OpenEoClientException("Requested provider {r!r} not available. Should be one of {p}.".format(
r=provider_id, p=list(providers.keys()))
)
provider = providers[provider_id]
elif len(providers) == 1:
# No provider id given, but there is only one anyway: we can handle that.
provider_id, provider = providers.popitem()
else:
raise OpenEoClientException("No provider_id given. Available: {p!r}.".format(
p=list(providers.keys()))
)
provider = OidcProviderInfo(issuer=provider["issuer"], scopes=provider.get("scopes"))
else:
# Per spec: '/credentials/oidc' will redirect to OpenID Connect discovery document
provider = OidcProviderInfo(discovery_url=self.build_url('/credentials/oidc'))
return provider_id, provider
def download_results(self,
target: Union[str, Path] = None) -> Dict[Path, dict]:
"""
Download job results into given folder (current working dir by default).
The names of the files are taken directly from the backend.
:param target: String/path, folder where to put the result files.
:return: file_list: Dict containing the downloaded file path as value and asset metadata
"""
target = Path(target or Path.cwd())
if target.exists() and not target.is_dir():
raise OpenEoClientException(
"The target argument must be a folder. Got {t!r}".format(
t=str(target)))
assets = {
target / f: m
for (f, m) in self._download_get_assets().items()
}
if len(assets) == 0:
raise OpenEoClientException(
"Expected at least one result file to download, but got 0.")
for path, metadata in assets.items():
self._download_url(metadata["href"], path)
return assets
def download_files(self, target: Union[str, Path] = None) -> Dict[Path, dict]:
target = Path(target or Path.cwd())
if target.exists() and not target.is_dir():
raise OpenEoClientException("The target argument must be a folder. Got {t!r}".format(t=str(target)))
assets = {target / f: m for (f, m) in self._get_assets().items()}
if len(assets) == 0:
raise OpenEoClientException("Expected at least one result file to download, but got 0.")
for path, metadata in assets.items():
self._download_url(metadata["href"], path)
return assets
def _get_oidc_provider(
self,
provider_id: Union[str,
None] = None) -> Tuple[str, OidcProviderInfo]:
"""
Get OpenID Connect discovery URL for given provider_id
:param provider_id: id of OIDC provider as specified by backend (/credentials/oidc).
Can be None if there is just one provider.
:return: updated provider_id and provider info object
"""
if self._api_version.at_least("1.0.0"):
oidc_info = self.get("/credentials/oidc",
expected_status=200).json()
providers = {p["id"]: p for p in oidc_info["providers"]}
_log.info(
"Found OIDC providers: {p}".format(p=list(providers.keys())))
if provider_id:
if provider_id not in providers:
raise OpenEoClientException(
"Requested OIDC provider {r!r} not available. Should be one of {p}."
.format(r=provider_id, p=list(providers.keys())))
provider = providers[provider_id]
elif len(providers) == 1:
provider_id, provider = providers.popitem()
_log.info(
"No OIDC provider given, but only one available: {p!r}. Use that one."
.format(p=provider_id))
else:
# Check if there is a single provider in the config to use.
backend = self._orig_url
provider_configs = self._get_auth_config(
).get_oidc_provider_configs(backend=backend)
intersection = set(provider_configs.keys()).intersection(
providers.keys())
if len(intersection) == 1:
provider_id = intersection.pop()
provider = providers[provider_id]
_log.info(
"No OIDC provider id given, but only one in config (backend {b!r}): {p!r}."
" Use that one.".format(b=backend, p=provider_id))
else:
raise OpenEoClientException(
"No OIDC provider id given. Pick one from: {p!r}.".
format(p=list(providers.keys())))
provider = OidcProviderInfo.from_dict(provider)
else:
# Per spec: '/credentials/oidc' will redirect to OpenID Connect discovery document
provider = OidcProviderInfo(
discovery_url=self.build_url('/credentials/oidc'))
return provider_id, provider
def get_asset(self, name: str = None) -> ResultAsset:
"""Get single asset by name or without name if there is only one."""
# TODO: also support getting a single asset by type or role?
assets = self.get_assets()
if len(assets) == 0:
raise OpenEoClientException("No assets in result.")
if name in assets:
return assets[name]
elif name is None and len(assets) == 1:
return assets.popitem()[1]
else:
raise OpenEoClientException(
"Failed to get single asset (name {n!r}) from {a}".format(
n=name, a=list(assets.keys())))
def download_files(self,
target: Union[Path, str] = None,
include_stac_metadata: bool = True) -> List[Path]:
"""
Download all assets to given folder.
:param target: path to folder to download to (must be a folder if it already exists)
:param include_stac_metadata: whether to download the job result metadata as a STAC (JSON) file.
:return: list of paths to the downloaded assets.
"""
target = Path(target or Path.cwd())
if target.exists() and not target.is_dir():
raise OpenEoClientException(
f"Target argument {target} exists but isn't a folder.")
ensure_dir(target)
downloaded = [a.download(target) for a in self.get_assets()]
if include_stac_metadata:
# TODO #184: convention for metadata file name?
metadata_file = target / "job-results.json"
# TODO #184: rewrite references to locally downloaded assets?
metadata_file.write_text(json.dumps(self.get_metadata()))
downloaded.append(metadata_file)
return downloaded
def authenticate_oidc_refresh_token(
self,
client_id: str,
refresh_token: str = None,
client_secret: str = None,
provider_id: str = None) -> 'Connection':
"""
OpenId Connect Refresh Token
WARNING: this API is in experimental phase
"""
provider_id, provider = self._get_oidc_provider(provider_id)
if refresh_token is None:
store = RefreshTokenStore()
# TODO: allow client_id/secret to be None and fetch it from config/cache?
refresh_token = store.get(issuer=provider.issuer,
client_id=client_id)
if refresh_token is None:
raise OpenEoClientException("No refresh token")
client_info = OidcClientInfo(client_id=client_id,
provider=provider,
client_secret=client_secret)
authenticator = OidcRefreshTokenAuthenticator(
client_info=client_info, refresh_token=refresh_token)
return self._authenticate_oidc(authenticator, provider_id=provider_id)
def create_job(self, process_graph: dict, title: str = None, description: str = None,
plan: str = None, budget=None,
additional: Dict = None) -> RESTJob:
"""
Posts a job to the back end.
:param process_graph: (flat) dict representing process graph
:param title: String title of the job
:param description: String description of the job
:param plan: billing plan
:param budget: Budget
:param additional: additional job options to pass to the backend
:return: job_id: String Job id of the new created job
"""
# TODO move all this (RESTJob factory) logic to RESTJob?
req = self._build_request_with_process_graph(
process_graph=process_graph,
title=title, description=description, plan=plan, budget=budget
)
if additional:
# TODO: get rid of this non-standard field? https://github.com/Open-EO/openeo-api/issues/276
req["job_options"] = additional
response = self.post("/jobs", json=req, expected_status=201)
if "openeo-identifier" in response.headers:
job_id = response.headers['openeo-identifier']
elif "location" in response.headers:
_log.warning("Backend did not explicitly respond with job id, will guess it from redirect URL.")
job_id = response.headers['location'].split("/")[-1]
else:
raise OpenEoClientException("Failed fo extract job id")
return RESTJob(job_id, self)
def request(self, method: str, path: str, headers: dict = None, auth: AuthBase = None,
check_error=True, expected_status=None, **kwargs):
"""Generic request send"""
url = self.build_url(path)
auth = auth or self.auth
if _log.isEnabledFor(logging.DEBUG):
_log.debug("Request `{m} {u}` with headers {h}, auth {a}, kwargs {k}".format(
m=method.upper(), u=url, h=headers and headers.keys(), a=type(auth).__name__, k=list(kwargs.keys()))
)
resp = self.session.request(
method=method,
url=url,
headers=self._merged_headers(headers),
auth=auth,
timeout=kwargs.pop("timeout", self.default_timeout),
**kwargs
)
# Check for API errors and unexpected HTTP status codes as desired.
status = resp.status_code
expected_status = ensure_list(expected_status) if expected_status else []
if check_error and status >= 400 and status not in expected_status:
self._raise_api_error(resp)
if expected_status and status not in expected_status:
raise OpenEoClientException("Got status code {s!r} for `{m} {p}` (expected {e!r})".format(
m=method.upper(), p=path, s=status, e=expected_status)
)
return resp
def _get_oidc_provider_and_client_info(
self, provider_id: str,
client_id: Union[str, None], client_secret: Union[str, None]
) -> Tuple[str, OidcClientInfo]:
"""
Resolve provider_id and client info (as given or from config)
:param provider_id: id of OIDC provider as specified by backend (/credentials/oidc).
Can be None if there is just one provider.
:return: (client_id, client_secret)
"""
provider_id, provider = self._get_oidc_provider(provider_id)
if client_id is None:
client_id, client_secret = self._get_auth_config().get_oidc_client_configs(
backend=self._orig_url, provider_id=provider_id
)
_log.info("Using client_id {c!r} from config (provider {p!r})".format(c=client_id, p=provider_id))
if client_id is None:
raise OpenEoClientException("No client ID found.")
client_info = OidcClientInfo(client_id=client_id, client_secret=client_secret, provider=provider)
return provider_id, client_info
def authenticate_oidc_refresh_token(
self,
client_id: str = None,
refresh_token: str = None,
client_secret: str = None,
provider_id: str = None) -> 'Connection':
"""
OpenId Connect Refresh Token
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id,
client_id=client_id,
client_secret=client_secret,
default_client_grant_types=[DefaultOidcClientGrant.REFRESH_TOKEN],
)
if refresh_token is None:
refresh_token = self._get_refresh_token_store().get_refresh_token(
issuer=client_info.provider.issuer,
client_id=client_info.client_id)
if refresh_token is None:
raise OpenEoClientException("No refresh token given or found")
authenticator = OidcRefreshTokenAuthenticator(
client_info=client_info, refresh_token=refresh_token)
return self._authenticate_oidc(authenticator, provider_id=provider_id)
def authenticate_oidc_refresh_token(
self,
client_id: str = None,
refresh_token: str = None,
client_secret: str = None,
provider_id: str = None) -> 'Connection':
"""
OpenId Connect Refresh Token
WARNING: this API is in experimental phase
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id,
client_id=client_id,
client_secret=client_secret)
if refresh_token is None:
refresh_token = self._get_refresh_token_store().get_refresh_token(
issuer=client_info.provider.issuer,
client_id=client_info.client_id)
if refresh_token is None:
raise OpenEoClientException("No refresh token given or found")
authenticator = OidcRefreshTokenAuthenticator(
client_info=client_info, refresh_token=refresh_token)
return self._authenticate_oidc(authenticator, provider_id=provider_id)
def _get_asset(self) -> Tuple[str, dict]:
assets = self._get_assets()
if len(assets) != 1:
raise OpenEoClientException(
"Expected one result file to download, but got {c}: {u!r}".
format(c=len(assets), u=assets))
return assets.popitem()
def authenticate_basic(self,
username: str = None,
password: str = None) -> 'Connection':
"""
Authenticate a user to the backend using basic username and password.
:param username: User name
:param password: User passphrase
"""
if username is None:
username, password = self._get_auth_config().get_basic_auth(
backend=self._orig_url)
if username is None:
raise OpenEoClientException(
"No username/password given or found.")
resp = self.get(
'/credentials/basic',
# /credentials/basic is the only endpoint that expects a Basic HTTP auth
auth=HTTPBasicAuth(username, password)).json()
# Switch to bearer based authentication in further requests.
if self._api_version.at_least("1.0.0"):
self.auth = BearerAuth(bearer='basic//{t}'.format(
t=resp["access_token"]))
else:
self.auth = BearerAuth(bearer=resp["access_token"])
return self
def soft_error(message: str):
"""Non breaking error (unless we had too much of them)"""
nonlocal _soft_error_count
_soft_error_count += 1
if _soft_error_count > soft_error_max:
raise OpenEoClientException("Excessive soft errors")
print_status(message)
time.sleep(connection_retry_interval)
def convert_callable_to_pgnode(callback: Callable, parent_parameters: Optional[List[str]] = None) -> PGNode:
"""
Convert given process callback to a PGNode.
>>> result = convert_callable_to_pgnode(lambda x: x + 5)
>>> assert isinstance(result, PGNode)
>>> result.flat_graph()
{"add1": {"process_id": "add", "arguments": {"x": {"from_parameter": "x"}, "y": 5}, "result": True}}
"""
# TODO: eliminate local import (due to circular dependency)?
from openeo.processes import ProcessBuilder
process_params = get_parameter_names(callback)
if parent_parameters is None:
# Due to lack of parent parameter information,
# we blindly use all callback's argument names as parameter names
if len(process_params) > 1:
_log.warning(f"Guessing callback parameters of {callback!r} from its arguments {process_params!r}")
kwargs = {p: ProcessBuilder({"from_parameter": p}) for p in process_params}
elif parent_parameters == ["x", "y"] and (len(process_params) == 1 or process_params[:1] == ["data"]):
# Special case: wrap all parent parameters in an array
kwargs = {process_params[0]: ProcessBuilder([{"from_parameter": p} for p in parent_parameters])}
else:
# Check for direct correspondence between callback arguments and parent parameters (or subset thereof).
common = set(parent_parameters).intersection(process_params)
if common:
kwargs = {p: ProcessBuilder({"from_parameter": p}) for p in common}
elif min(len(parent_parameters), len(process_params)) == 0:
kwargs = {}
elif min(len(parent_parameters), len(process_params)) == 1:
# Fallback for common case of just one callback argument (pass the main parameter),
# or one parent parameter (just pass that one)
kwargs = {process_params[0]: ProcessBuilder({"from_parameter": parent_parameters[0]})}
else:
raise OpenEoClientException(
f"Callback argument mismatch: expected (prefix of) {parent_parameters}, but found found {process_params!r}"
)
# "Evaluate" the callback, which should give a ProcessBuilder again to extract pgnode from
result = callback(**kwargs)
if not isinstance(result, ProcessBuilderBase):
raise OpenEoClientException(
f"Callback {callback} did not evaluate to ProcessBuilderBase. Got {result!r} instead"
)
return result.pgnode
def download_files(self,
target: Union[str, Path] = None) -> Dict[Path, dict]:
target = Path(target or Path.cwd())
if target.exists() and not target.is_dir():
raise OpenEoClientException(
f"Target argument {target} exists but isn't a folder.")
return {
a.download(target): a.metadata
for a in self.results.get_assets()
}
def download_files(self,
target: Union[str, Path] = None) -> Dict[Path, dict]:
target = Path(target or Path.cwd())
if target.exists() and not target.is_dir():
raise OpenEoClientException(
"The target argument must be a folder. Got {t!r}".format(
t=str(target)))
return {
asset.download(target): asset.metadata
for name, asset in self.results.get_assets().items()
}
def get_asset(self, name: str = None) -> ResultAsset:
"""
Get single asset by name or without name if there is only one.
"""
# TODO: also support getting a single asset by type or role?
assets = self.get_assets()
if len(assets) == 0:
raise OpenEoClientException("No assets in result.")
if name is None:
if len(assets) == 1:
return assets[0]
else:
raise MultipleAssetException(
"Multiple result assets for job {j}: {a}".format(
j=self._job.job_id, a=[a.name for a in assets]))
else:
try:
return next(a for a in assets if a.name == name)
except StopIteration:
raise OpenEoClientException("No asset {n!r} in: {a}".format(
n=name, a=[a.name for a in assets]))
def download_files(self, target: Union[Path, str] = None) -> List[Path]:
"""
Download all assets to given folder.
:param target: path to folder to download to (must be a folder if it already exists)
:return: list of paths to the downloaded assets.
"""
target = Path(target or Path.cwd())
if target.exists() and not target.is_dir():
raise OpenEoClientException(
"The target argument must be a folder. Got {t!r}".format(
t=str(target)))
return [a.download(target) for a in self.get_assets().values()]
def datacube_from_process(self, process_id: str, **kwargs) -> DataCube:
"""
Load a raster datacube, from a custom process.
@param process_id: The process id of the custom process.
@param kwargs: The arguments of the custom process
@return: A DataCube, without valid metadata, as the client is not aware of this custom process.
"""
if self._api_version.at_least("1.0.0"):
graph = PGNode(process_id, kwargs)
return DataCube(graph, self)
else:
raise OpenEoClientException(
"This method requires support for at least version 1.0.0 in the openEO backend.")
def download_file(self, target: Union[str, Path] = None, source: str = None) -> Path:
if source is None:
filename, metadata = self._get_asset()
url = metadata["href"]
else:
assets=self._get_assets()
if source not in assets:
raise OpenEoClientException("Remote file not found: "+source+" is not in "+str(list(assets)))
filename=source
url = assets[source]["href"]
target = Path(target or Path.cwd())
if target.is_dir():
target = target / filename
self._download_url(url, target)
return target
def _get_oidc_provider_and_client_info(
self,
provider_id: str,
client_id: Union[str, None],
client_secret: Union[str, None],
default_client_grant_types: Union[None,
List[DefaultOidcClientGrant]] = None
) -> Tuple[str, OidcClientInfo]:
"""
Resolve provider_id and client info (as given or from config)
:param provider_id: id of OIDC provider as specified by backend (/credentials/oidc).
Can be None if there is just one provider.
:return: OIDC provider id and client info
"""
provider_id, provider = self._get_oidc_provider(provider_id)
if client_id is None:
_log.debug("No client_id: checking config for prefered client_id")
client_id, client_secret = self._get_auth_config(
).get_oidc_client_configs(backend=self._orig_url,
provider_id=provider_id)
if client_id:
_log.info("Using client_id {c!r} from config (provider {p!r})".
format(c=client_id, p=provider_id))
if client_id is None and default_client_grant_types:
# Try "default_client" from backend's provider info.
_log.debug(
"No client_id given: checking default client in backend's provider info"
)
client_id = provider.get_default_client_id(
grant_types=default_client_grant_types)
if client_id:
_log.info(
"Using default client_id {c!r} from OIDC provider {p!r} info."
.format(c=client_id, p=provider_id))
if client_id is None:
raise OpenEoClientException("No client_id found.")
client_info = OidcClientInfo(client_id=client_id,
client_secret=client_secret,
provider=provider)
return provider_id, client_info
def download_file(self,
target: Union[Path, str] = None,
name: str = None) -> Path:
"""
Download single asset. Can be used when there is only one asset in the
:py:class:`JobResults`, or when the desired asset name is given explicitly.
:param target: path to download to. Can be an existing directory
(in which case the filename advertised by backend will be used)
or full file name. By default, the working directory will be used.
:param name: asset name to download (not required when there is only one asset)
:return: path of downloaded asset
"""
try:
return self.get_asset(name=name).download(target=target)
except MultipleAssetException:
raise OpenEoClientException(
"Can not use `download_file` with multiple assets. Use `download_files` instead."
)
def download_result(self, target: Union[str, Path] = None) -> Path:
"""
Download single job result to the target file path or into folder (current working dir by default).
Fails if there are multiple result files.
:param target: String or path where the file should be downloaded to.
"""
assets = self._download_get_assets()
if len(assets) != 1:
raise OpenEoClientException(
"Expected one result file to download, but got {c}: {u!r}".
format(c=len(assets), u=assets))
filename, metadata = assets.popitem()
url = metadata["href"]
target = Path(target or Path.cwd())
if target.is_dir():
target = target / filename
self._download_url(url, target)
return target
