def test_mask_polygon(s2cube, api_version):
polygon = shapely.geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0,
1.9]])
if api_version < ComparableVersion("1.0.0"):
expected_proces_id = "mask"
im = s2cube.mask(polygon)
else:
expected_proces_id = "mask_polygon"
im = s2cube.mask_polygon(mask=polygon)
graph = _get_leaf_node(im)
assert graph["process_id"] == expected_proces_id
assert graph["arguments"] == {
"data": {
'from_node': 'loadcollection1'
},
"mask": {
'coordinates':
(((0.0, 0.0), (1.9, 0.0), (1.9, 1.9), (0.0, 1.9), (0.0, 0.0)), ),
'crs': {
'properties': {
'name': 'EPSG:4326'
},
'type': 'name'
},
'type':
'Polygon'
}
}
def test_batch_job_metadata_to_api_dict():
api_version = ComparableVersion("1.0.0")
job = BatchJobMetadata(
id="123", status="running", created=datetime.datetime(2022, 1, 18, 16, 42, 0),
process={"add": {"process_id": "add", "arguments": {"x": 3, "y": 5}, "result": True}},
title="Untitled01", description="Lorem ipsum.",
progress=0.3,
cpu_time=datetime.timedelta(seconds=1000),
memory_time_megabyte=datetime.timedelta(seconds=2000),
started=datetime.datetime(2022, 1, 18, 17, 0, 0),
finished=datetime.datetime(2022, 1, 18, 17, 20, 0),
epsg=4326,
links=[{}],
)
assert job.to_api_dict(full=False, api_version=api_version) == {
"id": "123",
"title": "Untitled01", "description": "Lorem ipsum.",
"status": "running",
"progress": 0.3,
"created": "2022-01-18T16:42:00Z",
}
assert job.to_api_dict(full=True, api_version=api_version) == {
"id": "123",
"title": "Untitled01", "description": "Lorem ipsum.",
"process": {"add": {"process_id": "add", "arguments": {"x": 3, "y": 5}, "result": True}},
"status": "running",
"progress": 0.3,
"created": "2022-01-18T16:42:00Z",
"usage": {
"cpu": {"value": 1000, "unit": "cpu-seconds"},
"memory": {"value": 2000, "unit": "mb-seconds"},
"duration": {"value": 1200, "unit": "seconds"},
}
}
def test_aggregate_spatial(connection, api_version, reducer):
if api_version < ComparableVersion("1.0.0"):
pytest.skip()
polygon = shapely.geometry.shape(load_json_resource("data/polygon.json"))
res = (connection.load_collection("S2").filter_bbox(
3, 6, 52, 50).aggregate_spatial(geometries=polygon, reducer=reducer))
assert get_execute_graph(res) == load_json_resource(
'data/%s/aggregate_zonal_polygon.json' % api_version)
def test_aggregate_spatial_parameter_polygon(connection, api_version):
if api_version < ComparableVersion("1.0.0"):
pytest.skip()
geometries = Parameter("polygon")
res = (connection.load_collection("S2").filter_bbox(
3, 6, 52, 50).aggregate_spatial(geometries=geometries, reducer="mean"))
assert get_execute_graph(res) == load_json_resource(
'data/%s/aggregate_zonal_parameter.json' % api_version)
def test_load_collection_bands_common_name(connection, api_version):
im = connection.load_collection("S2", bands=["nir", "red"])
expected = load_json_resource('data/{v}/load_collection_bands.json'.format(v=api_version))
if api_version < ComparableVersion("1.0.0"):
expected["loadcollection1"]["arguments"]["bands"] = ["B08", "B04"]
else:
expected["loadcollection1"]["arguments"]["bands"] = ["nir", "red"]
assert im.flat_graph() == expected
def test_normalize_collection_metadata_no_id(self, caplog):
with pytest.raises(KeyError):
_normalize_collection_metadata(
{"foo": "bar"}, api_version=ComparableVersion("1.0.0"))
errors = [
r.getMessage() for r in caplog.records
if r.levelno == logging.ERROR
]
assert any("should have 'id' field" in m for m in errors)
def test_filter_bands_common_name(s2cube, api_version):
im = s2cube.filter_bands(["nir", "red"])
expected = load_json_resource('data/{v}/filter_bands.json'.format(v=api_version))
if api_version < ComparableVersion("1.0.0"):
expected["filterbands1"]["arguments"]["bands"] = []
expected["filterbands1"]["arguments"]["common_names"] = ["nir", "red"]
else:
expected["filterbands1"]["arguments"]["bands"] = ["nir", "red"]
assert im.flat_graph() == expected
def __init__(self,
api_version: str,
client: FlaskClient,
data_root: Path = None):
self.api_version = api_version
self.api_version_compare = ComparableVersion(self.api_version)
self.client = client
if data_root:
self.data_root = Path(data_root)
def test_aggregate_spatial_read_vector(connection, api_version, reducer):
if api_version < ComparableVersion("1.0.0"):
pytest.skip()
res = (connection.load_collection("S2").filter_bbox(
3, 6, 52, 50).aggregate_spatial(
geometries="/some/path/to/GeometryCollection.geojson",
reducer=reducer))
assert get_execute_graph(res) == load_json_resource(
'data/%s/aggregate_zonal_path.json' % api_version)
def test_comparable_version_right_referencing():
v = ComparableVersion('1.2.3')
assert v.equals('1.2.3')
assert v.above('0')
assert v.above('0.1')
assert v.above('0.1.2')
assert v.above('1.2')
assert v.above('1.2.2')
assert v.above('1.2.2b')
assert v.above('1.2.3') is False
assert v.above('1.2.20') is False
assert v.above('1.2.4') is False
assert v.above('1.10.4') is False
assert v.at_least('0')
assert v.at_least('1')
assert v.at_least('1.1')
assert v.at_least('1.10') is False
assert v.at_least('1.2')
assert v.at_least('1.02')
assert v.at_least('1.2.2')
assert v.at_least('1.2.3')
assert v.at_least('1.2.3a') is False
assert v.at_least('1.2.4') is False
assert v.at_least('1.3') is False
assert v.at_least('2') is False
assert v.below('2')
assert v.below('1.3')
assert v.below('1.2.4')
assert v.below('1.2.3b')
assert v.below('1.2.3') is False
assert v.below('1.2') is False
assert v.at_most('2')
assert v.at_most('1.3')
assert v.at_most('1.2.3c')
assert v.at_most('1.2.3')
assert v.at_most('1.02.03')
assert v.at_most('1.2.2b') is False
assert v.at_most('1.2') is False
assert v.at_most('1.10')
assert v.above(ComparableVersion('1.2'))
assert v.at_least(ComparableVersion('1.2.3a')) is False
assert v.at_most(ComparableVersion('1.02.03'))
def get_process_graph_dict(self, process_graph: dict) -> dict:
"""
Build dict containing process graph (e.g. to POST, or to expect in metadata),
according to API version
"""
if ComparableVersion("1.0.0").or_higher(self.api_version):
data = {"process": {'process_graph': process_graph}}
else:
data = {'process_graph': process_graph}
return data
def __init__(self,
api_version: str,
client: FlaskClient,
data_root: Path = None,
url_root: str = "/openeo/{api_version}"):
self.api_version = api_version
self.api_version_compare = ComparableVersion(self.api_version)
self.client = client
if data_root:
self.data_root = Path(data_root)
self.default_request_headers = {}
self.url_root = url_root.format(api_version=api_version)
def test_comparable_version_operators(b):
a = ComparableVersion("1.2.3")
assert (a == a) is True
assert (a != a) is False
assert (a > b) is True
assert (a >= b) is True
assert (a < b) is False
assert (a <= b) is False
assert (b < a) is True
assert (b <= a) is True
assert (b > a) is False
assert (b >= a) is False
def test_comparable_version_left_referencing():
v = ComparableVersion("1.2.3")
assert v.or_higher("1.2.2") is False
assert v.or_higher("1.2.3") is True
assert v.or_higher("1.2.4") is True
assert v.or_lower("1.2.2") is True
assert v.or_lower("1.2.3") is True
assert v.or_lower("1.2.4") is False
assert v.accept_higher("1.2.2") is False
assert v.accept_higher("1.2.3") is False
assert v.accept_higher("1.2.4") is True
assert v.accept_lower("1.2.2") is True
assert v.accept_lower("1.2.3") is False
assert v.accept_lower("1.2.4") is False
def test_authenticate_basic(requests_mock, api_version):
requests_mock.get(API_URL, json={"api_version": api_version})
conn = Connection(API_URL)
def text_callback(request, context):
assert request.headers["Authorization"] == "Basic am9objpqMGhu"
return '{"access_token":"w3lc0m3"}'
requests_mock.get(API_URL + 'credentials/basic', text=text_callback)
assert isinstance(conn.auth, NullAuth)
conn.authenticate_basic(username="******", password="******")
assert isinstance(conn.auth, BearerAuth)
if ComparableVersion(api_version).at_least("1.0.0"):
assert conn.auth.bearer == "basic//w3lc0m3"
else:
assert conn.auth.bearer == "w3lc0m3"
def get_process_graph_dict(self,
process_graph: dict,
title: str = None,
description: str = None) -> dict:
"""
Build dict containing process graph (e.g. to POST, or to expect in metadata),
according to API version
"""
if ComparableVersion("1.0.0").or_higher(self.api_version):
data = {"process": {'process_graph': process_graph}}
else:
data = {'process_graph': process_graph}
if title:
data["title"] = title
if description:
data["description"] = description
return data
def test_endpoint_registry():
app = flask.Flask(__name__)
bp = flask.Blueprint("test", __name__)
endpoint = EndpointRegistry()
@bp.route("/hello")
def hello():
return "not an endpoint"
@endpoint
@bp.route("/foo")
def foo():
return "simple endpoint"
@endpoint(hidden=True)
@bp.route("/secret")
def secret():
return "secret endpoint"
@endpoint(version=ComparableVersion("1.0.0").accept_lower)
@bp.route("/old")
def old():
return "old endpoint"
app.register_blueprint(bp, url_prefix="/bar")
result = endpoint.get_path_metadata(bp)
# Check metadata
assert len(result) == 3
paths, methods, metadatas = zip(*sorted(result))
assert paths == ('/foo', '/old', '/secret')
assert methods == ({"GET"}, ) * 3
assert metadatas[0].hidden is False
assert metadatas[0].for_version is None
assert metadatas[1].for_version("0.4.0") is True
assert metadatas[1].for_version("1.0.0") is False
assert metadatas[2].hidden is True
# Check that view functions still work
client = app.test_client()
assert b"not an endpoint" == client.get("/bar/hello").data
assert b"simple endpoint" == client.get("/bar/foo").data
assert b"secret endpoint" == client.get("/bar/secret").data
assert b"old endpoint" == client.get("/bar/old").data
def test_normalize_collection_metadata_minimal_100(self, caplog):
assert _normalize_collection_metadata(
{"id": "foobar"}, api_version=ComparableVersion("1.0.0")) == {
'id': 'foobar',
'stac_version': '0.9.0',
'description': 'foobar',
'extent': {
'spatial': [0, 0, 0, 0],
'temporal': [None, None]
},
'license': 'proprietary',
'links': [],
}
warnings = set(r.getMessage() for r in caplog.records
if r.levelno == logging.WARN)
assert warnings == {
"Collection 'foobar' metadata does not have field 'extent'."
}
def test_authenticate_basic_from_config(requests_mock, api_version):
user, pwd = "john281", "J0hndo3"
requests_mock.get(API_URL, json={"api_version": api_version})
def text_callback(request, context):
assert request.headers[
"Authorization"] == requests.auth._basic_auth_str(username=user,
password=pwd)
return '{"access_token":"w3lc0m3"}'
requests_mock.get(API_URL + 'credentials/basic', text=text_callback)
AuthConfig().set_basic_auth(backend=API_URL, username=user, password=pwd)
conn = Connection(API_URL)
assert isinstance(conn.auth, NullAuth)
conn.authenticate_basic()
assert isinstance(conn.auth, BearerAuth)
if ComparableVersion(api_version).at_least("1.0.0"):
assert conn.auth.bearer == "basic//w3lc0m3"
else:
assert conn.auth.bearer == "w3lc0m3"
def test_normalize_collection_metadata_dimensions_and_bands_100(
self, caplog):
metadata = {
"id": "foobar",
"properties": {
"cube:dimensions": {
"x": {
"type": "spatial"
},
"b": {
"type": "bands",
"values": ["B02", "B03"]
}
},
"eo:bands": [{
"name": "B02"
}, {
"name": "B03"
}]
}
}
res = _normalize_collection_metadata(
metadata, api_version=ComparableVersion("1.0.0"), full=True)
assert res["cube:dimensions"] == {
"x": {
"type": "spatial"
},
"b": {
"type": "bands",
"values": ["B02", "B03"]
}
}
assert res["summaries"]["eo:bands"] == [{
"name": "B02"
}, {
"name": "B03"
}]
import pytest
from openeo.capabilities import ComparableVersion
@pytest.mark.parametrize(["a", "b", "c"], [
(ComparableVersion("1.2.3"), ComparableVersion("1.2.3"),
ComparableVersion("2.3.4")),
(ComparableVersion("1.2.3"), "1.2.3", "2.3.4"),
("1.2.3", ComparableVersion("1.2.3"), ComparableVersion("2.3.4")),
])
def test_comparable_version_equals(a, b, c):
assert (a == b) is True
assert (a == c) is False
assert (a != b) is False
assert (a != c) is True
if isinstance(a, ComparableVersion):
assert a.equals(b) is True
assert a.equals(c) is False
@pytest.mark.parametrize("b", [
"0.9",
"1",
"1.2.2",
ComparableVersion("0.9"),
ComparableVersion("1.1"),
])
def test_comparable_version_operators(b):
a = ComparableVersion("1.2.3")
assert (a == a) is True
def api_version(request) -> ComparableVersion:
return ComparableVersion(request.param)
class Connection(RestApiConnection):
"""
Connection to an openEO backend.
"""
_MINIMUM_API_VERSION = ComparableVersion("0.4.0")
# Temporary workaround flag to enable for backends (e.g. EURAC) that expect id_token to be sent as bearer token
# TODO DEPRECATED To remove when all backends properly expect access_token
# see https://github.com/Open-EO/openeo-wcps-driver/issues/45
oidc_auth_user_id_token_as_bearer = False
def __init__(self,
url,
auth: AuthBase = None,
session: requests.Session = None,
default_timeout: int = None,
auth_config: AuthConfig = None,
refresh_token_store: RefreshTokenStore = None):
"""
Constructor of Connection, authenticates user.
:param url: String Backend root url
"""
if "://" not in url:
url = "https://" + url
self._orig_url = url
super().__init__(root_url=self.version_discovery(url, session=session),
auth=auth,
session=session,
default_timeout=default_timeout)
self._capabilities_cache = {}
# Initial API version check.
if self._api_version.below(self._MINIMUM_API_VERSION):
raise ApiVersionException(
"OpenEO API version should be at least {m!s}, but got {v!s}".
format(m=self._MINIMUM_API_VERSION, v=self._api_version))
self._auth_config = auth_config
self._refresh_token_store = refresh_token_store
@classmethod
def version_discovery(cls,
url: str,
session: requests.Session = None) -> str:
"""
Do automatic openEO API version discovery from given url, using a "well-known URI" strategy.
:param url: initial backend url (not including "/.well-known/openeo")
:return: root url of highest supported backend version
"""
try:
well_known_url_response = RestApiConnection(
url, session=session).get("/.well-known/openeo")
assert well_known_url_response.status_code == 200
versions = well_known_url_response.json()["versions"]
supported_versions = [
v for v in versions
if cls._MINIMUM_API_VERSION <= v["api_version"]
]
assert supported_versions
production_versions = [
v for v in supported_versions if v.get("production", True)
]
highest_version = max(production_versions or supported_versions,
key=lambda v: v["api_version"])
_log.debug("Highest supported version available in backend: %s" %
highest_version)
return highest_version['url']
except Exception:
# Be very lenient about failing on the well-known URI strategy.
return url
def _get_auth_config(self) -> AuthConfig:
if self._auth_config is None:
self._auth_config = AuthConfig()
return self._auth_config
def _get_refresh_token_store(self) -> RefreshTokenStore:
if self._refresh_token_store is None:
self._refresh_token_store = RefreshTokenStore()
return self._refresh_token_store
def authenticate_basic(self,
username: str = None,
password: str = None) -> 'Connection':
"""
Authenticate a user to the backend using basic username and password.
:param username: User name
:param password: User passphrase
"""
if username is None:
username, password = self._get_auth_config().get_basic_auth(
backend=self._orig_url)
if username is None:
raise OpenEoClientException(
"No username/password given or found.")
resp = self.get(
'/credentials/basic',
# /credentials/basic is the only endpoint that expects a Basic HTTP auth
auth=HTTPBasicAuth(username, password)).json()
# Switch to bearer based authentication in further requests.
if self._api_version.at_least("1.0.0"):
self.auth = BearerAuth(bearer='basic//{t}'.format(
t=resp["access_token"]))
else:
self.auth = BearerAuth(bearer=resp["access_token"])
return self
def _get_oidc_provider(
self,
provider_id: Union[str,
None] = None) -> Tuple[str, OidcProviderInfo]:
"""
Get OpenID Connect discovery URL for given provider_id
:param provider_id: id of OIDC provider as specified by backend (/credentials/oidc).
Can be None if there is just one provider.
:return: updated provider_id and provider info object
"""
if self._api_version.at_least("1.0.0"):
oidc_info = self.get("/credentials/oidc",
expected_status=200).json()
providers = {p["id"]: p for p in oidc_info["providers"]}
_log.info(
"Found OIDC providers: {p}".format(p=list(providers.keys())))
if provider_id:
if provider_id not in providers:
raise OpenEoClientException(
"Requested OIDC provider {r!r} not available. Should be one of {p}."
.format(r=provider_id, p=list(providers.keys())))
provider = providers[provider_id]
elif len(providers) == 1:
provider_id, provider = providers.popitem()
_log.info(
"No OIDC provider given, but only one available: {p!r}. Use that one."
.format(p=provider_id))
else:
# Check if there is a single provider in the config to use.
backend = self._orig_url
provider_configs = self._get_auth_config(
).get_oidc_provider_configs(backend=backend)
intersection = set(provider_configs.keys()).intersection(
providers.keys())
if len(intersection) == 1:
provider_id = intersection.pop()
provider = providers[provider_id]
_log.info(
"No OIDC provider id given, but only one in config (backend {b!r}): {p!r}."
" Use that one.".format(b=backend, p=provider_id))
else:
raise OpenEoClientException(
"No OIDC provider id given. Pick one from: {p!r}.".
format(p=list(providers.keys())))
provider = OidcProviderInfo.from_dict(provider)
else:
# Per spec: '/credentials/oidc' will redirect to OpenID Connect discovery document
provider = OidcProviderInfo(
discovery_url=self.build_url('/credentials/oidc'))
return provider_id, provider
def _get_oidc_provider_and_client_info(
self,
provider_id: str,
client_id: Union[str, None],
client_secret: Union[str, None],
default_client_grant_types: Union[None,
List[DefaultOidcClientGrant]] = None
) -> Tuple[str, OidcClientInfo]:
"""
Resolve provider_id and client info (as given or from config)
:param provider_id: id of OIDC provider as specified by backend (/credentials/oidc).
Can be None if there is just one provider.
:return: OIDC provider id and client info
"""
provider_id, provider = self._get_oidc_provider(provider_id)
if client_id is None:
_log.debug("No client_id: checking config for prefered client_id")
client_id, client_secret = self._get_auth_config(
).get_oidc_client_configs(backend=self._orig_url,
provider_id=provider_id)
if client_id:
_log.info("Using client_id {c!r} from config (provider {p!r})".
format(c=client_id, p=provider_id))
if client_id is None and default_client_grant_types:
# Try "default_client" from backend's provider info.
_log.debug(
"No client_id given: checking default client in backend's provider info"
)
client_id = provider.get_default_client_id(
grant_types=default_client_grant_types)
if client_id:
_log.info(
"Using default client_id {c!r} from OIDC provider {p!r} info."
.format(c=client_id, p=provider_id))
if client_id is None:
raise OpenEoClientException("No client_id found.")
client_info = OidcClientInfo(client_id=client_id,
client_secret=client_secret,
provider=provider)
return provider_id, client_info
def _authenticate_oidc(self,
authenticator: OidcAuthenticator,
provider_id: str,
store_refresh_token: bool = False) -> 'Connection':
"""
Authenticate through OIDC and set up bearer token (based on OIDC access_token) for further requests.
"""
tokens = authenticator.get_tokens(
request_refresh_token=store_refresh_token)
_log.info("Obtained tokens: {t}".format(
t=[k for k, v in tokens._asdict().items() if v]))
if store_refresh_token:
if tokens.refresh_token:
self._get_refresh_token_store().set_refresh_token(
issuer=authenticator.provider_info.issuer,
client_id=authenticator.client_id,
refresh_token=tokens.refresh_token)
else:
_log.warning(
"OIDC token response did not contain refresh token.")
token = tokens.access_token if not self.oidc_auth_user_id_token_as_bearer else tokens.id_token
if self._api_version.at_least("1.0.0"):
self.auth = BearerAuth(
bearer='oidc/{p}/{t}'.format(p=provider_id, t=token))
else:
self.auth = BearerAuth(bearer=token)
return self
def authenticate_oidc_authorization_code(
self,
client_id: str = None,
client_secret: str = None,
provider_id: str = None,
timeout: int = None,
server_address: Tuple[str, int] = None,
webbrowser_open: Callable = None,
store_refresh_token=False,
) -> 'Connection':
"""
OpenID Connect Authorization Code Flow (with PKCE).
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id,
client_id=client_id,
client_secret=client_secret,
default_client_grant_types=[DefaultOidcClientGrant.AUTH_CODE_PKCE],
)
authenticator = OidcAuthCodePkceAuthenticator(
client_info=client_info,
webbrowser_open=webbrowser_open,
timeout=timeout,
server_address=server_address)
return self._authenticate_oidc(authenticator,
provider_id=provider_id,
store_refresh_token=store_refresh_token)
def authenticate_oidc_client_credentials(
self,
client_id: str = None,
client_secret: str = None,
provider_id: str = None,
store_refresh_token=False,
) -> 'Connection':
"""
OpenID Connect Client Credentials flow.
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id,
client_id=client_id,
client_secret=client_secret)
authenticator = OidcClientCredentialsAuthenticator(
client_info=client_info)
return self._authenticate_oidc(authenticator,
provider_id=provider_id,
store_refresh_token=store_refresh_token)
def authenticate_oidc_resource_owner_password_credentials(
self,
username: str,
password: str,
client_id: str = None,
client_secret: str = None,
provider_id: str = None,
store_refresh_token=False) -> 'Connection':
"""
OpenId Connect Resource Owner Password Credentials
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id,
client_id=client_id,
client_secret=client_secret)
# TODO: also get username and password from config?
authenticator = OidcResourceOwnerPasswordAuthenticator(
client_info=client_info, username=username, password=password)
return self._authenticate_oidc(authenticator,
provider_id=provider_id,
store_refresh_token=store_refresh_token)
def authenticate_oidc_refresh_token(
self,
client_id: str = None,
refresh_token: str = None,
client_secret: str = None,
provider_id: str = None) -> 'Connection':
"""
OpenId Connect Refresh Token
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id,
client_id=client_id,
client_secret=client_secret,
default_client_grant_types=[DefaultOidcClientGrant.REFRESH_TOKEN],
)
if refresh_token is None:
refresh_token = self._get_refresh_token_store().get_refresh_token(
issuer=client_info.provider.issuer,
client_id=client_info.client_id)
if refresh_token is None:
raise OpenEoClientException("No refresh token given or found")
authenticator = OidcRefreshTokenAuthenticator(
client_info=client_info, refresh_token=refresh_token)
return self._authenticate_oidc(authenticator, provider_id=provider_id)
def authenticate_oidc_device(self,
client_id: str = None,
client_secret: str = None,
provider_id: str = None,
store_refresh_token=False,
use_pkce: Union[bool, None] = None,
**kwargs) -> 'Connection':
"""
Authenticate with OAuth Device Authorization grant/flow
:param use_pkce: Use PKCE instead of client secret.
If not set explicitly to `True` (use PKCE) or `False` (use client secret),
it will be attempted to detect the best mode automatically.
Note that PKCE for device code is not widely supported among OIDC providers.
.. versionchanged:: 0.5.1 Add :py:obj:`use_pkce` argument
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id,
client_id=client_id,
client_secret=client_secret,
default_client_grant_types=[
DefaultOidcClientGrant.DEVICE_CODE_PKCE
],
)
authenticator = OidcDeviceAuthenticator(client_info=client_info,
use_pkce=use_pkce,
**kwargs)
return self._authenticate_oidc(authenticator,
provider_id=provider_id,
store_refresh_token=store_refresh_token)
def authenticate_oidc(self,
provider_id: str = None,
client_id: Union[str, None] = None,
client_secret: Union[str, None] = None,
store_refresh_token: bool = True):
"""
Do OpenID Connect authentication, first trying refresh tokens and falling back on device code flow.
.. versionadded:: 0.6.0
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id,
client_id=client_id,
client_secret=client_secret,
default_client_grant_types=[
DefaultOidcClientGrant.DEVICE_CODE_PKCE,
DefaultOidcClientGrant.REFRESH_TOKEN
])
# Try refresh token first.
refresh_token = self._get_refresh_token_store().get_refresh_token(
issuer=client_info.provider.issuer,
client_id=client_info.client_id)
if refresh_token:
try:
_log.info(
"Found refresh token: trying refresh token based authentication."
)
authenticator = OidcRefreshTokenAuthenticator(
client_info=client_info, refresh_token=refresh_token)
con = self._authenticate_oidc(
authenticator,
provider_id=provider_id,
store_refresh_token=store_refresh_token)
# TODO: pluggable/jupyter-aware display function?
print("Authenticated using refresh token.")
return con
except OidcException as e:
_log.info(
"Refresh token based authentication failed: {e}.".format(
e=e))
# Fall back on device code flow
# TODO: make it possible to do other fallback flows too?
_log.info("Trying device code flow.")
authenticator = OidcDeviceAuthenticator(client_info=client_info)
con = self._authenticate_oidc(authenticator,
provider_id=provider_id,
store_refresh_token=store_refresh_token)
print("Authenticated using device code flow.")
return con
def describe_account(self) -> str:
"""
Describes the currently authenticated user account.
"""
return self.get('/me').json()
@deprecated("use :py:meth:`list_jobs` instead", version="0.4.10")
def user_jobs(self) -> dict:
return self.list_jobs()
def list_collections(self) -> List[dict]:
"""
Loads all available imagecollections types.
:return: list of collection meta data dictionaries
"""
data = self.get('/collections').json()["collections"]
return VisualList("collections", data=data)
def list_collection_ids(self) -> List[str]:
"""
Get list of all collection ids
:return: list of collection ids
"""
return [
collection['id'] for collection in self.list_collections()
if 'id' in collection
]
def capabilities(self) -> RESTCapabilities:
"""
Loads all available capabilities.
:return: data_dict: Dict All available data types
"""
if "capabilities" not in self._capabilities_cache:
self._capabilities_cache["capabilities"] = RESTCapabilities(
self.get('/').json(), self._orig_url)
return self._capabilities_cache["capabilities"]
def list_output_formats(self) -> dict:
if self._api_version.at_least("1.0.0"):
return self.list_file_formats()["output"]
else:
return self.get('/output_formats').json()
list_file_types = legacy_alias(list_output_formats, "list_file_types")
def list_file_formats(self) -> dict:
"""
Get available input and output formats
"""
if "file_formats" not in self._capabilities_cache:
self._capabilities_cache["file_formats"] = self.get(
'/file_formats').json()
return VisualDict("file-formats",
data=self._capabilities_cache["file_formats"])
def list_service_types(self) -> dict:
"""
Loads all available service types.
:return: data_dict: Dict All available service types
"""
if "service_types" not in self._capabilities_cache:
self._capabilities_cache["service_types"] = self.get(
'/service_types').json()
return VisualDict("service-types",
data=self._capabilities_cache["service_types"])
def list_udf_runtimes(self) -> dict:
"""
Loads all available UDF runtimes.
:return: data_dict: Dict All available UDF runtimes
"""
if "udf_runtimes" not in self._capabilities_cache:
self._capabilities_cache["udf_runtimes"] = self.get(
'/udf_runtimes').json()
return VisualDict("udf-runtimes",
data=self._capabilities_cache["udf_runtimes"])
def list_services(self) -> dict:
"""
Loads all available services of the authenticated user.
:return: data_dict: Dict All available services
"""
# TODO return parsed service objects
services = self.get('/services').json()["services"]
return VisualList("data-table",
data=services,
parameters={'columns': 'services'})
def describe_collection(self, name) -> dict:
# TODO: Maybe create some kind of Data class.
"""
Loads detailed information of a specific image collection.
:param name: String Id of the collection
:return: data_dict: Dict Detailed information about the collection
"""
data = self.get('/collections/{}'.format(name)).json()
return VisualDict("collection", data=data)
def collection_items(self,
name,
spatial_extent: Optional[List[float]] = None,
temporal_extent: Optional[List[Union[
str, datetime.datetime]]] = None,
limit: int = None) -> Iterator[dict]:
"""
Loads items for a specific image collection.
May not be available for all collections.
This is an experimental API and is subject to change.
:param name: String Id of the collection
:param spatial_extent: Limits the items to the given bounding box in WGS84:
1. Lower left corner, coordinate axis 1
2. Lower left corner, coordinate axis 2
3. Upper right corner, coordinate axis 1
4. Upper right corner, coordinate axis 2
:param temporal_extent: Limits the items to the specified temporal interval.
:param limit: The amount of items per request/page. If None, the back-end decides.
The interval has to be specified as an array with exactly two elements (start, end).
Also supports open intervals by setting one of the boundaries to None, but never both.
:return: data_list: List A list of items
"""
url = '/collections/{}/items'.format(name)
params = {}
if spatial_extent:
params["bbox"] = ",".join(str(c) for c in spatial_extent)
if temporal_extent:
params["datetime"] = "/".join(
".." if t is None else rfc3339.normalize(t)
for t in temporal_extent)
if limit is not None and limit > 0:
params['limit'] = limit
return paginate(
self, url, params, lambda response, page: VisualDict(
"items",
data=response,
parameters={
'show-map': True,
'heading': 'Page {} - Items'.format(page)
}))
def collection_metadata(self, name) -> CollectionMetadata:
return CollectionMetadata(metadata=self.describe_collection(name))
def list_processes(self) -> List[dict]:
# TODO: Maybe format the result dictionary so that the process_id is the key of the dictionary.
"""
Loads all available processes of the back end.
:return: processes_dict: Dict All available processes of the back end.
"""
data = self.get('/processes').json()["processes"]
return VisualList("processes", data=data)
def list_jobs(self) -> dict:
"""
Lists all jobs of the authenticated user.
:return: job_list: Dict of all jobs of the user.
"""
# TODO: Parse the result so that there get Job classes returned?
jobs = self.get('/jobs').json()["jobs"]
return VisualList("data-table",
data=jobs,
parameters={'columns': 'jobs'})
def save_user_defined_process(
self,
user_defined_process_id: str,
process_graph: Union[dict, ProcessBuilderBase],
parameters: List[Union[dict, Parameter]] = None,
public: bool = False,
summary: str = None,
description: str = None) -> RESTUserDefinedProcess:
"""
Saves a process graph and its metadata in the backend as a user-defined process for the authenticated user.
:param user_defined_process_id: unique identifier for the user-defined process
:param process_graph: a process graph
:param parameters: a list of parameters
:param public: visible to other users?
:param summary: A short summary of what the process does.
:param description: Detailed description to explain the entity. CommonMark 0.29 syntax MAY be used for rich text representation.
:return: a RESTUserDefinedProcess instance
"""
if user_defined_process_id in set(p["id"]
for p in self.list_processes()):
warnings.warn(
"Defining user-defined process {u!r} with same id as a pre-defined process"
.format(u=user_defined_process_id))
if not parameters:
warnings.warn(
"Defining user-defined process {u!r} without parameters".
format(u=user_defined_process_id))
udp = RESTUserDefinedProcess(
user_defined_process_id=user_defined_process_id, connection=self)
udp.store(process_graph=process_graph,
parameters=parameters,
public=public,
summary=summary,
description=description)
return udp
def list_user_defined_processes(self) -> List[dict]:
"""
Lists all user-defined processes of the authenticated user.
"""
return self.get("/process_graphs").json()["processes"]
def user_defined_process(
self, user_defined_process_id: str) -> RESTUserDefinedProcess:
"""
Get the user-defined process based on its id. The process with the given id should already exist.
:param user_defined_process_id: the id of the user-defined process
:return: a RESTUserDefinedProcess instance
"""
return RESTUserDefinedProcess(
user_defined_process_id=user_defined_process_id, connection=self)
def validate_processgraph(self, process_graph):
# Endpoint: POST /validate
raise NotImplementedError()
@property
def _api_version(self) -> ComparableVersion:
# TODO make this a public property (it's also useful outside the Connection class)
return self.capabilities().api_version_check
def datacube_from_process(self,
process_id: str,
namespace: str = None,
**kwargs) -> DataCube:
"""
Load a raster datacube, from a custom process.
:param process_id: The process id of the custom process.
:param namespace: optional: process namespace
:param kwargs: The arguments of the custom process
:return: A DataCube, without valid metadata, as the client is not aware of this custom process.
"""
if self._api_version.at_least("1.0.0"):
graph = PGNode(process_id, namespace=namespace, arguments=kwargs)
return DataCube(graph=graph, connection=self)
else:
raise OpenEoClientException(
"This method requires support for at least version 1.0.0 in the openEO backend."
)
def load_collection(
self,
collection_id: str,
spatial_extent: Optional[Dict[str, float]] = None,
temporal_extent: Optional[List[Union[str, datetime.datetime,
datetime.date]]] = None,
bands: Optional[List[str]] = None,
properties: Optional[Dict[str, Union[str, PGNode, Callable]]] = None
) -> DataCube:
"""
Load a DataCube by collection id.
:param collection_id: image collection identifier
:param spatial_extent: limit data to specified bounding box or polygons
:param temporal_extent: limit data to specified temporal interval
:param bands: only add the specified bands
:param properties: limit data by metadata property predicates
:return: a datacube containing the requested data
"""
if self._api_version.at_least("1.0.0"):
return DataCube.load_collection(collection_id=collection_id,
connection=self,
spatial_extent=spatial_extent,
temporal_extent=temporal_extent,
bands=bands,
properties=properties)
else:
return ImageCollectionClient.load_collection(
collection_id=collection_id,
session=self,
spatial_extent=spatial_extent,
temporal_extent=temporal_extent,
bands=bands)
imagecollection = legacy_alias(load_collection, name="imagecollection")
def create_service(self, graph: dict, type: str, **kwargs) -> dict:
# TODO: type hint for graph: is it a nested or a flat one?
req = self._build_request_with_process_graph(process_graph=graph,
type=type,
**kwargs)
response = self.post(path="/services", json=req, expected_status=201)
# TODO: "location" is url of the service metadata, not (base) url of service (https://github.com/Open-EO/openeo-api/issues/269)
# TODO: fetch this metadata and return a full metadata object instead?
return {
'url': response.headers.get('Location'),
'service_id': response.headers.get("OpenEO-Identifier"),
}
def remove_service(self, service_id: str):
"""
Stop and remove a secondary web service.
:param service_id: service identifier
:return:
"""
response = self.delete('/services/' + service_id)
@deprecated("Use :py:meth:`openeo.rest.job.RESTJob.get_results` instead.",
version="0.4.10")
def job_results(self, job_id) -> dict:
"""Get batch job results metadata."""
return RESTJob(job_id, connection=self).list_results()
@deprecated("Use :py:meth:`openeo.rest.job.RESTJob.logs` instead.",
version="0.4.10")
def job_logs(self, job_id, offset) -> list:
"""Get batch job logs."""
return RESTJob(job_id, connection=self).logs(offset=offset)
def list_files(self):
"""
Lists all files that the logged in user uploaded.
:return: file_list: List of the user uploaded files.
"""
files = self.get('/files').json()['files']
return VisualList("data-table",
data=files,
parameters={'columns': 'files'})
def create_file(self, path):
"""
Creates virtual file
:return: file object.
"""
# No endpoint just returns a file object.
raise NotImplementedError()
def _build_request_with_process_graph(self, process_graph: Union[dict,
Any],
**kwargs) -> dict:
"""
Prepare a json payload with a process graph to submit to /result, /services, /jobs, ...
:param process_graph: flat dict representing a process graph
"""
result = kwargs
process_graph = as_flat_graph(process_graph)
if self._api_version.at_least("1.0.0"):
result["process"] = {"process_graph": process_graph}
else:
result["process_graph"] = process_graph
return result
# TODO: unify `download` and `execute` better: e.g. `download` always writes to disk, `execute` returns result (raw or as JSON decoded dict)
def download(self,
graph: dict,
outputfile: Union[Path, str, None] = None,
timeout: int = 30 * 60):
"""
Downloads the result of a process graph synchronously,
and save the result to the given file or return bytes object if no outputfile is specified.
This method is useful to export binary content such as images. For json content, the execute method is recommended.
:param graph: (flat) dict representing a process graph
:param outputfile: output file
"""
request = self._build_request_with_process_graph(process_graph=graph)
response = self.post(path="/result",
json=request,
stream=True,
timeout=timeout)
if outputfile is not None:
with Path(outputfile).open(mode="wb") as f:
for chunk in response.iter_content(chunk_size=None):
f.write(chunk)
else:
return response.content
def execute(self, process_graph: dict):
"""
Execute a process graph synchronously.
:param process_graph: (flat) dict representing a process graph
"""
req = self._build_request_with_process_graph(
process_graph=process_graph)
return self.post(path="/result", json=req).json()
def create_job(self,
process_graph: dict,
title: str = None,
description: str = None,
plan: str = None,
budget=None,
additional: Dict = None) -> RESTJob:
"""
Posts a job to the back end.
:param process_graph: (flat) dict representing process graph
:param title: String title of the job
:param description: String description of the job
:param plan: billing plan
:param budget: Budget
:param additional: additional job options to pass to the backend
:return: job_id: String Job id of the new created job
"""
# TODO move all this (RESTJob factory) logic to RESTJob?
req = self._build_request_with_process_graph(
process_graph=process_graph,
**dict_no_none(title=title,
description=description,
plan=plan,
budget=budget))
if additional:
# TODO: get rid of this non-standard field? https://github.com/Open-EO/openeo-api/issues/276
req["job_options"] = additional
response = self.post("/jobs", json=req, expected_status=201)
if "openeo-identifier" in response.headers:
job_id = response.headers['openeo-identifier']
elif "location" in response.headers:
_log.warning(
"Backend did not explicitly respond with job id, will guess it from redirect URL."
)
job_id = response.headers['location'].split("/")[-1]
else:
raise OpenEoClientException("Failed fo extract job id")
return RESTJob(job_id, self)
def job(self, job_id: str):
"""
Get the job based on the id. The job with the given id should already exist.
Use :py:meth:`openeo.rest.connection.Connection.create_job` to create new jobs
:param job_id: the job id of an existing job
:return: A job object.
"""
return RESTJob(job_id, self)
def load_disk_collection(self,
format: str,
glob_pattern: str,
options: dict = {}) -> ImageCollectionClient:
"""
Loads image data from disk as an ImageCollection.
:param format: the file format, e.g. 'GTiff'
:param glob_pattern: a glob pattern that matches the files to load from disk
:param options: options specific to the file format
:return: the data as an ImageCollection
"""
if self._api_version.at_least("1.0.0"):
return DataCube.load_disk_collection(self, format, glob_pattern,
**options)
else:
return ImageCollectionClient.load_disk_collection(
self, format, glob_pattern, **options)
import pytest
from openeo.capabilities import ComparableVersion
@pytest.mark.parametrize("b", [
"0.9",
"1",
"1.2.2",
ComparableVersion("0.9"),
ComparableVersion("1.1"),
])
def test_comparable_version_operators(b):
a = ComparableVersion("1.2.3")
assert (a == a) is True
assert (a != a) is False
assert (a > b) is True
assert (a >= b) is True
assert (a < b) is False
assert (a <= b) is False
assert (b < a) is True
assert (b <= a) is True
assert (b > a) is False
assert (b >= a) is False
def test_comparable_version_right_referencing():
v = ComparableVersion('1.2.3')
assert v.above('0')
assert v.above('0.1')
assert v.above('0.1.2')
def apply_process(process_id: str, args: Dict, viewingParameters):
parent_process = viewingParameters.get('parent_process')
if 'filter_daterange' == process_id or 'filter_temporal' == process_id:
"""
filter_daterange <= pre 0.3.x
filter_temporal >= 0.4.x
"""
# TODO `viewingParameters` is function argument, but written to/manipulated (used as some kind of state object)
# which is not obvious and confusing when debugging
viewingParameters = viewingParameters or {}
if 'extent' in args:
#version >= 0.4
extent = args['extent']
if len(extent) != 2:
raise AttributeError(
"extent property should be an array of length 2, but got: "
+ str(extent))
viewingParameters["from"] = extent[0]
viewingParameters["to"] = extent[1]
else:
viewingParameters["from"] = extract_arg(args, "from")
viewingParameters["to"] = extract_arg(args, "to")
elif 'filter_bbox' == process_id:
viewingParameters = viewingParameters or {}
if "left" in args:
# <=0.3.x
viewingParameters["left"] = extract_arg(args, "left")
viewingParameters["right"] = extract_arg(args, "right")
viewingParameters["top"] = extract_arg(args, "top")
viewingParameters["bottom"] = extract_arg(args, "bottom")
viewingParameters["srs"] = extract_arg(args, "srs")
else:
extent = args
if "extent" in args:
extent = args["extent"]
# >=0.4.x
viewingParameters["left"] = extract_arg(extent, "west")
viewingParameters["right"] = extract_arg(extent, "east")
viewingParameters["top"] = extract_arg(extent, "north")
viewingParameters["bottom"] = extract_arg(extent, "south")
viewingParameters["srs"] = extent.get("crs") or "EPSG:4326"
elif process_id in [
'zonal_statistics', 'aggregate_polygon', 'aggregate_spatial'
]:
polygons = extract_arg_list(args, ['regions', 'polygons'])
if viewingParameters.get("left") is None:
if "type" in polygons: # it's GeoJSON
geometries = _as_geometry_collection(
polygons
) if polygons['type'] == 'FeatureCollection' else polygons
bbox = shape(geometries).bounds
if "from_node" in polygons: # it's a dereferenced from_node that contains a DelayedVector
geometries = convert_node(polygons["node"], viewingParameters)
bbox = geometries.bounds
viewingParameters["left"] = bbox[0]
viewingParameters["right"] = bbox[2]
viewingParameters["bottom"] = bbox[1]
viewingParameters["top"] = bbox[3]
viewingParameters["srs"] = "EPSG:4326"
args[
'polygons'] = geometries # might as well cache the value instead of re-evaluating it further on
elif 'filter_bands' == process_id:
viewingParameters = viewingParameters or {}
viewingParameters["bands"] = extract_arg(args, "bands")
elif 'apply' == parent_process:
if "data" in viewingParameters:
# The `apply` process passes it's `data` parameter as `x` parameter to subprocess
viewingParameters["x"] = viewingParameters["data"]
#first we resolve child nodes and arguments
args = {
name: convert_node(expr, viewingParameters)
for (name, expr) in args.items()
}
#when all arguments and dependencies are resolved, we can run the process
if parent_process == "apply":
image_collection = extract_arg_list(args, ['x', 'data', 'imagery'])
if process_id == "run_udf":
udf = _get_udf(args)
return image_collection.apply_tiles(udf)
else:
return image_collection.apply(process_id, args)
elif parent_process in ["reduce", "reduce_dimension"]:
image_collection = extract_arg_list(args, ['data', 'imagery'])
dimension = extract_arg(viewingParameters, 'dimension')
binary = viewingParameters.get('binary', False)
dimension, band_dim, temporal_dim = _check_dimension(
cube=image_collection, dim=dimension, process=parent_process)
if 'run_udf' == process_id and not binary:
if dimension == temporal_dim:
udf = _get_udf(args)
#EP-2760 a special case of reduce where only a single udf based callback is provided. The more generic case is not yet supported.
return image_collection.apply_tiles_spatiotemporal(udf)
elif dimension == band_dim:
udf = _get_udf(args)
return image_collection.apply_tiles(udf)
return image_collection.reduce(process_id, dimension)
elif parent_process == 'apply_dimension':
image_collection = extract_arg(args, 'data')
dimension = viewingParameters.get(
'dimension', None
) # By default, applies the the process on all pixel values (as apply does).
dimension, band_dim, temporal_dim = _check_dimension(
cube=image_collection, dim=dimension, process=parent_process)
if process_id == "run_udf":
udf = _get_udf(args)
if dimension == temporal_dim:
return image_collection.apply_tiles_spatiotemporal(udf)
else:
return image_collection.apply_tiles(udf)
else:
return image_collection.apply_dimension(process_id, dimension)
elif parent_process in ['aggregate_polygon', 'aggregate_spatial']:
image_collection = extract_arg_list(args, ['data', 'imagery'])
binary = viewingParameters.get('binary', False)
name = viewingParameters.get('name', 'result')
polygons = extract_arg(viewingParameters, 'polygons')
# can be either (inline) GeoJSON or something returned by read_vector
is_geojson = isinstance(polygons, Dict)
if is_geojson:
geometries = shape(polygons)
return image_collection.zonal_statistics(geometries,
func=process_id)
# TODO: rename to aggregate_polygon?
return image_collection.zonal_statistics(polygons.path,
func=process_id)
elif parent_process == 'aggregate_temporal':
image_collection = extract_arg_list(args, ['data', 'imagery'])
intervals = extract_arg(viewingParameters, 'intervals')
labels = extract_arg(viewingParameters, 'labels')
dimension = viewingParameters.get('dimension', None)
dimension, _, _ = _check_dimension(cube=image_collection,
dim=dimension,
process=parent_process)
return image_collection.aggregate_temporal(intervals, labels,
process_id, dimension)
else:
if ComparableVersion("1.0.0").or_higher(viewingParameters["version"]):
process_function = process_registry_100.get_function(process_id)
else:
process_function = process_registry_040.get_function(process_id)
return process_function(args, viewingParameters)
class Connection(RestApiConnection):
"""
Connection to an openEO backend.
"""
_MINIMUM_API_VERSION = ComparableVersion("0.4.0")
# Temporary workaround flag to enable for backends (e.g. EURAC) that expect id_token to be sent as bearer token
# TODO DEPRECATED To remove when all backends properly expect access_token
# see https://github.com/Open-EO/openeo-wcps-driver/issues/45
oidc_auth_user_id_token_as_bearer = False
def __init__(
self, url, auth: AuthBase = None, session: requests.Session = None, default_timeout: int = None,
auth_config: AuthConfig = None, refresh_token_store: RefreshTokenStore = None
):
"""
Constructor of Connection, authenticates user.
:param url: String Backend root url
"""
self._orig_url = url
super().__init__(
root_url=self.version_discovery(url, session=session),
auth=auth, session=session, default_timeout=default_timeout
)
self._capabilities_cache = {}
# Initial API version check.
if self._api_version.below(self._MINIMUM_API_VERSION):
raise ApiVersionException("OpenEO API version should be at least {m!s}, but got {v!s}".format(
m=self._MINIMUM_API_VERSION, v=self._api_version)
)
self._auth_config = auth_config
self._refresh_token_store = refresh_token_store or RefreshTokenStore()
@classmethod
def version_discovery(cls, url: str, session: requests.Session = None) -> str:
"""
Do automatic openEO API version discovery from given url, using a "well-known URI" strategy.
:param url: initial backend url (not including "/.well-known/openeo")
:return: root url of highest supported backend version
"""
try:
well_known_url_response = RestApiConnection(url, session=session).get("/.well-known/openeo")
assert well_known_url_response.status_code == 200
versions = well_known_url_response.json()["versions"]
supported_versions = [v for v in versions if cls._MINIMUM_API_VERSION <= v["api_version"]]
assert supported_versions
production_versions = [v for v in supported_versions if v.get("production", True)]
highest_version = max(production_versions or supported_versions, key=lambda v: v["api_version"])
_log.debug("Highest supported version available in backend: %s" % highest_version)
return highest_version['url']
except Exception:
# Be very lenient about failing on the well-known URI strategy.
return url
def _get_auth_config(self) -> AuthConfig:
if self._auth_config is None:
self._auth_config = AuthConfig()
return self._auth_config
def authenticate_basic(self, username: str = None, password: str = None) -> 'Connection':
"""
Authenticate a user to the backend using basic username and password.
:param username: User name
:param password: User passphrase
"""
if username is None:
username, password = self._get_auth_config().get_basic_auth(backend=self._orig_url)
if username is None:
raise OpenEoClientException("No username/password given or found.")
resp = self.get(
'/credentials/basic',
# /credentials/basic is the only endpoint that expects a Basic HTTP auth
auth=HTTPBasicAuth(username, password)
).json()
# Switch to bearer based authentication in further requests.
if self._api_version.at_least("1.0.0"):
self.auth = BearerAuth(bearer='basic//{t}'.format(t=resp["access_token"]))
else:
self.auth = BearerAuth(bearer=resp["access_token"])
return self
def authenticate_OIDC(
self, client_id: str,
provider_id: str = None,
webbrowser_open=None,
timeout=120,
server_address: Tuple[str, int] = None
) -> 'Connection':
"""
Authenticates a user to the backend using OpenID Connect.
:param client_id: Client id to use for OpenID Connect authentication
:param webbrowser_open: optional handler for the initial OAuth authentication request
(opens a webbrowser by default)
:param timeout: number of seconds after which to abort the authentication procedure
:param server_address: optional tuple (hostname, port_number) to serve the OAuth redirect callback on
TODO: deprecated?
"""
# TODO: option to increase log level temporarily?
provider_id, provider = self._get_oidc_provider(provider_id)
client_info = OidcClientInfo(client_id=client_id, provider=provider)
authenticator = OidcAuthCodePkceAuthenticator(
client_info=client_info,
webbrowser_open=webbrowser_open,
timeout=timeout,
server_address=server_address,
)
return self._authenticate_oidc(authenticator, provider_id=provider_id)
def _get_oidc_provider(self, provider_id: Union[str, None] = None) -> Tuple[str, OidcProviderInfo]:
"""
Get OpenID Connect discovery URL for given provider_id
:param provider_id: id of OIDC provider as specified by backend (/credentials/oidc).
Can be None if there is just one provider.
:return: updated provider_id and provider info object
"""
if self._api_version.at_least("1.0.0"):
oidc_info = self.get("/credentials/oidc", expected_status=200).json()
providers = {p["id"]: p for p in oidc_info["providers"]}
_log.info("Found OIDC providers: {p}".format(p=list(providers.keys())))
if provider_id:
if provider_id not in providers:
raise OpenEoClientException("Requested provider {r!r} not available. Should be one of {p}.".format(
r=provider_id, p=list(providers.keys()))
)
provider = providers[provider_id]
elif len(providers) == 1:
# No provider id given, but there is only one anyway: we can handle that.
provider_id, provider = providers.popitem()
else:
raise OpenEoClientException("No provider_id given. Available: {p!r}.".format(
p=list(providers.keys()))
)
provider = OidcProviderInfo(issuer=provider["issuer"], scopes=provider.get("scopes"))
else:
# Per spec: '/credentials/oidc' will redirect to OpenID Connect discovery document
provider = OidcProviderInfo(discovery_url=self.build_url('/credentials/oidc'))
return provider_id, provider
def _get_oidc_provider_and_client_info(
self, provider_id: str,
client_id: Union[str, None], client_secret: Union[str, None]
) -> Tuple[str, OidcClientInfo]:
"""
Resolve provider_id and client info (as given or from config)
:param provider_id: id of OIDC provider as specified by backend (/credentials/oidc).
Can be None if there is just one provider.
:return: (client_id, client_secret)
"""
provider_id, provider = self._get_oidc_provider(provider_id)
if client_id is None:
client_id, client_secret = self._get_auth_config().get_oidc_client_configs(
backend=self._orig_url, provider_id=provider_id
)
_log.info("Using client_id {c!r} from config (provider {p!r})".format(c=client_id, p=provider_id))
if client_id is None:
raise OpenEoClientException("No client ID found.")
client_info = OidcClientInfo(client_id=client_id, client_secret=client_secret, provider=provider)
return provider_id, client_info
def _authenticate_oidc(
self,
authenticator: OidcAuthenticator,
provider_id: str,
store_refresh_token: bool = False
) -> 'Connection':
"""
Authenticate through OIDC and set up bearer token (based on OIDC access_token) for further requests.
"""
tokens = authenticator.get_tokens()
_log.info("Obtained tokens: {t}".format(t=[k for k, v in tokens._asdict().items() if v]))
if tokens.refresh_token and store_refresh_token:
self._refresh_token_store.set_refresh_token(
issuer=authenticator.provider_info.issuer,
client_id=authenticator.client_id,
refresh_token=tokens.refresh_token
)
token = tokens.access_token if not self.oidc_auth_user_id_token_as_bearer else tokens.id_token
if self._api_version.at_least("1.0.0"):
self.auth = BearerAuth(bearer='oidc/{p}/{t}'.format(p=provider_id, t=token))
else:
self.auth = BearerAuth(bearer=token)
return self
def authenticate_oidc_authorization_code(
self,
client_id: str = None,
client_secret: str = None,
provider_id: str = None,
timeout: int = None,
server_address: Tuple[str, int] = None,
webbrowser_open: Callable = None,
store_refresh_token=False,
) -> 'Connection':
"""
OpenID Connect Authorization Code Flow (with PKCE).
WARNING: this API is in experimental phase
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id, client_id=client_id, client_secret=client_secret
)
authenticator = OidcAuthCodePkceAuthenticator(
client_info=client_info,
webbrowser_open=webbrowser_open, timeout=timeout, server_address=server_address
)
return self._authenticate_oidc(authenticator, provider_id=provider_id, store_refresh_token=store_refresh_token)
def authenticate_oidc_client_credentials(
self,
client_id: str = None,
client_secret: str = None,
provider_id: str = None,
store_refresh_token=False,
) -> 'Connection':
"""
OpenID Connect Client Credentials flow.
WARNING: this API is in experimental phase
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id, client_id=client_id, client_secret=client_secret
)
authenticator = OidcClientCredentialsAuthenticator(client_info=client_info)
return self._authenticate_oidc(authenticator, provider_id=provider_id, store_refresh_token=store_refresh_token)
def authenticate_oidc_resource_owner_password_credentials(
self,
username: str, password: str,
client_id: str = None,
client_secret: str = None,
provider_id: str = None,
store_refresh_token=False
) -> 'Connection':
"""
OpenId Connect Resource Owner Password Credentials
WARNING: this API is in experimental phase
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id, client_id=client_id, client_secret=client_secret
)
# TODO: also get username and password from config?
authenticator = OidcResourceOwnerPasswordAuthenticator(
client_info=client_info, username=username, password=password
)
return self._authenticate_oidc(authenticator, provider_id=provider_id, store_refresh_token=store_refresh_token)
def authenticate_oidc_refresh_token(
self, client_id: str = None, refresh_token: str = None, client_secret: str = None, provider_id: str = None
) -> 'Connection':
"""
OpenId Connect Refresh Token
WARNING: this API is in experimental phase
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id, client_id=client_id, client_secret=client_secret
)
if refresh_token is None:
refresh_token = self._refresh_token_store.get_refresh_token(
issuer=client_info.provider.issuer,
client_id=client_info.client_id
)
if refresh_token is None:
raise OpenEoClientException("No refresh token given or found")
authenticator = OidcRefreshTokenAuthenticator(client_info=client_info, refresh_token=refresh_token)
return self._authenticate_oidc(authenticator, provider_id=provider_id)
def authenticate_oidc_device(
self, client_id: str=None, client_secret: str=None, provider_id: str = None,
store_refresh_token=False,
**kwargs
) -> 'Connection':
"""
Authenticate with OAuth Device Authorization grant/flow
WARNING: this API is in experimental phase
"""
provider_id, client_info = self._get_oidc_provider_and_client_info(
provider_id=provider_id, client_id=client_id, client_secret=client_secret
)
authenticator = OidcDeviceAuthenticator(client_info=client_info, **kwargs)
return self._authenticate_oidc(authenticator, provider_id=provider_id, store_refresh_token=store_refresh_token)
def describe_account(self) -> str:
"""
Describes the currently authenticated user account.
"""
return self.get('/me').json()
def user_jobs(self) -> dict:
"""
Loads all jobs of the current user.
:return: jobs: Dict All jobs of the user
"""
# TODO duplication with `list_jobs()` method
return self.get('/jobs').json()["jobs"]
def list_collections(self) -> List[dict]:
"""
Loads all available imagecollections types.
:return: list of collection meta data dictionaries
"""
return self.get('/collections').json()["collections"]
def list_collection_ids(self) -> List[str]:
"""
Get list of all collection ids
:return: list of collection ids
"""
return [collection['id'] for collection in self.list_collections() if 'id' in collection]
def capabilities(self) -> RESTCapabilities:
"""
Loads all available capabilities.
:return: data_dict: Dict All available data types
"""
if "capabilities" not in self._capabilities_cache:
self._capabilities_cache["capabilities"] = RESTCapabilities(self.get('/').json())
return self._capabilities_cache["capabilities"]
def list_output_formats(self) -> dict:
if self._api_version.at_least("1.0.0"):
return self.list_file_formats()["output"]
else:
return self.get('/output_formats').json()
list_file_types = legacy_alias(list_output_formats, "list_file_types")
def list_file_formats(self) -> dict:
"""
Get available input and output formats
"""
if "file_formats" not in self._capabilities_cache:
self._capabilities_cache["file_formats"] = self.get('/file_formats').json()
return self._capabilities_cache["file_formats"]
def list_service_types(self) -> dict:
"""
Loads all available service types.
:return: data_dict: Dict All available service types
"""
return self.get('/service_types').json()
def list_services(self) -> dict:
"""
Loads all available services of the authenticated user.
:return: data_dict: Dict All available service types
"""
# TODO return parsed service objects
return self.get('/services').json()
def describe_collection(self, name) -> dict:
# TODO: Maybe create some kind of Data class.
"""
Loads detailed information of a specific image collection.
:param name: String Id of the collection
:return: data_dict: Dict Detailed information about the collection
"""
return self.get('/collections/{}'.format(name)).json()
def collection_metadata(self, name) -> CollectionMetadata:
return CollectionMetadata(metadata=self.describe_collection(name))
def list_processes(self) -> List[dict]:
# TODO: Maybe format the result dictionary so that the process_id is the key of the dictionary.
"""
Loads all available processes of the back end.
:return: processes_dict: Dict All available processes of the back end.
"""
return self.get('/processes').json()["processes"]
def list_jobs(self) -> dict:
"""
Lists all jobs of the authenticated user.
:return: job_list: Dict of all jobs of the user.
"""
# TODO: Maybe format the result so that there get Job classes returned.
# TODO: duplication with `user_jobs()` method
return self.get('/jobs').json()["jobs"]
def save_user_defined_process(
self, user_defined_process_id: str, process_graph: dict,
parameters: List[Union[dict, Parameter]] = None, public: bool = False) -> RESTUserDefinedProcess:
"""
Saves a process graph and its metadata in the backend as a user-defined process for the authenticated user.
:param user_defined_process_id: unique identifier for the user-defined process
:param process_graph: a process graph
:param parameters: a list of parameters
:param public: visible to other users?
:return: a RESTUserDefinedProcess instance
"""
if user_defined_process_id in set(p["id"] for p in self.list_processes()):
warnings.warn("Defining user-defined process {u!r} with same id as a pre-defined process".format(
u=user_defined_process_id))
udp = RESTUserDefinedProcess(user_defined_process_id=user_defined_process_id, connection=self)
udp.store(process_graph=process_graph, parameters=parameters, public=public)
return udp
def list_user_defined_processes(self) -> List[dict]:
"""
Lists all user-defined processes of the authenticated user.
"""
return self.get("/process_graphs").json()["processes"]
def user_defined_process(self, user_defined_process_id: str) -> RESTUserDefinedProcess:
"""
Get the user-defined process based on its id. The process with the given id should already exist.
:param user_defined_process_id: the id of the user-defined process
:return: a RESTUserDefinedProcess instance
"""
return RESTUserDefinedProcess(user_defined_process_id=user_defined_process_id, connection=self)
def validate_processgraph(self, process_graph):
# Endpoint: POST /validate
raise NotImplementedError()
@property
def _api_version(self) -> ComparableVersion:
# TODO make this a public property (it's also useful outside the Connection class)
return self.capabilities().api_version_check
def datacube_from_process(self, process_id: str, **kwargs) -> DataCube:
"""
Load a raster datacube, from a custom process.
@param process_id: The process id of the custom process.
@param kwargs: The arguments of the custom process
@return: A DataCube, without valid metadata, as the client is not aware of this custom process.
"""
if self._api_version.at_least("1.0.0"):
graph = PGNode(process_id, kwargs)
return DataCube(graph, self)
else:
raise OpenEoClientException(
"This method requires support for at least version 1.0.0 in the openEO backend.")
def load_collection(self, collection_id: str, **kwargs) -> Union[ImageCollectionClient, DataCube]:
"""
Load an image collection by collection id
see :py:meth:`openeo.rest.imagecollectionclient.ImageCollectionClient.load_collection`
for available arguments.
:param collection_id: image collection identifier (string)
:return: ImageCollectionClient
"""
if self._api_version.at_least("1.0.0"):
return DataCube.load_collection(collection_id=collection_id, connection=self, **kwargs)
else:
return ImageCollectionClient.load_collection(collection_id=collection_id, session=self, **kwargs)
# Legacy alias.
imagecollection = load_collection
def create_service(self, graph: dict, type: str, **kwargs) -> dict:
# TODO: type hint for graph: is it a nested or a flat one?
req = self._build_request_with_process_graph(process_graph=graph, type=type, **kwargs)
response = self.post(path="/services", json=req, expected_status=201)
# TODO: "location" is url of the service metadata, not (base) url of service (https://github.com/Open-EO/openeo-api/issues/269)
# TODO: fetch this metadata and return a full metadata object instead?
return {
'url': response.headers.get('Location'),
'service_id': response.headers.get("OpenEO-Identifier"),
}
def remove_service(self, service_id: str):
"""
Stop and remove a secondary web service.
:param service_id: service identifier
:return:
"""
response = self.delete('/services/' + service_id)
def job_results(self, job_id):
return self.get("/jobs/{}/results".format(job_id)).json()
def job_logs(self, job_id, offset):
return self.get("/jobs/{}/logs".format(job_id), params={'offset': offset}).json()
def list_files(self):
"""
Lists all files that the logged in user uploaded.
:return: file_list: List of the user uploaded files.
"""
return self.get('/files').json()['files']
def create_file(self, path):
"""
Creates virtual file
:return: file object.
"""
# No endpoint just returns a file object.
raise NotImplementedError()
def _build_request_with_process_graph(self, process_graph: dict, **kwargs) -> dict:
"""
Prepare a json payload with a process graph to submit to /result, /services, /jobs, ...
:param process_graph: flat dict representing a process graph
"""
result = kwargs
if self._api_version.at_least("1.0.0"):
result["process"] = {"process_graph": process_graph}
else:
result["process_graph"] = process_graph
return result
# TODO: Maybe rename to execute and merge with execute().
def download(self, graph: dict, outputfile: Union[Path, str, None] = None):
"""
Downloads the result of a process graph synchronously,
and save the result to the given file or return bytes object if no outputfile is specified.
This method is useful to export binary content such as images. For json content, the execute method is recommended.
:param graph: (flat) dict representing a process graph
:param outputfile: output file
"""
request = self._build_request_with_process_graph(process_graph=graph)
r = self.post(path="/result", json=request, stream=True, timeout=1000)
if outputfile is not None:
with Path(outputfile).open(mode="wb") as f:
shutil.copyfileobj(r.raw, f)
else:
return r.content
def execute(self, process_graph: dict):
"""
Execute a process graph synchronously.
:param process_graph: (flat) dict representing a process graph
"""
req = self._build_request_with_process_graph(process_graph=process_graph)
return self.post(path="/result", json=req).json()
def create_job(self, process_graph: dict, title: str = None, description: str = None,
plan: str = None, budget=None,
additional: Dict = None) -> RESTJob:
"""
Posts a job to the back end.
:param process_graph: (flat) dict representing process graph
:param title: String title of the job
:param description: String description of the job
:param plan: billing plan
:param budget: Budget
:param additional: additional job options to pass to the backend
:return: job_id: String Job id of the new created job
"""
# TODO move all this (RESTJob factory) logic to RESTJob?
req = self._build_request_with_process_graph(
process_graph=process_graph,
title=title, description=description, plan=plan, budget=budget
)
if additional:
# TODO: get rid of this non-standard field? https://github.com/Open-EO/openeo-api/issues/276
req["job_options"] = additional
response = self.post("/jobs", json=req, expected_status=201)
if "openeo-identifier" in response.headers:
job_id = response.headers['openeo-identifier']
elif "location" in response.headers:
_log.warning("Backend did not explicitly respond with job id, will guess it from redirect URL.")
job_id = response.headers['location'].split("/")[-1]
else:
raise OpenEoClientException("Failed fo extract job id")
return RESTJob(job_id, self)
def job(self, job_id: str):
"""
Get the job based on the id. The job with the given id should already exist.
Use :py:meth:`openeo.rest.connection.Connection.create_job` to create new jobs
:param job_id: the job id of an existing job
:return: A job object.
"""
return RESTJob(job_id, self)
def load_disk_collection(self, format: str, glob_pattern: str, options: dict = {}) -> ImageCollectionClient:
"""
Loads image data from disk as an ImageCollection.
:param format: the file format, e.g. 'GTiff'
:param glob_pattern: a glob pattern that matches the files to load from disk
:param options: options specific to the file format
:return: the data as an ImageCollection
"""
if self._api_version.at_least("1.0.0"):
return DataCube.load_disk_collection(self, format, glob_pattern, **options)
else:
return ImageCollectionClient.load_disk_collection(self, format, glob_pattern, **options)
except pkg_resources.DistributionNotFound:
version_info[package] = "n/a"
return {
'date': date_to_rfc3339(datetime.datetime.utcnow()),
'versions': version_info
}
@openeo_bp.route('/health')
def health():
return jsonify({
"health": backend_implementation.health_check()
})
@api_endpoint(version=ComparableVersion("0.3.1").or_lower)
@openeo_bp.route('/capabilities')
def capabilities():
return jsonify([
"/data",
"/execute",
"/processes"
])
@api_endpoint(version=ComparableVersion("1.0.0").accept_lower)
@openeo_bp.route('/output_formats')
def output_formats():
# TODO deprecated endpoint, remove it when v0.4 API support is not necessary anymore
return jsonify(backend_implementation.file_formats()["output"])
def requested_api_version() -> ComparableVersion:
"""Get the currently requested API version as a ComparableVersion object"""
return ComparableVersion(g.api_version)
def result_callback(request, context):
post_data = request.json()
pg = (post_data["process"] if api_version >= ComparableVersion("1.0.0") else post_data)["process_graph"]
assert pg["saveresult1"]["arguments"]["format"] == expected_format
return b"data"