def test_dict_no_none_args():
assert dict_no_none() == {}
assert dict_no_none({"a": 3}) == {"a": 3}
assert dict_no_none({
"a": 3,
"b": 0,
"c": "foo"
}) == {
"a": 3,
"b": 0,
"c": "foo"
}
assert dict_no_none({
"a": 3,
"b": "",
"c": "foo"
}) == {
"a": 3,
"b": "",
"c": "foo"
}
assert dict_no_none({
"a": 3,
"b": None,
"c": "foo"
}) == {
"a": 3,
"c": "foo"
}
def to_dict(self) -> dict:
"""
Convert this hypercube into a dictionary that can be converted into
a valid JSON representation
>>> example = {
... "id": "test_data",
... "data": [
... [[0.0, 0.1], [0.2, 0.3]],
... [[0.0, 0.1], [0.2, 0.3]],
... ],
... "dimension": [
... {"name": "time", "coordinates": ["2001-01-01", "2001-01-02"]},
... {"name": "X", "coordinates": [50.0, 60.0]},
... {"name": "Y"},
... ],
... }
"""
xd = self._array.to_dict()
return dict_no_none({
"id": xd.get("name"),
"data": xd.get("data"),
"description": deep_get(xd, "attrs", "description", default=None),
"dimensions": [
dict_no_none(
name=dim,
coordinates=deep_get(xd, "coords", dim, "data", default=None)
)
for dim in xd.get("dims", [])
]
})
def _setup_connection(api_version, requests_mock) -> Connection:
requests_mock.get(API_URL + "/", json={"api_version": api_version})
# Classic Sentinel2 collection
sentinel2_bands = [
("B01", "coastal aerosol"), ("B02", "blue"), ("B03", "green"), ("B04", "red"), ("B05", "nir"),
("B06", None), ("B07", None), ("B08", "nir"), ("B8A", "nir08"), ("B09", "nir09"),
("B11", "swir16"), ("B12", "swir22"),
]
requests_mock.get(API_URL + "/collections/SENTINEL2", json={
"id": "SENTINEL2",
"cube:dimensions": {
"x": {"type": "spatial"},
"y": {"type": "spatial"},
"t": {"type": "temporal"},
"bands": {"type": "bands", "values": [n for n, _ in sentinel2_bands]}
},
"summaries": {
"eo:bands": [dict_no_none(name=n, common_name=c) for n, c in sentinel2_bands]
},
})
# TODO: add other collections: Landsat, Modis, ProbaV, ...
return openeo.connect(API_URL)
def _build_token_response(self,
sub="123",
name="john",
include_id_token=True) -> str:
"""Build JSON serialized access/id/refresh token response (and store tokens for use in assertions)"""
access_token = self._jwt_encode({},
dict_no_none(
sub=sub,
name=name,
nonce=self.state.get("nonce")))
res = {"access_token": access_token}
# Attempt to simulate real world refresh token support.
if "offline_access" in self.scopes_supported:
# "offline_access" scope as suggested in spec
# (https://openid.net/specs/openid-connect-core-1_0.html#OfflineAccess)
# Implemented by Microsoft, EGI Check-in
include_refresh_token = "offline_access" in self.state.get(
"scope", "").split(" ")
else:
# Google OAuth style: no support for "offline_access", return refresh token automatically?
include_refresh_token = True
if include_refresh_token:
res["refresh_token"] = self._jwt_encode({}, {"foo": "refresh"})
if include_id_token:
res["id_token"] = access_token
self.state.update(res)
return json.dumps(res)
def filter_bbox(self,
west,
east,
north,
south,
crs=None,
base=None,
height=None) -> 'ImageCollection':
"""
Limits the ImageCollection to a given spatial bounding box.
:param west: west boundary (longitude / easting)
:param east: east boundary (longitude / easting)
:param north: north boundary (latitude / northing)
:param south: south boundary (latitude / northing)
:param crs: spatial reference system of boundaries as
proj4 or EPSG:12345 like string
:param base: lower left corner coordinate axis 3
:param height: upper right corner coordinate axis 3
:return: An image collection cropped to the specified bounding box.
https://open-eo.github.io/openeo-api/v/0.4.1/processreference/#filter_bbox
# TODO: allow passing some kind of bounding box object? e.g. a (xmin, ymin, xmax, ymax) tuple?
"""
# Subclasses are expected to implement this method, but for bit of backwards compatibility
# with old style subclasses we forward to `bbox_filter`
# TODO: replace this with raise NotImplementedError() or decorate with @abstractmethod
kwargs = dict(west=west, east=east, north=north, south=south)
kwargs.update(dict_no_none(crs=crs, base=base, height=height))
return self.bbox_filter(**kwargs)
def build_process_dict(
process_graph: Union[dict, ProcessBuilderBase],
process_id: Optional[str] = None,
summary: Optional[str] = None,
description: Optional[str] = None,
parameters: Optional[List[Union[Parameter, dict]]] = None,
returns: Optional[dict] = None,
) -> dict:
"""
Build a dictionary describing a process with metadaa (`process_graph`, `parameters`, `description`, ...)
:param process_graph: dict or builder representing a process graph
:param process_id: identifier of the process
:param summary: short summary of what the process does
:param description: detailed description
:param parameters: list of process parameters (which have name, schema, default value, ...)
:param returns: description and schema of what process returns
:return: dictionary in openEO "process graph with metadata" format
"""
process = dict_no_none(process_graph=as_flat_graph(process_graph),
id=process_id,
summary=summary,
description=description,
returns=returns)
if parameters is not None:
process["parameters"] = [
(p if isinstance(p, Parameter) else Parameter(**p)).to_dict()
for p in parameters
]
return process
def to_api_dict(self, full=True, api_version: ComparableVersion = None) -> dict:
"""
API-version-aware conversion of batch job metadata to jsonable openEO API compatible dict.
see https://openeo.org/documentation/1.0/developers/api/reference.html#operation/describe-job
"""
# Basic/full fields to export
fields = ["id", "title", "description", "status", "progress", "created", "updated", "plan", "costs", "budget"]
if full:
fields.extend(["process"])
result = {f: getattr(self, f) for f in fields}
# Additional cleaning and massaging.
result["created"] = rfc3339.datetime(self.created) if self.created else None
result["updated"] = rfc3339.datetime(self.updated) if self.updated else None
if full:
usage = self.usage or {}
if self.cpu_time:
usage["cpu"] = {"value": int(round(self.cpu_time.total_seconds())), "unit": "cpu-seconds"}
if self.duration:
usage["duration"] = {"value": int(round(self.duration.total_seconds())), "unit": "seconds"}
if self.memory_time_megabyte:
usage["memory"] = {"value": int(round(self.memory_time_megabyte.total_seconds())), "unit": "mb-seconds"}
if usage:
result["usage"] = usage
if api_version and api_version.below("1.0.0"):
result["process_graph"] = result.pop("process", {}).get("process_graph")
result["submitted"] = result.pop("created", None)
# TODO wider status checking coverage?
if result["status"] == "created":
result["status"] = "submitted"
return dict_no_none(result)
def leaveProcess(self, process_id: str, arguments: dict, namespace: Union[str, None]):
node_id = self._node_id_generator.generate(process_id)
self._flattened[node_id] = dict_no_none(
process_id=process_id,
arguments=self._argument_stack.pop(),
namespace=namespace,
)
self._last_node_id = node_id
def _jsonable_service_metadata(metadata: ServiceMetadata, full=True) -> dict:
"""API-version-aware conversion of service metadata to jsonable dict"""
d = metadata.prepare_for_json()
if not full:
d.pop("process")
d.pop("attributes")
if requested_api_version().below("1.0.0"):
d["process_graph"] = d.pop("process", {}).get("process_graph")
d["parameters"] = d.pop("configuration", None) or ({} if full else None)
d["submitted"] = d.pop("created", None)
return dict_no_none(**d)
def run_udf(
self, udf: str, runtime: str, version: Optional[str] = None, context: Optional[dict] = None
) -> "VectorCube":
"""
.. versionadded:: 0.10.0
"""
return self.process(
process_id="run_udf",
data=self, udf=udf, runtime=runtime,
arguments=dict_no_none({"version": version, "context": context}),
)
def _deep_copy(x):
"""PGNode aware deep copy helper"""
if isinstance(x, PGNode):
return dict_no_none(process_id=x.process_id, arguments=_deep_copy(x.arguments), namespace=x.namespace)
if isinstance(x, Parameter):
return {"from_parameter": x.name}
elif isinstance(x, dict):
return {str(k): _deep_copy(v) for k, v in x.items()}
elif isinstance(x, (list, tuple)):
return type(x)(_deep_copy(v) for v in x)
elif isinstance(x, (str, int, float)) or x is None:
return x
else:
raise ValueError(repr(x))
def ndvi(self, nir: str = None, red: str = None, target_band: str = None) -> 'DataCube':
""" Normalized Difference Vegetation Index (NDVI)
:param nir: (optional) name of NIR band
:param red: (optional) name of red band
:param target_band: (optional) name of the newly created band
:return: a DataCube instance
"""
return self.process(
process_id='ndvi',
arguments=dict_no_none(
data={'from_node': self._pg},
nir=nir, red=red, target_band=target_band
)
)
def mask_polygon(
self, mask: Union[Polygon, MultiPolygon, str, pathlib.Path] = None,
srs="EPSG:4326", replacement=None, inside: bool = None
) -> 'DataCube':
"""
Applies a polygon mask to a raster data cube. To apply a raster mask use `mask`.
All pixels for which the point at the pixel center does not intersect with any
polygon (as defined in the Simple Features standard by the OGC) are replaced.
This behaviour can be inverted by setting the parameter `inside` to true.
The pixel values are replaced with the value specified for `replacement`,
which defaults to `no data`.
:param mask: A polygon, provided as a :class:`shapely.geometry.Polygon` or :class:`shapely.geometry.MultiPolygon`, or a filename pointing to a valid vector file
:param srs: The reference system of the provided polygon, by default this is Lat Lon (EPSG:4326).
:param replacement: the value to replace the masked pixels with
"""
if isinstance(mask, (str, pathlib.Path)):
# TODO: default to loading file client side?
# TODO: change read_vector to load_uploaded_files https://github.com/Open-EO/openeo-processes/pull/106
read_vector = self.process(
process_id='read_vector',
arguments={'filename': str(mask)}
)
mask = {'from_node': read_vector._pg}
elif isinstance(mask, shapely.geometry.base.BaseGeometry):
if mask.area == 0:
raise ValueError("Mask {m!s} has an area of {a!r}".format(m=mask, a=mask.area))
mask = shapely.geometry.mapping(mask)
mask['crs'] = {
'type': 'name',
'properties': {'name': srs}
}
else:
# Assume mask is already a valid GeoJSON object
assert "type" in mask
return self.process(
process_id="mask_polygon",
arguments=dict_no_none(
data={"from_node": self._pg},
mask=mask,
replacement=replacement,
inside=inside
)
)
def _jsonable_batch_job_metadata(metadata: BatchJobMetadata, full=True) -> dict:
"""API-version-aware conversion of service metadata to jsonable dict"""
d = metadata.prepare_for_json()
# Fields to export
fields = ['id', 'title', 'description', 'status', 'created', 'updated', 'plan', 'costs', 'budget']
if full:
fields.extend(['process', 'progress'])
d = {k: v for (k, v) in d.items() if k in fields}
if requested_api_version().below("1.0.0"):
d["process_graph"] = d.pop("process", {}).get("process_graph")
d["submitted"] = d.pop("created", None)
# TODO wider status checking coverage?
if d["status"] == "created":
d["status"] = "submitted"
return dict_no_none(**d)
def filter_bbox(self,
west,
east,
north,
south,
crs=None,
base=None,
height=None) -> 'ImageCollection':
extent = {'west': west, 'east': east, 'north': north, 'south': south}
extent.update(dict_no_none(crs=crs, base=base, height=height))
return self.graph_add_process(process_id='filter_bbox',
args={
'data': {
'from_node': self.node_id
},
'extent': extent
})
def rename_labels(self, dimension: str, target: list, source: list = None) -> 'DataCube':
""" Renames the labels of the specified dimension in the data cube from source to target.
:param dimension: Dimension name
:param target: The new names for the labels.
:param source: The names of the labels as they are currently in the data cube.
:return: An DataCube instance
"""
return self.process(
process_id='rename_labels',
arguments=dict_no_none(
data={'from_node': self._pg},
dimension=self.metadata.assert_valid_dimension(dimension),
target=target,
source=source
)
)
def _build_token_response(self,
sub="123",
name="john",
include_id_token=True) -> str:
"""Build JSON serialized access/id/refresh token response (and store tokens for use in assertions)"""
access_token = self._jwt_encode({},
dict_no_none(
sub=sub,
name=name,
nonce=self.state.get("nonce")))
res = {
"access_token": access_token,
"refresh_token": self._jwt_encode({}, {"foo": "refresh"})
}
if include_id_token:
res["id_token"] = access_token
self.state.update(res)
return json.dumps(res)
def __init__(
self,
requests_mock: requests_mock.Mocker,
oidc_discovery_url: str,
expected_grant_type: Union[str, None],
expected_client_id: str = "myclient",
expected_fields: dict = None,
provider_root_url: str = "https://auth.test",
state: dict = None,
scopes_supported: List[str] = None,
device_code_flow_support: bool = True,
):
self.requests_mock = requests_mock
self.oidc_discovery_url = oidc_discovery_url
self.expected_grant_type = expected_grant_type
self.grant_request_history = []
self.expected_client_id = expected_client_id
self.expected_fields = expected_fields or {}
self.expected_authorization_code = None
self.provider_root_url = provider_root_url
self.authorization_endpoint = provider_root_url + "/auth"
self.token_endpoint = provider_root_url + "/token"
self.device_code_endpoint = provider_root_url + "/device_code" if device_code_flow_support else None
self.state = state or {}
self.scopes_supported = scopes_supported or [
"openid", "email", "profile"
]
self.requests_mock.get(
oidc_discovery_url,
text=json.dumps(
dict_no_none({
# Rudimentary OpenID Connect discovery document
"issuer": self.provider_root_url,
"authorization_endpoint": self.authorization_endpoint,
"token_endpoint": self.token_endpoint,
"device_authorization_endpoint": self.device_code_endpoint,
"scopes_supported": self.scopes_supported
})))
self.requests_mock.post(self.token_endpoint, text=self.token_callback)
if self.device_code_endpoint:
self.requests_mock.post(self.device_code_endpoint,
text=self.device_code_callback)
def rename_dimension(self, source: str, target: str):
"""
Renames a dimension in the data cube while preserving all other properties.
:param source: The current name of the dimension. Fails with a DimensionNotAvailable error if the specified dimension does not exist.
:param target: A new Name for the dimension. Fails with a DimensionExists error if a dimension with the specified name exists.
:return: A new datacube with the dimension renamed.
"""
if target in self.metadata.dimension_names():
raise ValueError('Target dimension name conflicts with existing dimension: %s.' % target)
return self.process(
process_id='rename_dimension',
arguments=dict_no_none(
data={'from_node': self._pg},
source=self.metadata.assert_valid_dimension(source),
target=target
)
)
def get_tokens(self) -> AccessTokenResult:
"""
Do OpenID authentication flow with PKCE:
get auth code and exchange for access and id token
"""
# Get auth code from authentication provider
auth_code_result = self._get_auth_code()
# Exchange authentication code for access token
result = self._do_token_post_request(post_data=dict_no_none(
grant_type=self.grant_type,
client_id=self.client_id,
client_secret=self.client_secret,
redirect_uri=auth_code_result.redirect_uri,
code=auth_code_result.auth_code,
code_verifier=auth_code_result.code_verifier,
))
return self._get_access_token_result(
result, expected_nonce=auth_code_result.nonce)
def create_job(self,
process_graph: dict,
title: str = None,
description: str = None,
plan: str = None,
budget=None,
additional: Dict = None) -> RESTJob:
"""
Posts a job to the back end.
:param process_graph: (flat) dict representing process graph
:param title: String title of the job
:param description: String description of the job
:param plan: billing plan
:param budget: Budget
:param additional: additional job options to pass to the backend
:return: job_id: String Job id of the new created job
"""
# TODO move all this (RESTJob factory) logic to RESTJob?
req = self._build_request_with_process_graph(
process_graph=process_graph,
**dict_no_none(title=title,
description=description,
plan=plan,
budget=budget))
if additional:
# TODO: get rid of this non-standard field? https://github.com/Open-EO/openeo-api/issues/276
req["job_options"] = additional
response = self.post("/jobs", json=req, expected_status=201)
if "openeo-identifier" in response.headers:
job_id = response.headers['openeo-identifier']
elif "location" in response.headers:
_log.warning(
"Backend did not explicitly respond with job id, will guess it from redirect URL."
)
job_id = response.headers['location'].split("/")[-1]
else:
raise OpenEoClientException("Failed fo extract job id")
return RESTJob(job_id, self)
def mask(self, mask: 'DataCube' = None, replacement=None) -> 'DataCube':
"""
Applies a mask to a raster data cube. To apply a vector mask use `mask_polygon`.
A mask is a raster data cube for which corresponding pixels among `data` and `mask`
are compared and those pixels in `data` are replaced whose pixels in `mask` are non-zero
(for numbers) or true (for boolean values).
The pixel values are replaced with the value specified for `replacement`,
which defaults to null (no data).
:param mask: the raster mask
:param replacement: the value to replace the masked pixels with
"""
return self.process(
process_id="mask",
arguments=dict_no_none(
data={'from_node': self._pg},
mask={'from_node': mask._pg},
replacement=replacement
)
)
def test_dict_no_none_kwargs():
assert dict_no_none() == {}
assert dict_no_none(a=3) == {"a": 3}
assert dict_no_none(a=3, b=0, c="foo") == {"a": 3, "b": 0, "c": "foo"}
assert dict_no_none(a=3, b="", c="foo") == {"a": 3, "b": "", "c": "foo"}
assert dict_no_none(a=3, b=None, c="foo") == {"a": 3, "c": "foo"}
