def to_api_dict(self, full=True, api_version: ComparableVersion = None) -> dict:
"""
API-version-aware conversion of batch job metadata to jsonable openEO API compatible dict.
see https://openeo.org/documentation/1.0/developers/api/reference.html#operation/describe-job
"""
# Basic/full fields to export
fields = ["id", "title", "description", "status", "progress", "created", "updated", "plan", "costs", "budget"]
if full:
fields.extend(["process"])
result = {f: getattr(self, f) for f in fields}
# Additional cleaning and massaging.
result["created"] = rfc3339.datetime(self.created) if self.created else None
result["updated"] = rfc3339.datetime(self.updated) if self.updated else None
if full:
usage = self.usage or {}
if self.cpu_time:
usage["cpu"] = {"value": int(round(self.cpu_time.total_seconds())), "unit": "cpu-seconds"}
if self.duration:
usage["duration"] = {"value": int(round(self.duration.total_seconds())), "unit": "seconds"}
if self.memory_time_megabyte:
usage["memory"] = {"value": int(round(self.memory_time_megabyte.total_seconds())), "unit": "mb-seconds"}
if usage:
result["usage"] = usage
if api_version and api_version.below("1.0.0"):
result["process_graph"] = result.pop("process", {}).get("process_graph")
result["submitted"] = result.pop("created", None)
# TODO wider status checking coverage?
if result["status"] == "created":
result["status"] = "submitted"
return dict_no_none(result)
def set_status(self, job_id: str, user_id: str, status: str) -> None:
"""Updates a registered batch job with its status. Additionally, updates its "updated" property."""
self.patch(job_id,
user_id,
status=status,
updated=rfc3339.datetime(datetime.utcnow()))
_log.debug("batch job {j} -> {s}".format(j=job_id, s=status))
def register(self,
job_id: str,
user_id: str,
api_version: str,
specification: dict,
title: str = None,
description: str = None) -> dict:
"""Registers a to-be-run batch job."""
# TODO: use `BatchJobMetadata` instead of free form dict here?
job_info = {
'job_id': job_id,
'user_id': user_id,
'status': 'created',
# TODO: move api_Version into specification?
'api_version': api_version,
# TODO: why json-encoding `specification` when the whole job_info dict will be json-encoded anyway?
'specification': json.dumps(specification),
'application_id': None,
'created': rfc3339.datetime(datetime.utcnow()),
'title': title,
'description': description,
}
self._create(job_info)
return job_info
def test_rfc3339_datetime():
assert "2020-03-17T00:00:00Z" == rfc3339.datetime("2020-03-17")
assert "2020-03-17T00:00:00Z" == rfc3339.datetime("2020/03/17")
assert "2020-03-17T00:00:00Z" == rfc3339.datetime("2020:03:17")
assert "2020-03-17T00:00:00Z" == rfc3339.datetime("2020_03_17")
assert "2020-03-17T12:34:56Z" == rfc3339.datetime("2020-03-17-12-34-56")
assert "2020-03-17T12:34:00Z" == rfc3339.datetime("2020-03-17-12-34")
assert "2020-03-17T12:34:56Z" == rfc3339.datetime("2020:03:17:12:34:56")
assert "2020-03-17T12:34:56Z" == rfc3339.datetime("2020/03/17/12/34/56")
assert "2020-03-17T12:34:00Z" == rfc3339.datetime("2020/03/17/12/34")
assert "2020-03-17T12:34:56Z" == rfc3339.datetime("2020_03_17_12_34_56")
assert "2020-03-17T12:34:56Z" == rfc3339.datetime("2020-03-17T12:34:56Z")
assert "2020-03-17T00:00:00Z" == rfc3339.datetime(date(2020, 3, 17))
assert "2020-03-17T12:34:56Z" == rfc3339.datetime(datetime(2020, 3, 17, 12, 34, 56))
assert "2020-03-17T00:00:00Z" == rfc3339.datetime((2020, 3, 17))
assert "2020-03-17T00:00:00Z" == rfc3339.datetime([2020, 3, 17])
assert "2020-03-17T00:00:00Z" == rfc3339.datetime(2020, 3, 17)
assert "2020-03-17T12:34:56Z" == rfc3339.datetime((2020, 3, 17, 12, 34, 56))
assert "2020-03-17T12:34:56Z" == rfc3339.datetime([2020, 3, 17, 12, 34, 56])
assert "2020-03-17T12:34:56Z" == rfc3339.datetime(2020, 3, 17, 12, 34, 56)
assert "2020-03-17T12:34:00Z" == rfc3339.datetime(2020, 3, 17, 12, 34)
assert "2020-03-17T12:34:56Z" == rfc3339.datetime((2020, "3", 17, "12", "34", 56))
assert "2020-09-17T12:34:56Z" == rfc3339.datetime([2020, "09", 17, "12", "34", 56])
assert "2020-09-17T12:34:56Z" == rfc3339.datetime(2020, "09", "17", "12", "34", 56)
def load_test_collection(
collection_id: str,
collection_metadata: GeopysparkCubeMetadata,
extent,
srs: str,
from_date: str,
to_date: str,
bands=None,
correlation_id: str = "NA",
) -> Dict[int, geopyspark.TiledRasterLayer]:
"""
Load synthetic data as test collection
:param collection_id:
:param collection_metadata:
:param extent:
:param srs:
:param from_date:
:param to_date:
:param bands:
:param correlation_id:
:return:
"""
# TODO: support more test collections
assert collection_id == "TestCollection-LonLat4x4"
grid_size: float = 1.0
tile_size = 4
# TODO: support other srs'es?
assert srs == "EPSG:4326"
# Get bounds of tiling layout
extent = geopyspark.Extent(extent.xmin(), extent.ymin(), extent.xmax(),
extent.ymax())
col_min = int(math.floor(extent.xmin / grid_size))
row_min = int(math.floor(extent.ymin / grid_size))
col_max = int(math.ceil(extent.xmax / grid_size) - 1)
row_max = int(math.ceil(extent.ymax / grid_size) - 1)
# Simulate sparse range of observation dates
from_date = rfc3339.parse_datetime(rfc3339.datetime(from_date))
to_date = rfc3339.parse_datetime(rfc3339.datetime(to_date))
dates = dates_between(from_date, to_date)
# Build RDD of tiles with requested bands.
tile_builder = TestCollectionLonLat(tile_size=tile_size,
grid_size=grid_size)
bands = bands or [b.name for b in collection_metadata.bands]
rdd_data = [(SpaceTimeKey(col, row, date),
tile_builder.get_tile(bands=bands,
col=col,
row=row,
date=date))
for col in range(col_min, col_max + 1)
for row in range(row_min, row_max + 1) for date in dates]
rdd = SparkContext.getOrCreate().parallelize(rdd_data)
metadata = Metadata(
bounds=Bounds(SpaceTimeKey(col_min, row_min, min(dates)),
SpaceTimeKey(col_max, row_max, max(dates))),
crs="+proj=longlat +datum=WGS84 +no_defs ",
cell_type=CellType.FLOAT64,
extent=extent,
layout_definition=LayoutDefinition(
extent=geopyspark.Extent(col_min * grid_size, row_min * grid_size,
(col_max + 1) * grid_size,
(row_max + 1) * grid_size),
tileLayout=TileLayout(layoutCols=col_max - col_min + 1,
layoutRows=row_max - row_min + 1,
tileCols=tile_size,
tileRows=tile_size)))
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
return {0: layer}
def prepare_for_json(self) -> dict:
"""Prepare metadata for JSON serialization"""
d = self._asdict() # pylint: disable=no-member
d["created"] = rfc3339.datetime(self.created) if self.created else None
return d
def utcnow_rfc3339() -> str:
"""Current datetime formatted as RFC-3339 string."""
return rfc3339.datetime(datetime.utcnow())