def test_batch_job_metadata_to_api_dict():
api_version = ComparableVersion("1.0.0")
job = BatchJobMetadata(
id="123", status="running", created=datetime.datetime(2022, 1, 18, 16, 42, 0),
process={"add": {"process_id": "add", "arguments": {"x": 3, "y": 5}, "result": True}},
title="Untitled01", description="Lorem ipsum.",
progress=0.3,
cpu_time=datetime.timedelta(seconds=1000),
memory_time_megabyte=datetime.timedelta(seconds=2000),
started=datetime.datetime(2022, 1, 18, 17, 0, 0),
finished=datetime.datetime(2022, 1, 18, 17, 20, 0),
epsg=4326,
links=[{}],
)
assert job.to_api_dict(full=False, api_version=api_version) == {
"id": "123",
"title": "Untitled01", "description": "Lorem ipsum.",
"status": "running",
"progress": 0.3,
"created": "2022-01-18T16:42:00Z",
}
assert job.to_api_dict(full=True, api_version=api_version) == {
"id": "123",
"title": "Untitled01", "description": "Lorem ipsum.",
"process": {"add": {"process_id": "add", "arguments": {"x": 3, "y": 5}, "result": True}},
"status": "running",
"progress": 0.3,
"created": "2022-01-18T16:42:00Z",
"usage": {
"cpu": {"value": 1000, "unit": "cpu-seconds"},
"memory": {"value": 2000, "unit": "mb-seconds"},
"duration": {"value": 1200, "unit": "seconds"},
}
}
def test_batch_job_metadata_from_api_dict_basic():
job = BatchJobMetadata.from_api_dict({
"id": "ba7c470b", "created": "2021-06-18T12:34:56Z", "status": "running",
})
assert job.id == "ba7c470b"
assert job.created == datetime.datetime(2021, 6, 18, 12, 34, 56)
assert job.status == "running"
# Full round trip check
assert job == BatchJobMetadata.from_api_dict(job.to_api_dict())
def test_batch_job_metadata_from_api_dict_auto_conversions():
job = BatchJobMetadata.from_api_dict({
"id": "ba7c470b",
"status": "running",
"created": "2021-06-18T12:34:56Z",
"updated": "2021-06-20T20:20:20Z",
})
assert job.created == datetime.datetime(2021, 6, 18, 12, 34, 56)
assert job.updated == datetime.datetime(2021, 6, 20, 20, 20, 20)
# Full round trip check
assert job == BatchJobMetadata.from_api_dict(job.to_api_dict())
def _parse_job_info(self, job_info: dict) -> BatchJobMetadata:
status = job_info.get("status")
if status == "submitted":
status = "created"
return BatchJobMetadata(id=job_info["job_id"],
process=json.loads(job_info["specification"]),
status=status,
created=parse_rfc3339(job_info["created"])
if "created" in job_info else None)
def test_batch_job_metadata_from_api_dict_usage():
job = BatchJobMetadata.from_api_dict({
"id": "ba7c470b", "created": "2021-06-18T12:34:56Z", "status": "running",
"usage": {
"cpu": {"value": 1000, "unit": "cpu-seconds"},
"memory": {"value": 2000, "unit": "mb-seconds"},
"duration": {"value": 3000, "unit": "seconds"},
}
})
assert job.id == "ba7c470b"
assert job.created == datetime.datetime(2021, 6, 18, 12, 34, 56)
assert job.status == "running"
assert job.cpu_time == datetime.timedelta(seconds=1000)
assert job.memory_time_megabyte == datetime.timedelta(seconds=2000)
assert job.duration == datetime.timedelta(seconds=3000)
assert job.duration_ == datetime.timedelta(seconds=3000)
# Full round trip check
assert job == BatchJobMetadata.from_api_dict(job.to_api_dict())
def create_job(self, user_id: str, job_specification: dict,
api_version: str) -> BatchJobMetadata:
job_id = str(uuid.uuid4())
with JobRegistry() as registry:
job_info = registry.register(job_id=job_id,
user_id=user_id,
api_version=api_version,
specification=job_specification)
return BatchJobMetadata(id=job_id,
process=job_specification,
status=job_info["status"],
created=parse_rfc3339(job_info["created"]))
def create_job(self,
user_id: str,
process: dict,
api_version: str,
metadata: dict,
job_options: dict = None) -> BatchJobMetadata:
job_id = self.generate_job_id()
job_info = BatchJobMetadata(id=job_id,
status="created",
process=process,
created=utcnow(),
job_options=job_options,
title=metadata.get("title"),
description=metadata.get("description"))
self._job_registry[(user_id, job_id)] = job_info
return job_info
def _jsonable_batch_job_metadata(metadata: BatchJobMetadata, full=True) -> dict:
"""API-version-aware conversion of service metadata to jsonable dict"""
d = metadata.prepare_for_json()
# Fields to export
fields = ['id', 'title', 'description', 'status', 'created', 'updated', 'plan', 'costs', 'budget']
if full:
fields.extend(['process', 'progress'])
d = {k: v for (k, v) in d.items() if k in fields}
if requested_api_version().below("1.0.0"):
d["process_graph"] = d.pop("process", {}).get("process_graph")
d["submitted"] = d.pop("created", None)
# TODO wider status checking coverage?
if d["status"] == "created":
d["status"] = "submitted"
return dict_no_none(**d)
def _fresh_job_registry(next_job_id):
"""Set up a fresh job registry and predefine next job id"""
with mock.patch.object(dummy_backend.DummyBatchJobs,
'generate_job_id',
return_value=next_job_id):
dummy_backend.DummyBatchJobs._job_registry = {
(TEST_USER, '07024ee9-7847-4b8a-b260-6c879a2b3cdc'):
BatchJobMetadata(
id='07024ee9-7847-4b8a-b260-6c879a2b3cdc',
status='running',
process={
'process_graph': {
'foo': {
'process_id': 'foo',
'arguments': {}
}
}
},
created=datetime(2017, 1, 1, 9, 32, 12),
)
}
yield
def job_info_to_metadata(job_info: dict) -> BatchJobMetadata:
"""Convert job info dict to BatchJobMetadata"""
status = job_info.get("status")
if status == "submitted":
status = "created"
specification = job_info["specification"]
if isinstance(specification, str):
specification = json.loads(specification)
job_options = specification.pop("job_options", None)
def map_safe(prop: str, f):
value = job_info.get(prop)
return f(value) if value else None
return BatchJobMetadata(
id=job_info["job_id"],
process=specification,
title=job_info.get("title"),
description=job_info.get("description"),
status=status,
created=map_safe("created", rfc3339.parse_datetime),
updated=map_safe("updated", rfc3339.parse_datetime),
job_options=job_options,
started=map_safe("started", rfc3339.parse_datetime),
finished=map_safe("finished", rfc3339.parse_datetime),
memory_time_megabyte=map_safe(
"memory_time_megabyte_seconds",
lambda seconds: timedelta(seconds=seconds)),
cpu_time=map_safe("cpu_time_seconds",
lambda seconds: timedelta(seconds=seconds)),
geometry=job_info.get("geometry"),
bbox=job_info.get("bbox"),
start_datetime=map_safe("start_datetime", rfc3339.parse_datetime),
end_datetime=map_safe("end_datetime", rfc3339.parse_datetime),
instruments=job_info.get("instruments", []),
epsg=job_info.get("epsg"),
links=job_info.get("links", []))
def test_batch_job_metadata_from_api_dict_emtpy():
with pytest.raises(KeyError, match="Missing BatchJobMetadata fields: created, id, status"):
_ = BatchJobMetadata.from_api_dict({})