def test_download_results(con100, requests_mock, tmp_path):
requests_mock.get(API_URL + "/jobs/jj/results",
json={
"assets": {
"1.tiff": {
"href": API_URL + "/dl/jjr1.tiff",
"type": "image/tiff; application=geotiff"
},
"2.tiff": {
"href": API_URL + "/dl/jjr2.tiff",
"type": "image/tiff; application=geotiff"
},
}
})
requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT)
requests_mock.get(API_URL + "/dl/jjr2.tiff", content=TIFF_CONTENT)
job = RESTJob("jj", connection=con100)
target = as_path(tmp_path / "folder")
target.mkdir()
downloads = job.download_results(target)
assert downloads == {
target / "1.tiff": {
"href": API_URL + "/dl/jjr1.tiff",
"type": "image/tiff; application=geotiff"
},
target / "2.tiff": {
"href": API_URL + "/dl/jjr2.tiff",
"type": "image/tiff; application=geotiff"
},
}
assert set(p.name for p in target.iterdir()) == {"1.tiff", "2.tiff"}
with (target / "1.tiff").open("rb") as f:
assert f.read() == TIFF_CONTENT
def test_download_result_multiple(con100, requests_mock, tmp_path):
requests_mock.get(API_URL + "/jobs/jj/results", json={"assets": {
"1.tiff": {"href": API_URL + "/dl/jjr1.tiff"},
"2.tiff": {"href": API_URL + "/dl/jjr2.tiff"},
}})
job = RESTJob("jj", connection=con100)
with pytest.raises(OpenEoClientException, match="Expected one result file to download, but got 2"):
job.download_result(tmp_path / "res.tiff")
def test_download_result(con100, requests_mock, tmp_path):
requests_mock.get(API_URL + "/jobs/jj/results", json={"assets": {
"1.tiff": {"href": API_URL + "/dl/jjr1.tiff"},
}})
requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT)
job = RESTJob("jj", connection=con100)
target = as_path(tmp_path / "result.tiff")
res = job.download_result(target)
assert res == target
with target.open("rb") as f:
assert f.read() == TIFF_CONTENT
def test_get_results_download_files(con100, requests_mock, tmp_path):
requests_mock.get(API_URL + "/jobs/jj/results",
json={
"assets": {
"1.tiff": {
"href": API_URL + "/dl/jjr1.tiff",
"type": "image/tiff; application=geotiff"
},
"2.tiff": {
"href": API_URL + "/dl/jjr2.tiff",
"type": "image/tiff; application=geotiff"
},
}
})
requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT)
requests_mock.get(API_URL + "/dl/jjr2.tiff", content=TIFF_CONTENT)
job = RESTJob("jj", connection=con100)
assert job.list_results() == {
'assets': {
'1.tiff': {
'href': 'https://oeo.test/dl/jjr1.tiff',
'type': 'image/tiff; application=geotiff'
},
'2.tiff': {
'href': 'https://oeo.test/dl/jjr2.tiff',
'type': 'image/tiff; application=geotiff'
}
}
}
target = as_path(tmp_path / "folder")
target.mkdir()
results = job.get_results()
assets = job.get_results().get_assets()
assert {n: a.metadata
for n, a in assets.items()} == {
"1.tiff": {
"href": API_URL + "/dl/jjr1.tiff",
"type": "image/tiff; application=geotiff"
},
"2.tiff": {
"href": API_URL + "/dl/jjr2.tiff",
"type": "image/tiff; application=geotiff"
},
}
downloads = results.download_files(target)
assert set(downloads) == {target / "1.tiff", target / "2.tiff"}
assert set(p.name for p in target.iterdir()) == {"1.tiff", "2.tiff"}
with (target / "1.tiff").open("rb") as f:
assert f.read() == TIFF_CONTENT
def test_download_result_folder(con100, requests_mock, tmp_path):
requests_mock.get(API_URL + "/jobs/jj/results", json={"assets": {
"1.tiff": {"href": API_URL + "/dl/jjr1.tiff"},
}})
requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT)
job = RESTJob("jj", connection=con100)
target = as_path(tmp_path / "folder")
target.mkdir()
res = job.download_result(target)
assert res == target / "1.tiff"
assert list(p.name for p in target.iterdir()) == ["1.tiff"]
with (target / "1.tiff").open("rb") as f:
assert f.read() == TIFF_CONTENT
def create_job(self,
process_graph: Dict,
output_format: str = None,
output_parameters: Dict = {},
title: str = None,
description: str = None,
plan: str = None,
budget=None,
additional: Dict = {}):
"""
Posts a job to the back end.
:param process_graph: String data of the job (e.g. process graph)
:param output_format: String Output format of the execution - DEPRECATED in 0.4.0
:param output_parameters: Dict of additional output parameters - DEPRECATED in 0.4.0
:param title: String title of the job
:param description: String description of the job
:param budget: Budget
:return: job_id: String Job id of the new created job
"""
process_graph = {
"process_graph": process_graph,
"title": title,
"description": description,
"plan": plan,
"budget": budget
}
if not self._api_version.at_least('0.4.0'):
process_graph["output"] = {
"format": output_format,
"parameters": output_parameters
}
job_status = self.post("/jobs", process_graph)
job = None
if job_status.status_code == 201:
job_info = job_status.headers._store
if "openeo-identifier" in job_info:
job_id = job_info['openeo-identifier'][1]
job = RESTJob(job_id, self)
elif "location" in job_info:
job_id = job_info['location'][1].split("/")[-1]
job = RESTJob(job_id, self)
else:
self._handle_error_response(job_status)
return job
def test_get_results_download_file_other_domain(con100, requests_mock,
tmp_path):
"""https://github.com/Open-EO/openeo-python-client/issues/201"""
secret = "!secret token!"
requests_mock.get(API_URL + '/credentials/basic',
json={"access_token": secret})
def get_results(request, context):
assert "auth" in repr(request.headers).lower()
assert secret in repr(request.headers)
return {
"assets": {
"1.tiff": {
"href": "https://evilcorp.test/dl/jjr1.tiff",
"type": "image/tiff; application=geotiff"
},
}
}
def download_tiff(request, context):
assert "auth" not in repr(request.headers).lower()
assert secret not in repr(request.headers)
return TIFF_CONTENT
requests_mock.get(API_URL + "/jobs/jj1/results", json=get_results)
requests_mock.get("https://evilcorp.test/dl/jjr1.tiff",
content=download_tiff)
con100.authenticate_basic("john", "j0hn")
job = RESTJob("jj1", connection=con100)
target = as_path(tmp_path / "result.tiff")
res = job.get_results().download_file(target)
assert res == target
with target.open("rb") as f:
assert f.read() == TIFF_CONTENT
def test_get_results_multiple_download_single(con100, requests_mock, tmp_path):
requests_mock.get(API_URL + "/jobs/jj/results",
json={
"assets": {
"1.tiff": {
"href": API_URL + "/dl/jjr1.tiff"
},
"2.tiff": {
"href": API_URL + "/dl/jjr2.tiff"
},
}
})
job = RESTJob("jj", connection=con100)
expected = r"Failed to get single asset \(name None\) from \['1.tiff', '2.tiff'\]"
with pytest.raises(OpenEoClientException, match=expected):
job.get_results().download_file(tmp_path / "res.tiff")
def create_job(self, process_graph: dict, title: str = None, description: str = None,
plan: str = None, budget=None,
additional: Dict = None) -> RESTJob:
"""
Posts a job to the back end.
:param process_graph: (flat) dict representing process graph
:param title: String title of the job
:param description: String description of the job
:param plan: billing plan
:param budget: Budget
:param additional: additional job options to pass to the backend
:return: job_id: String Job id of the new created job
"""
# TODO move all this (RESTJob factory) logic to RESTJob?
req = self._build_request_with_process_graph(
process_graph=process_graph,
title=title, description=description, plan=plan, budget=budget
)
if additional:
# TODO: get rid of this non-standard field? https://github.com/Open-EO/openeo-api/issues/276
req["job_options"] = additional
response = self.post("/jobs", json=req, expected_status=201)
if "openeo-identifier" in response.headers:
job_id = response.headers['openeo-identifier']
elif "location" in response.headers:
_log.warning("Backend did not explicitly respond with job id, will guess it from redirect URL.")
job_id = response.headers['location'].split("/")[-1]
else:
raise OpenEoClientException("Failed fo extract job id")
return RESTJob(job_id, self)
def job_with_1_asset(con100, requests_mock, tmp_path) -> RESTJob:
requests_mock.get(API_URL + "/jobs/jj1/results", json={"assets": {
"1.tiff": {"href": API_URL + "/dl/jjr1.tiff", "type": "image/tiff; application=geotiff"},
}})
requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT)
job = RESTJob("jj1", connection=con100)
return job
def get_job(self, job_id: str):
"""
Returns job with the given id
:param job_id: Jpob Id
:return: job: RESTJob job object with the job id
"""
job = RESTJob(job_id, self)
return job
def test_result_asset_load_bytes(con100, requests_mock):
href = API_URL + "/dl/jjr1.tiff"
requests_mock.get(href, content=TIFF_CONTENT)
job = RESTJob("jj", connection=con100)
asset = ResultAsset(job, name="out.tiff", href=href, metadata={"type": "image/tiff; application=geotiff"})
res = asset.load_bytes()
assert res == TIFF_CONTENT
def test_result_asset_load_json(con100, requests_mock):
href = API_URL + "/dl/jjr1.json"
requests_mock.get(href, json={"bands": [1, 2, 3]})
job = RESTJob("jj", connection=con100)
asset = ResultAsset(job, name="out.json", href=href, metadata={"type": "application/json"})
res = asset.load_json()
assert res == {"bands": [1, 2, 3]}
def send_job(self, out_format=None, **format_options) -> Job:
"""
Sends a job to the backend and returns a ClientJob instance.
:param out_format: String Format of the job result.
:param format_options: String Parameters for the job result format
:return: status: ClientJob resulting job.
"""
if out_format:
return RESTJob(
self.session.job({
"process_graph": self.graph,
'output': {
'format': out_format,
'parameters': format_options
}
}), self.session)
else:
return RESTJob(self.session.job({"process_graph": self.graph}),
self.session)
def test_download_result_040(session040, requests_mock, tmp_path):
requests_mock.get(API_URL + "/jobs/jj/results",
json={"links": [
{
"href": API_URL + "/dl/jjr1.tiff"
},
]})
requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT)
job = RESTJob("jj", connection=session040)
assert job.list_results() == {
'links': [{
'href': 'https://oeo.test/dl/jjr1.tiff'
}]
}
target = as_path(tmp_path / "result.tiff")
res = job.download_result(target)
assert res == target
with target.open("rb") as f:
assert f.read() == TIFF_CONTENT
def job(self, job_id: str):
"""
Get the job based on the id. The job with the given id should already exist.
Use :py:meth:`openeo.rest.connection.Connection.create_job` to create new jobs
:param job_id: the job id of an existing job
:return: A job object.
"""
return RESTJob(job_id, self)
def create_job(self,
process_graph,
output_format=None,
output_parameters={},
title=None,
description=None,
plan=None,
budget=None,
additional={},
updated=None,
deleted=None):
"""
Posts a job to the back end.
:param process_graph: String data of the job (e.g. process graph)
:return: job_id: String Job id of the new created job
"""
process_graph = {"process_graph": process_graph}
# Simulating Changes at the data back end.
if updated:
process_graph["updated"] = updated
if deleted:
process_graph["deleted"] = deleted
job_status = self.post("/jobs", process_graph)
job = None
if job_status.status_code == 201:
job_info = job_status.headers._store
if "openeo-identifier" in job_info:
job_id = job_info['openeo-identifier'][1]
job = RESTJob(job_id, self)
elif "location" in job_info:
job_id = job_info['location'][1].split("/")[-1]
job = RESTJob(job_id, self)
else:
return None #raise Exception(job_status)
return job
def test_get_results_multiple_download_single_by_name(con100, requests_mock,
tmp_path):
requests_mock.get(API_URL + "/jobs/jj/results",
json={
"assets": {
"1.tiff": {
"href": API_URL + "/dl/jjr1.tiff"
},
"2.tiff": {
"href": API_URL + "/dl/jjr2.tiff"
},
}
})
requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT)
job = RESTJob("jj", connection=con100)
target = as_path(tmp_path / "res.tiff")
path = job.get_results().download_file(target, name="1.tiff")
assert path == target
assert list(p.name for p in tmp_path.iterdir()) == ["res.tiff"]
with path.open("rb") as f:
assert f.read() == TIFF_CONTENT
def test_result_asset_download_file(con100, requests_mock, tmp_path):
href = API_URL + "/dl/jjr1.tiff"
requests_mock.get(href, content=TIFF_CONTENT)
job = RESTJob("jj", connection=con100)
asset = ResultAsset(job, name="1.tiff", href=href, metadata={'type': 'image/tiff; application=geotiff'})
target = as_path(tmp_path / "res.tiff")
path = asset.download(target)
assert isinstance(path, Path)
assert path.name == "res.tiff"
with path.open("rb") as f:
assert f.read() == TIFF_CONTENT
def job_logs(self, job_id, offset) -> list:
"""Get batch job logs."""
return RESTJob(job_id, connection=self).logs(offset=offset)
job = timeseries.send_job()
logging.debug("{}".format(job.job_id))
status = job.queue()
logging.debug("{}".format(status))
# minutes = 0
# minutes_steps = 1
#
# while status != "Finished":
# time.sleep(minutes_steps*60)
# minutes += 1
# status = job.status()['status']
# logging.debug("After {} minutes the status is: {}".format(minutes, status))
job = RESTJob(97, session)
job.download(OUTPUT_FILE)
# JSON:
#
# {
# "process_graph":{
# "process_id":"min_time",
# "args":{
# "imagery":{
# "process_id":"NDVI",
# "args":{
# "imagery":{
# "process_id":"filter_daterange",
# "args":{
def job_results(self, job_id) -> dict:
"""Get batch job results metadata."""
return RESTJob(job_id, connection=self).list_results()
def job_results(self, job_id):
return RESTJob(job_id, connection=self).list_results()
logging.info("Starting Job A...")
jobA.start_job()
# Wait until the job execution was finished
logging.info("Job A Processing...")
desc = jobA.describe_job
while desc["status"] == "submitted":
desc = jobA.describe_job
''' 2. Researcher B re-runs the same experiment (job B). '''
logging.info("2. Researcher B re-runs the same experiment (job B).")
# Get the already executed job A by Id
from openeo.rest.job import RESTJob
jobA_new = RESTJob(jobA.job_id, con)
# Get process graph of job A and create new job B with it
pgA = jobA_new.describe_job["process_graph"]
logging.info("Creating Job B and retrieving Job B ID...")
jobB = con.create_job(pgA)
logging.info("Job B with ID {} created...".format(jobB.job_id))
logging.info("Starting Job B...")
jobB.start_job()
''' 3. Researcher runs a different experiment (job C). '''
logging.info("3. Researcher runs a different experiment (job C).")
# Choose dataset
processes = con.get_processes()
