def test_download_results(con100, requests_mock, tmp_path): requests_mock.get(API_URL + "/jobs/jj/results", json={ "assets": { "1.tiff": { "href": API_URL + "/dl/jjr1.tiff", "type": "image/tiff; application=geotiff" }, "2.tiff": { "href": API_URL + "/dl/jjr2.tiff", "type": "image/tiff; application=geotiff" }, } }) requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT) requests_mock.get(API_URL + "/dl/jjr2.tiff", content=TIFF_CONTENT) job = RESTJob("jj", connection=con100) target = as_path(tmp_path / "folder") target.mkdir() downloads = job.download_results(target) assert downloads == { target / "1.tiff": { "href": API_URL + "/dl/jjr1.tiff", "type": "image/tiff; application=geotiff" }, target / "2.tiff": { "href": API_URL + "/dl/jjr2.tiff", "type": "image/tiff; application=geotiff" }, } assert set(p.name for p in target.iterdir()) == {"1.tiff", "2.tiff"} with (target / "1.tiff").open("rb") as f: assert f.read() == TIFF_CONTENT
def test_download_result_multiple(con100, requests_mock, tmp_path): requests_mock.get(API_URL + "/jobs/jj/results", json={"assets": { "1.tiff": {"href": API_URL + "/dl/jjr1.tiff"}, "2.tiff": {"href": API_URL + "/dl/jjr2.tiff"}, }}) job = RESTJob("jj", connection=con100) with pytest.raises(OpenEoClientException, match="Expected one result file to download, but got 2"): job.download_result(tmp_path / "res.tiff")
def test_download_result(con100, requests_mock, tmp_path): requests_mock.get(API_URL + "/jobs/jj/results", json={"assets": { "1.tiff": {"href": API_URL + "/dl/jjr1.tiff"}, }}) requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT) job = RESTJob("jj", connection=con100) target = as_path(tmp_path / "result.tiff") res = job.download_result(target) assert res == target with target.open("rb") as f: assert f.read() == TIFF_CONTENT
def test_get_results_download_files(con100, requests_mock, tmp_path): requests_mock.get(API_URL + "/jobs/jj/results", json={ "assets": { "1.tiff": { "href": API_URL + "/dl/jjr1.tiff", "type": "image/tiff; application=geotiff" }, "2.tiff": { "href": API_URL + "/dl/jjr2.tiff", "type": "image/tiff; application=geotiff" }, } }) requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT) requests_mock.get(API_URL + "/dl/jjr2.tiff", content=TIFF_CONTENT) job = RESTJob("jj", connection=con100) assert job.list_results() == { 'assets': { '1.tiff': { 'href': 'https://oeo.test/dl/jjr1.tiff', 'type': 'image/tiff; application=geotiff' }, '2.tiff': { 'href': 'https://oeo.test/dl/jjr2.tiff', 'type': 'image/tiff; application=geotiff' } } } target = as_path(tmp_path / "folder") target.mkdir() results = job.get_results() assets = job.get_results().get_assets() assert {n: a.metadata for n, a in assets.items()} == { "1.tiff": { "href": API_URL + "/dl/jjr1.tiff", "type": "image/tiff; application=geotiff" }, "2.tiff": { "href": API_URL + "/dl/jjr2.tiff", "type": "image/tiff; application=geotiff" }, } downloads = results.download_files(target) assert set(downloads) == {target / "1.tiff", target / "2.tiff"} assert set(p.name for p in target.iterdir()) == {"1.tiff", "2.tiff"} with (target / "1.tiff").open("rb") as f: assert f.read() == TIFF_CONTENT
def test_download_result_folder(con100, requests_mock, tmp_path): requests_mock.get(API_URL + "/jobs/jj/results", json={"assets": { "1.tiff": {"href": API_URL + "/dl/jjr1.tiff"}, }}) requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT) job = RESTJob("jj", connection=con100) target = as_path(tmp_path / "folder") target.mkdir() res = job.download_result(target) assert res == target / "1.tiff" assert list(p.name for p in target.iterdir()) == ["1.tiff"] with (target / "1.tiff").open("rb") as f: assert f.read() == TIFF_CONTENT
def create_job(self, process_graph: Dict, output_format: str = None, output_parameters: Dict = {}, title: str = None, description: str = None, plan: str = None, budget=None, additional: Dict = {}): """ Posts a job to the back end. :param process_graph: String data of the job (e.g. process graph) :param output_format: String Output format of the execution - DEPRECATED in 0.4.0 :param output_parameters: Dict of additional output parameters - DEPRECATED in 0.4.0 :param title: String title of the job :param description: String description of the job :param budget: Budget :return: job_id: String Job id of the new created job """ process_graph = { "process_graph": process_graph, "title": title, "description": description, "plan": plan, "budget": budget } if not self._api_version.at_least('0.4.0'): process_graph["output"] = { "format": output_format, "parameters": output_parameters } job_status = self.post("/jobs", process_graph) job = None if job_status.status_code == 201: job_info = job_status.headers._store if "openeo-identifier" in job_info: job_id = job_info['openeo-identifier'][1] job = RESTJob(job_id, self) elif "location" in job_info: job_id = job_info['location'][1].split("/")[-1] job = RESTJob(job_id, self) else: self._handle_error_response(job_status) return job
def test_get_results_download_file_other_domain(con100, requests_mock, tmp_path): """https://github.com/Open-EO/openeo-python-client/issues/201""" secret = "!secret token!" requests_mock.get(API_URL + '/credentials/basic', json={"access_token": secret}) def get_results(request, context): assert "auth" in repr(request.headers).lower() assert secret in repr(request.headers) return { "assets": { "1.tiff": { "href": "https://evilcorp.test/dl/jjr1.tiff", "type": "image/tiff; application=geotiff" }, } } def download_tiff(request, context): assert "auth" not in repr(request.headers).lower() assert secret not in repr(request.headers) return TIFF_CONTENT requests_mock.get(API_URL + "/jobs/jj1/results", json=get_results) requests_mock.get("https://evilcorp.test/dl/jjr1.tiff", content=download_tiff) con100.authenticate_basic("john", "j0hn") job = RESTJob("jj1", connection=con100) target = as_path(tmp_path / "result.tiff") res = job.get_results().download_file(target) assert res == target with target.open("rb") as f: assert f.read() == TIFF_CONTENT
def test_get_results_multiple_download_single(con100, requests_mock, tmp_path): requests_mock.get(API_URL + "/jobs/jj/results", json={ "assets": { "1.tiff": { "href": API_URL + "/dl/jjr1.tiff" }, "2.tiff": { "href": API_URL + "/dl/jjr2.tiff" }, } }) job = RESTJob("jj", connection=con100) expected = r"Failed to get single asset \(name None\) from \['1.tiff', '2.tiff'\]" with pytest.raises(OpenEoClientException, match=expected): job.get_results().download_file(tmp_path / "res.tiff")
def create_job(self, process_graph: dict, title: str = None, description: str = None, plan: str = None, budget=None, additional: Dict = None) -> RESTJob: """ Posts a job to the back end. :param process_graph: (flat) dict representing process graph :param title: String title of the job :param description: String description of the job :param plan: billing plan :param budget: Budget :param additional: additional job options to pass to the backend :return: job_id: String Job id of the new created job """ # TODO move all this (RESTJob factory) logic to RESTJob? req = self._build_request_with_process_graph( process_graph=process_graph, title=title, description=description, plan=plan, budget=budget ) if additional: # TODO: get rid of this non-standard field? https://github.com/Open-EO/openeo-api/issues/276 req["job_options"] = additional response = self.post("/jobs", json=req, expected_status=201) if "openeo-identifier" in response.headers: job_id = response.headers['openeo-identifier'] elif "location" in response.headers: _log.warning("Backend did not explicitly respond with job id, will guess it from redirect URL.") job_id = response.headers['location'].split("/")[-1] else: raise OpenEoClientException("Failed fo extract job id") return RESTJob(job_id, self)
def job_with_1_asset(con100, requests_mock, tmp_path) -> RESTJob: requests_mock.get(API_URL + "/jobs/jj1/results", json={"assets": { "1.tiff": {"href": API_URL + "/dl/jjr1.tiff", "type": "image/tiff; application=geotiff"}, }}) requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT) job = RESTJob("jj1", connection=con100) return job
def get_job(self, job_id: str): """ Returns job with the given id :param job_id: Jpob Id :return: job: RESTJob job object with the job id """ job = RESTJob(job_id, self) return job
def test_result_asset_load_bytes(con100, requests_mock): href = API_URL + "/dl/jjr1.tiff" requests_mock.get(href, content=TIFF_CONTENT) job = RESTJob("jj", connection=con100) asset = ResultAsset(job, name="out.tiff", href=href, metadata={"type": "image/tiff; application=geotiff"}) res = asset.load_bytes() assert res == TIFF_CONTENT
def test_result_asset_load_json(con100, requests_mock): href = API_URL + "/dl/jjr1.json" requests_mock.get(href, json={"bands": [1, 2, 3]}) job = RESTJob("jj", connection=con100) asset = ResultAsset(job, name="out.json", href=href, metadata={"type": "application/json"}) res = asset.load_json() assert res == {"bands": [1, 2, 3]}
def send_job(self, out_format=None, **format_options) -> Job: """ Sends a job to the backend and returns a ClientJob instance. :param out_format: String Format of the job result. :param format_options: String Parameters for the job result format :return: status: ClientJob resulting job. """ if out_format: return RESTJob( self.session.job({ "process_graph": self.graph, 'output': { 'format': out_format, 'parameters': format_options } }), self.session) else: return RESTJob(self.session.job({"process_graph": self.graph}), self.session)
def test_download_result_040(session040, requests_mock, tmp_path): requests_mock.get(API_URL + "/jobs/jj/results", json={"links": [ { "href": API_URL + "/dl/jjr1.tiff" }, ]}) requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT) job = RESTJob("jj", connection=session040) assert job.list_results() == { 'links': [{ 'href': 'https://oeo.test/dl/jjr1.tiff' }] } target = as_path(tmp_path / "result.tiff") res = job.download_result(target) assert res == target with target.open("rb") as f: assert f.read() == TIFF_CONTENT
def job(self, job_id: str): """ Get the job based on the id. The job with the given id should already exist. Use :py:meth:`openeo.rest.connection.Connection.create_job` to create new jobs :param job_id: the job id of an existing job :return: A job object. """ return RESTJob(job_id, self)
def create_job(self, process_graph, output_format=None, output_parameters={}, title=None, description=None, plan=None, budget=None, additional={}, updated=None, deleted=None): """ Posts a job to the back end. :param process_graph: String data of the job (e.g. process graph) :return: job_id: String Job id of the new created job """ process_graph = {"process_graph": process_graph} # Simulating Changes at the data back end. if updated: process_graph["updated"] = updated if deleted: process_graph["deleted"] = deleted job_status = self.post("/jobs", process_graph) job = None if job_status.status_code == 201: job_info = job_status.headers._store if "openeo-identifier" in job_info: job_id = job_info['openeo-identifier'][1] job = RESTJob(job_id, self) elif "location" in job_info: job_id = job_info['location'][1].split("/")[-1] job = RESTJob(job_id, self) else: return None #raise Exception(job_status) return job
def test_get_results_multiple_download_single_by_name(con100, requests_mock, tmp_path): requests_mock.get(API_URL + "/jobs/jj/results", json={ "assets": { "1.tiff": { "href": API_URL + "/dl/jjr1.tiff" }, "2.tiff": { "href": API_URL + "/dl/jjr2.tiff" }, } }) requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT) job = RESTJob("jj", connection=con100) target = as_path(tmp_path / "res.tiff") path = job.get_results().download_file(target, name="1.tiff") assert path == target assert list(p.name for p in tmp_path.iterdir()) == ["res.tiff"] with path.open("rb") as f: assert f.read() == TIFF_CONTENT
def test_result_asset_download_file(con100, requests_mock, tmp_path): href = API_URL + "/dl/jjr1.tiff" requests_mock.get(href, content=TIFF_CONTENT) job = RESTJob("jj", connection=con100) asset = ResultAsset(job, name="1.tiff", href=href, metadata={'type': 'image/tiff; application=geotiff'}) target = as_path(tmp_path / "res.tiff") path = asset.download(target) assert isinstance(path, Path) assert path.name == "res.tiff" with path.open("rb") as f: assert f.read() == TIFF_CONTENT
def job_logs(self, job_id, offset) -> list: """Get batch job logs.""" return RESTJob(job_id, connection=self).logs(offset=offset)
job = timeseries.send_job() logging.debug("{}".format(job.job_id)) status = job.queue() logging.debug("{}".format(status)) # minutes = 0 # minutes_steps = 1 # # while status != "Finished": # time.sleep(minutes_steps*60) # minutes += 1 # status = job.status()['status'] # logging.debug("After {} minutes the status is: {}".format(minutes, status)) job = RESTJob(97, session) job.download(OUTPUT_FILE) # JSON: # # { # "process_graph":{ # "process_id":"min_time", # "args":{ # "imagery":{ # "process_id":"NDVI", # "args":{ # "imagery":{ # "process_id":"filter_daterange", # "args":{
def job_results(self, job_id) -> dict: """Get batch job results metadata.""" return RESTJob(job_id, connection=self).list_results()
def job_results(self, job_id): return RESTJob(job_id, connection=self).list_results()
logging.info("Starting Job A...") jobA.start_job() # Wait until the job execution was finished logging.info("Job A Processing...") desc = jobA.describe_job while desc["status"] == "submitted": desc = jobA.describe_job ''' 2. Researcher B re-runs the same experiment (job B). ''' logging.info("2. Researcher B re-runs the same experiment (job B).") # Get the already executed job A by Id from openeo.rest.job import RESTJob jobA_new = RESTJob(jobA.job_id, con) # Get process graph of job A and create new job B with it pgA = jobA_new.describe_job["process_graph"] logging.info("Creating Job B and retrieving Job B ID...") jobB = con.create_job(pgA) logging.info("Job B with ID {} created...".format(jobB.job_id)) logging.info("Starting Job B...") jobB.start_job() ''' 3. Researcher runs a different experiment (job C). ''' logging.info("3. Researcher runs a different experiment (job C).") # Choose dataset processes = con.get_processes()