def prepare(self): visible = self.is_visible() if not visible: # includes private visibility and non-existing cases if visible is None: LOGGER.info( "Process [%s] access is unauthorized on [%s] - deploying as admin.", self.process, self.url) elif visible is False: LOGGER.info( "Process [%s] is not deployed on [%s] - deploying.", self.process, self.url) # TODO: Maybe always redeploy? What about cases of outdated deployed process? try: self.deploy() except Exception as exc: pass_http_error(exc, [HTTPConflict]) if visible: LOGGER.info( "Process [%s] already deployed and visible on [%s] - executing.", self.process, self.url) else: LOGGER.info("Process [%s] enforcing to public visibility.", self.process) try: self.set_visibility(visibility=VISIBILITY_PUBLIC) except Exception as exc: pass_http_error(exc, HTTPNotFound) LOGGER.warning( "Process [%s] failed setting public visibility. " "Assuming feature is not supported by ADES and process is already public.", self.process)
def test_pass_http_error_doesnt_raise_multi_pyramid_error(): http_errors = [HTTPNotFound, HTTPInternalServerError] for err in http_errors: # test try/except try: # normal usage try/except try: raise_http_error(err) except Exception as ex: utils.pass_http_error(ex, http_errors) except PyramidHTTPError: pytest.fail("PyramidHTTPError should be ignored but was raised.")
def test_pass_http_error_doesnt_raise_requests_error(): http_errors = [HTTPNotFound, HTTPInternalServerError] for err in http_errors: req_err = make_http_error(err) # test try/except try: # normal usage try/except try: raise_http_error(req_err) except Exception as ex: utils.pass_http_error(ex, err) except RequestsHTTPError: pytest.fail("RequestsHTTPError should be ignored but was raised.")
def test_pass_http_error_raises_other_error_with_multi_pyramid_error(): with pytest.raises(ValueError): try: raise ValueError("Test Error") except Exception as ex: utils.pass_http_error(ex, [HTTPConflict, HTTPInternalServerError])
def test_pass_http_error_raises_other_error_with_single_pyramid_error(): with pytest.raises(ValueError): try: raise ValueError("Test Error") except Exception as ex: utils.pass_http_error(ex, HTTPConflict)
def test_pass_http_error_raises_requests_error_with_multi_pyramid_error(): with pytest.raises(RequestsHTTPError): try: raise_http_error(make_http_error(HTTPNotFound)) except Exception as ex: utils.pass_http_error(ex, [HTTPConflict, HTTPInternalServerError])
def test_pass_http_error_raises_requests_error_with_single_pyramid_error(): with pytest.raises(RequestsHTTPError): try: raise_http_error(make_http_error(HTTPNotFound)) except Exception as ex: utils.pass_http_error(ex, HTTPConflict)
def test_pass_http_error_raises_pyramid_error_with_multi_pyramid_error(): with pytest.raises(HTTPNotFound): try: raise_http_error(HTTPNotFound) except Exception as ex: pass_http_error(ex, [HTTPConflict, HTTPInternalServerError])
def test_pass_http_error_raises_pyramid_error_with_single_pyramid_error(): with pytest.raises(HTTPNotFound): try: raise_http_error(HTTPNotFound) except Exception as ex: pass_http_error(ex, HTTPConflict)
def execute(self, workflow_inputs, out_dir, expected_outputs): # TODO: test visible = self.is_visible() if not visible: # includes private visibility and non-existing cases if visible is None: LOGGER.info( "Process [%s] access is unauthorized on [%s] - deploying as admin.", self.process, self.url) elif visible is False: LOGGER.info( "Process [%s] is not deployed on [%s] - deploying.", self.process, self.url) # TODO: Maybe always redeploy? What about cases of outdated deployed process? try: self.deploy() except Exception as exc: # FIXME: support for Spacebel, avoid conflict error incorrectly handled, remove 500 when fixed pass_http_error(exc, [HTTPConflict, HTTPInternalServerError]) LOGGER.info("Process [%s] enforced to public visibility.", self.process) try: self.set_visibility(visibility=VISIBILITY_PUBLIC) # TODO: support for Spacebel, remove when visibility route properly implemented on ADES except Exception as exc: pass_http_error(exc, HTTPNotFound) self.update_status("Preparing execute request for remote ADES.", REMOTE_JOB_PROGRESS_REQ_PREP, status.STATUS_RUNNING) LOGGER.debug("Execute process WPS request for [%s]", self.process) execute_body_inputs = [] execute_req_id = "id" execute_req_input_val_href = "href" execute_req_input_val_data = "data" for workflow_input_key, workflow_input_value in workflow_inputs.items( ): if isinstance(workflow_input_value, list): for workflow_input_value_item in workflow_input_value: if isinstance( workflow_input_value_item, dict) and "location" in workflow_input_value_item: execute_body_inputs.append({ execute_req_id: workflow_input_key, execute_req_input_val_href: workflow_input_value_item["location"] }) else: execute_body_inputs.append({ execute_req_id: workflow_input_key, execute_req_input_val_data: workflow_input_value_item }) else: if isinstance(workflow_input_value, dict) and "location" in workflow_input_value: execute_body_inputs.append({ execute_req_id: workflow_input_key, execute_req_input_val_href: workflow_input_value["location"] }) else: execute_body_inputs.append({ execute_req_id: workflow_input_key, execute_req_input_val_data: workflow_input_value }) for exec_input in execute_body_inputs: if execute_req_input_val_href in exec_input and isinstance( exec_input[execute_req_input_val_href], str): if exec_input[execute_req_input_val_href].startswith( "{0}://".format(OPENSEARCH_LOCAL_FILE_SCHEME)): exec_input[execute_req_input_val_href] = "file{0}".format( exec_input[execute_req_input_val_href] [len(OPENSEARCH_LOCAL_FILE_SCHEME):]) elif exec_input[execute_req_input_val_href].startswith( "file://"): exec_input[execute_req_input_val_href] = self.host_file( exec_input[execute_req_input_val_href]) LOGGER.debug("Hosting intermediate input [%s] : [%s]", exec_input[execute_req_id], exec_input[execute_req_input_val_href]) execute_body_outputs = [{ execute_req_id: output, "transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE } for output in expected_outputs] self.update_status("Executing job on remote ADES.", REMOTE_JOB_PROGRESS_EXECUTION, status.STATUS_RUNNING) execute_body = dict(mode=EXECUTE_MODE_ASYNC, response=EXECUTE_RESPONSE_DOCUMENT, inputs=execute_body_inputs, outputs=execute_body_outputs) request_url = self.url + process_jobs_uri.format( process_id=self.process) response = self.make_request(method="POST", url=request_url, json=execute_body, retry=True) if response.status_code != 201: raise Exception( "Was expecting a 201 status code from the execute request : {0}" .format(request_url)) job_status_uri = response.headers["Location"] job_status = self.get_job_status(job_status_uri) job_status_value = status.map_status(job_status["status"]) self.update_status( "Monitoring job on remote ADES : {0}".format(job_status_uri), REMOTE_JOB_PROGRESS_MONITORING, status.STATUS_RUNNING) while job_status_value not in status.JOB_STATUS_CATEGORIES[ status.STATUS_CATEGORY_FINISHED]: sleep(5) job_status = self.get_job_status(job_status_uri) job_status_value = status.map_status(job_status["status"]) LOGGER.debug( get_log_monitor_msg(job_status["jobID"], job_status_value, job_status.get("percentCompleted", 0), get_any_message(job_status), job_status.get("statusLocation"))) self.update_status( get_job_log_msg(status=job_status_value, message=get_any_message(job_status), progress=job_status.get("percentCompleted", 0), duration=job_status.get( "duration", None)), # get if available map_progress(job_status.get("percentCompleted", 0), REMOTE_JOB_PROGRESS_MONITORING, REMOTE_JOB_PROGRESS_FETCH_OUT), status.STATUS_RUNNING) if job_status_value != status.STATUS_SUCCEEDED: LOGGER.debug( get_log_monitor_msg(job_status["jobID"], job_status_value, job_status.get("percentCompleted", 0), get_any_message(job_status), job_status.get("statusLocation"))) raise Exception(job_status) self.update_status("Fetching job outputs from remote ADES.", REMOTE_JOB_PROGRESS_FETCH_OUT, status.STATUS_RUNNING) results = self.get_job_results(job_status["jobID"]) for result in results: if get_any_id(result) in expected_outputs: # This is where cwl expect the output file to be written # TODO We will probably need to handle multiple output value... dst_fn = "/".join([ out_dir.rstrip("/"), expected_outputs[get_any_id(result)] ]) # TODO Should we handle other type than File reference? resp = request_extra("get", get_any_value(result), allow_redirects=True, settings=self.settings) LOGGER.debug( "Fetching result output from [%s] to cwl output destination: [%s]", get_any_value(result), dst_fn) with open(dst_fn, mode="wb") as dst_fh: dst_fh.write(resp.content) self.update_status("Execution on remote ADES completed.", REMOTE_JOB_PROGRESS_COMPLETED, status.STATUS_SUCCEEDED)