def prepare(self):
visible = self.is_visible()
if not visible: # includes private visibility and non-existing cases
if visible is None:
LOGGER.info(
"Process [%s] access is unauthorized on [%s] - deploying as admin.",
self.process, self.url)
elif visible is False:
LOGGER.info(
"Process [%s] is not deployed on [%s] - deploying.",
self.process, self.url)
# TODO: Maybe always redeploy? What about cases of outdated deployed process?
try:
self.deploy()
except Exception as exc:
pass_http_error(exc, [HTTPConflict])
if visible:
LOGGER.info(
"Process [%s] already deployed and visible on [%s] - executing.",
self.process, self.url)
else:
LOGGER.info("Process [%s] enforcing to public visibility.",
self.process)
try:
self.set_visibility(visibility=VISIBILITY_PUBLIC)
except Exception as exc:
pass_http_error(exc, HTTPNotFound)
LOGGER.warning(
"Process [%s] failed setting public visibility. "
"Assuming feature is not supported by ADES and process is already public.",
self.process)
def test_pass_http_error_doesnt_raise_multi_pyramid_error():
http_errors = [HTTPNotFound, HTTPInternalServerError]
for err in http_errors:
# test try/except
try:
# normal usage try/except
try:
raise_http_error(err)
except Exception as ex:
utils.pass_http_error(ex, http_errors)
except PyramidHTTPError:
pytest.fail("PyramidHTTPError should be ignored but was raised.")
def test_pass_http_error_doesnt_raise_requests_error():
http_errors = [HTTPNotFound, HTTPInternalServerError]
for err in http_errors:
req_err = make_http_error(err)
# test try/except
try:
# normal usage try/except
try:
raise_http_error(req_err)
except Exception as ex:
utils.pass_http_error(ex, err)
except RequestsHTTPError:
pytest.fail("RequestsHTTPError should be ignored but was raised.")
def test_pass_http_error_raises_other_error_with_multi_pyramid_error():
with pytest.raises(ValueError):
try:
raise ValueError("Test Error")
except Exception as ex:
utils.pass_http_error(ex, [HTTPConflict, HTTPInternalServerError])
def test_pass_http_error_raises_other_error_with_single_pyramid_error():
with pytest.raises(ValueError):
try:
raise ValueError("Test Error")
except Exception as ex:
utils.pass_http_error(ex, HTTPConflict)
def test_pass_http_error_raises_requests_error_with_multi_pyramid_error():
with pytest.raises(RequestsHTTPError):
try:
raise_http_error(make_http_error(HTTPNotFound))
except Exception as ex:
utils.pass_http_error(ex, [HTTPConflict, HTTPInternalServerError])
def test_pass_http_error_raises_requests_error_with_single_pyramid_error():
with pytest.raises(RequestsHTTPError):
try:
raise_http_error(make_http_error(HTTPNotFound))
except Exception as ex:
utils.pass_http_error(ex, HTTPConflict)
def test_pass_http_error_raises_pyramid_error_with_multi_pyramid_error():
with pytest.raises(HTTPNotFound):
try:
raise_http_error(HTTPNotFound)
except Exception as ex:
pass_http_error(ex, [HTTPConflict, HTTPInternalServerError])
def test_pass_http_error_raises_pyramid_error_with_single_pyramid_error():
with pytest.raises(HTTPNotFound):
try:
raise_http_error(HTTPNotFound)
except Exception as ex:
pass_http_error(ex, HTTPConflict)
def execute(self, workflow_inputs, out_dir, expected_outputs):
# TODO: test
visible = self.is_visible()
if not visible: # includes private visibility and non-existing cases
if visible is None:
LOGGER.info(
"Process [%s] access is unauthorized on [%s] - deploying as admin.",
self.process, self.url)
elif visible is False:
LOGGER.info(
"Process [%s] is not deployed on [%s] - deploying.",
self.process, self.url)
# TODO: Maybe always redeploy? What about cases of outdated deployed process?
try:
self.deploy()
except Exception as exc:
# FIXME: support for Spacebel, avoid conflict error incorrectly handled, remove 500 when fixed
pass_http_error(exc, [HTTPConflict, HTTPInternalServerError])
LOGGER.info("Process [%s] enforced to public visibility.",
self.process)
try:
self.set_visibility(visibility=VISIBILITY_PUBLIC)
# TODO: support for Spacebel, remove when visibility route properly implemented on ADES
except Exception as exc:
pass_http_error(exc, HTTPNotFound)
self.update_status("Preparing execute request for remote ADES.",
REMOTE_JOB_PROGRESS_REQ_PREP, status.STATUS_RUNNING)
LOGGER.debug("Execute process WPS request for [%s]", self.process)
execute_body_inputs = []
execute_req_id = "id"
execute_req_input_val_href = "href"
execute_req_input_val_data = "data"
for workflow_input_key, workflow_input_value in workflow_inputs.items(
):
if isinstance(workflow_input_value, list):
for workflow_input_value_item in workflow_input_value:
if isinstance(
workflow_input_value_item,
dict) and "location" in workflow_input_value_item:
execute_body_inputs.append({
execute_req_id:
workflow_input_key,
execute_req_input_val_href:
workflow_input_value_item["location"]
})
else:
execute_body_inputs.append({
execute_req_id:
workflow_input_key,
execute_req_input_val_data:
workflow_input_value_item
})
else:
if isinstance(workflow_input_value,
dict) and "location" in workflow_input_value:
execute_body_inputs.append({
execute_req_id:
workflow_input_key,
execute_req_input_val_href:
workflow_input_value["location"]
})
else:
execute_body_inputs.append({
execute_req_id:
workflow_input_key,
execute_req_input_val_data:
workflow_input_value
})
for exec_input in execute_body_inputs:
if execute_req_input_val_href in exec_input and isinstance(
exec_input[execute_req_input_val_href], str):
if exec_input[execute_req_input_val_href].startswith(
"{0}://".format(OPENSEARCH_LOCAL_FILE_SCHEME)):
exec_input[execute_req_input_val_href] = "file{0}".format(
exec_input[execute_req_input_val_href]
[len(OPENSEARCH_LOCAL_FILE_SCHEME):])
elif exec_input[execute_req_input_val_href].startswith(
"file://"):
exec_input[execute_req_input_val_href] = self.host_file(
exec_input[execute_req_input_val_href])
LOGGER.debug("Hosting intermediate input [%s] : [%s]",
exec_input[execute_req_id],
exec_input[execute_req_input_val_href])
execute_body_outputs = [{
execute_req_id:
output,
"transmissionMode":
EXECUTE_TRANSMISSION_MODE_REFERENCE
} for output in expected_outputs]
self.update_status("Executing job on remote ADES.",
REMOTE_JOB_PROGRESS_EXECUTION,
status.STATUS_RUNNING)
execute_body = dict(mode=EXECUTE_MODE_ASYNC,
response=EXECUTE_RESPONSE_DOCUMENT,
inputs=execute_body_inputs,
outputs=execute_body_outputs)
request_url = self.url + process_jobs_uri.format(
process_id=self.process)
response = self.make_request(method="POST",
url=request_url,
json=execute_body,
retry=True)
if response.status_code != 201:
raise Exception(
"Was expecting a 201 status code from the execute request : {0}"
.format(request_url))
job_status_uri = response.headers["Location"]
job_status = self.get_job_status(job_status_uri)
job_status_value = status.map_status(job_status["status"])
self.update_status(
"Monitoring job on remote ADES : {0}".format(job_status_uri),
REMOTE_JOB_PROGRESS_MONITORING, status.STATUS_RUNNING)
while job_status_value not in status.JOB_STATUS_CATEGORIES[
status.STATUS_CATEGORY_FINISHED]:
sleep(5)
job_status = self.get_job_status(job_status_uri)
job_status_value = status.map_status(job_status["status"])
LOGGER.debug(
get_log_monitor_msg(job_status["jobID"], job_status_value,
job_status.get("percentCompleted", 0),
get_any_message(job_status),
job_status.get("statusLocation")))
self.update_status(
get_job_log_msg(status=job_status_value,
message=get_any_message(job_status),
progress=job_status.get("percentCompleted", 0),
duration=job_status.get(
"duration", None)), # get if available
map_progress(job_status.get("percentCompleted",
0), REMOTE_JOB_PROGRESS_MONITORING,
REMOTE_JOB_PROGRESS_FETCH_OUT),
status.STATUS_RUNNING)
if job_status_value != status.STATUS_SUCCEEDED:
LOGGER.debug(
get_log_monitor_msg(job_status["jobID"], job_status_value,
job_status.get("percentCompleted", 0),
get_any_message(job_status),
job_status.get("statusLocation")))
raise Exception(job_status)
self.update_status("Fetching job outputs from remote ADES.",
REMOTE_JOB_PROGRESS_FETCH_OUT,
status.STATUS_RUNNING)
results = self.get_job_results(job_status["jobID"])
for result in results:
if get_any_id(result) in expected_outputs:
# This is where cwl expect the output file to be written
# TODO We will probably need to handle multiple output value...
dst_fn = "/".join([
out_dir.rstrip("/"), expected_outputs[get_any_id(result)]
])
# TODO Should we handle other type than File reference?
resp = request_extra("get",
get_any_value(result),
allow_redirects=True,
settings=self.settings)
LOGGER.debug(
"Fetching result output from [%s] to cwl output destination: [%s]",
get_any_value(result), dst_fn)
with open(dst_fn, mode="wb") as dst_fh:
dst_fh.write(resp.content)
self.update_status("Execution on remote ADES completed.",
REMOTE_JOB_PROGRESS_COMPLETED,
status.STATUS_SUCCEEDED)
