def fetch_wps_process(job, wps_url, headers, settings):
"""
Retrieves the WPS process description from the local or remote WPS reference URL.
"""
try:
wps = get_wps_client(wps_url, settings, headers=headers, language=job.accept_language)
raise_on_xml_exception(wps._capabilities) # noqa
except Exception as ex:
job.save_log(errors=ex, message="Failed WPS client creation for process [{!s}]".format(job.process))
raise OWSNoApplicableCode("Failed to retrieve WPS capabilities. Error: [{}].".format(str(ex)))
try:
wps_process = wps.describeprocess(job.process)
except Exception as ex:
raise OWSNoApplicableCode("Failed to retrieve WPS process description. Error: [{}].".format(str(ex)))
return wps_process
def get_pywps_service(environ=None, is_worker=False):
"""
Generates the PyWPS Service that provides *older* WPS-1/2 XML endpoint.
"""
environ = environ or {}
try:
# get config file
settings = get_settings(app)
pywps_cfg = environ.get("PYWPS_CFG") or settings.get(
"PYWPS_CFG") or os.getenv("PYWPS_CFG")
if not isinstance(
pywps_cfg,
ConfigParser) or not settings.get("weaver.wps_configured"):
load_pywps_config(app, config=pywps_cfg)
# call pywps application with processes filtered according to the adapter's definition
process_store = get_db(app).get_store(StoreProcesses)
processes_wps = [
process.wps() for process in process_store.list_processes(
visibility=VISIBILITY_PUBLIC)
]
service = WorkerService(processes_wps,
is_worker=is_worker,
settings=settings)
except Exception as ex:
LOGGER.exception(
"Error occurred during PyWPS Service and/or Processes setup.")
raise OWSNoApplicableCode(
"Failed setup of PyWPS Service and/or Processes. Error [{!r}]".
format(ex))
return service
def execute(self, identifier, wps_request, uuid):
# type: (str, WPSRequest, str) -> Union[WPSResponse, HTTPValid]
"""
Submit WPS request to corresponding WPS-REST endpoint and convert back for requested ``Accept`` content-type.
Overrides the original execute operation, that instead will get handled by :meth:`execute_job` following
callback from Celery Worker that handles process job creation and monitoring.
If ``Accept`` is JSON, the result is directly returned from :meth:`_submit_job`.
If ``Accept`` is XML or undefined, :class:`WorkerExecuteResponse` converts the received JSON with XML template.
"""
result = self._submit_job(wps_request)
if not isinstance(result, dict): # JSON
return result # direct WPS response
# otherwise, recreate the equivalent content with expected XML template format
job_id = result["jobID"]
job_url = result["location"]
wps_process = self.processes.get(wps_request.identifier)
# when called by the WSGI app, 'WorkerExecuteResponse.__call__' on will generate the XML from 'doc' property,
# which itself is generated by template substitution of data from above 'json' property
try:
return WorkerExecuteResponse(wps_request,
job_id,
wps_process,
job_url,
settings=self.settings)
except Exception as ex: # noqa
LOGGER.exception(
"Error building XML response by PyWPS Service during WPS Execute result from worker."
)
message = "Failed building XML response from WPS Execute result. Error [{!r}]".format(
ex)
raise OWSNoApplicableCode(message, locator=job_id)
def get_pywps_service(environ=None, is_worker=False):
# type: (SettingsType, bool) -> WorkerService
"""
Generates the PyWPS Service that provides WPS-1/2 XML endpoint.
"""
environ = environ or {}
try:
# get config file
registry = get_registry()
settings = get_settings(registry)
pywps_cfg = environ.get("PYWPS_CFG") or settings.get(
"PYWPS_CFG") or os.getenv("PYWPS_CFG")
if not isinstance(
pywps_cfg,
ConfigParser) or not settings.get("weaver.wps_configured"):
load_pywps_config(settings, config=pywps_cfg)
# call pywps application with processes filtered according to the adapter's definition
process_store = get_db(registry).get_store(
StoreProcesses) # type: StoreProcesses
processes_wps = [
process.wps() for process in process_store.list_processes(
visibility=Visibility.PUBLIC)
]
service = WorkerService(processes_wps,
is_worker=is_worker,
settings=settings)
except Exception as ex:
LOGGER.exception(
"Error occurred during PyWPS Service and/or Processes setup.")
raise OWSNoApplicableCode(
f"Failed setup of PyWPS Service and/or Processes. Error [{ex!r}]")
return service
def prepare(self):
LOGGER.debug("Execute WPS-1 provider: [%s]", self.provider)
LOGGER.debug("Execute WPS-1 process: [%s]", self.process)
try:
self.wps_provider = get_wps_client(self.provider,
headers=self.cookies)
raise_on_xml_exception(
self.wps_provider._capabilities) # noqa: W0212
except Exception as ex:
raise OWSNoApplicableCode(
"Failed to retrieve WPS capabilities. Error: [{}].".format(
str(ex)))
try:
self.wps_process = self.wps_provider.describeprocess(self.process)
except Exception as ex:
raise OWSNoApplicableCode(
"Failed to retrieve WPS process description. Error: [{}].".
format(str(ex)))
def prepare(self):
LOGGER.debug("Execute WPS-1 provider: [%s]", self.provider)
LOGGER.debug("Execute WPS-1 process: [%s]", self.process)
try:
headers = {}
headers.update(self.get_auth_cookies())
headers.update(self.get_auth_headers())
self.wps_provider = get_wps_client(self.provider, headers=headers)
raise_on_xml_exception(
self.wps_provider._capabilities) # noqa: W0212
except Exception as ex:
raise OWSNoApplicableCode(
f"Failed to retrieve WPS capabilities. Error: [{ex!s}].")
try:
self.wps_process = self.wps_provider.describeprocess(self.process)
except Exception as ex:
raise OWSNoApplicableCode(
f"Failed to retrieve WPS process description. Error: [{ex!s}]."
)
def execute(self, identifier, wps_request, uuid):
# type: (str, Union[WPSRequest, WorkerRequest], str) -> Union[WPSResponse, HTTPValid]
"""
Handles the ``Execute`` KVP/XML request submitted on the WPS endpoint.
Submit WPS request to corresponding WPS-REST endpoint and convert back for requested ``Accept`` content-type.
Overrides the original execute operation, that will instead be handled by :meth:`execute_job` following
callback from Celery Worker, which handles process job creation and monitoring.
If ``Accept`` is JSON, the result is directly returned from :meth:`_submit_job`.
If ``Accept`` is XML or undefined, :class:`WorkerExecuteResponse` converts the received JSON with XML template.
"""
result = self._submit_job(wps_request)
if not isinstance(result, dict):
return result # pre-built HTTP response with JSON contents when requested
# otherwise, recreate the equivalent content with expected XML template format
job_id = result["jobID"]
wps_process = self.processes.get(wps_request.identifier)
# because we are building the XML response (and JSON not explicitly requested)
# caller is probably a WPS-1 client also expecting a status XML file
# remap the status location accordingly from the current REST endpoint
job_url = result["location"]
if urlparse(job_url).path.endswith(f"/jobs/{job_id}"):
# file status does not exist yet since client calling this method is waiting for it
# pywps will generate it once the WorkerExecuteResponse is returned
status_path = get_wps_local_status_location(job_url,
self.settings,
must_exist=False)
wps_dir = get_wps_output_dir(self.settings)
wps_url = get_wps_output_url(self.settings)
job_url = status_path.replace(wps_dir, wps_url, 1)
# when called by the WSGI app, 'WorkerExecuteResponse.__call__' will generate the XML from 'doc' property,
# which itself is generated by template substitution of data from above 'json' property
try:
return WorkerExecuteResponse(wps_request,
job_id,
wps_process,
job_url,
settings=self.settings)
except Exception as ex: # noqa
LOGGER.exception(
"Error building XML response by PyWPS Service during WPS Execute result from worker."
)
message = f"Failed building XML response from WPS Execute result. Error [{ex!r}]"
raise OWSNoApplicableCode(message, locator=job_id)
def fetch_wps_process(job, wps_url, headers, settings):
# type: (Job, str, HeadersType, SettingsType) -> ProcessOWS
"""
Retrieves the WPS process description from the local or remote WPS reference URL.
"""
try:
wps = get_wps_client(wps_url,
settings,
headers=headers,
language=job.accept_language)
raise_on_xml_exception(wps._capabilities) # noqa
except Exception as ex:
job.save_log(
errors=ex,
message=f"Failed WPS client creation for process [{job.process!s}]"
)
raise OWSNoApplicableCode(
f"Failed to retrieve WPS capabilities. Error: [{ex!s}].")
try:
wps_process = wps.describeprocess(job.process)
except Exception as ex: # pragma: no cover
raise OWSNoApplicableCode(
f"Failed to retrieve WPS process description. Error: [{ex!s}].")
return wps_process
def wrapped(*_, **__):
try:
return function(*_, **__)
except (WeaverException, OWSException, HTTPException) as exc:
if isinstance(exc, WeaverException) and not isinstance(exc, OWSException):
return OWSNoApplicableCode(str(exc), locator="service", content_type=CONTENT_TYPE_TEXT_XML)
if isinstance(exc, HTTPException):
# override default pre-generated plain text content-type such that
# resulting exception generates the response content with requested accept or XML by default
exc.headers.setdefault("Accept", CONTENT_TYPE_TEXT_XML)
exc.headers.pop("Content-Type", None)
if isinstance(exc, HTTPNotFound):
exc = OWSNotFound(str(exc), locator="service", status=exc)
elif isinstance(exc, HTTPForbidden):
exc = OWSAccessForbidden(str(exc), locator="service", status=exc)
else:
exc = OWSException(str(exc), locator="service", status=exc)
return exc # return to avoid raising, raise would be caught by parent pywps call wrapping 'function'
def execute(self, workflow_inputs, out_dir, expected_outputs):
self.update_status("Preparing execute request for remote WPS1 provider.",
REMOTE_JOB_PROGRESS_REQ_PREP, status.STATUS_RUNNING)
LOGGER.debug("Execute process WPS request for %s", self.process)
try:
try:
wps = WebProcessingService(url=self.provider, headers=self.cookies, verify=self.verify)
raise_on_xml_exception(wps._capabilities) # noqa: W0212
except Exception as ex:
raise OWSNoApplicableCode("Failed to retrieve WPS capabilities. Error: [{}].".format(str(ex)))
try:
process = wps.describeprocess(self.process)
except Exception as ex:
raise OWSNoApplicableCode("Failed to retrieve WPS process description. Error: [{}].".format(str(ex)))
# prepare inputs
complex_inputs = []
for process_input in process.dataInputs:
if WPS_COMPLEX_DATA in process_input.dataType:
complex_inputs.append(process_input.identifier)
# remove any 'null' input, should employ the 'default' of the remote WPS process
inputs_provided_keys = filter(lambda i: workflow_inputs[i] != "null", workflow_inputs)
wps_inputs = []
for input_key in inputs_provided_keys:
input_val = workflow_inputs[input_key]
# in case of array inputs, must repeat (id,value)
# in case of complex input (File), obtain location, otherwise get data value
if not isinstance(input_val, list):
input_val = [input_val]
input_values = []
for val in input_val:
if isinstance(val, dict):
val = val["location"]
# owslib only accepts strings, not numbers directly
if isinstance(val, (int, float)):
val = str(val)
if val.startswith("file://"):
# we need to host file starting with file:// scheme
val = self.host_file(val)
input_values.append(val)
# need to use ComplexDataInput structure for complex input
# TODO: BoundingBox not supported
for input_value in input_values:
if input_key in complex_inputs:
input_value = ComplexDataInput(input_value)
wps_inputs.append((input_key, input_value))
# prepare outputs
outputs = [(o.identifier, o.dataType == WPS_COMPLEX_DATA) for o in process.processOutputs
if o.identifier in expected_outputs]
self.update_status("Executing job on remote WPS1 provider.",
REMOTE_JOB_PROGRESS_EXECUTION, status.STATUS_RUNNING)
mode = EXECUTE_MODE_ASYNC
execution = wps.execute(self.process, inputs=wps_inputs, output=outputs, mode=mode, lineage=True)
if not execution.process and execution.errors:
raise execution.errors[0]
self.update_status("Monitoring job on remote WPS1 provider : [{0}]".format(self.provider),
REMOTE_JOB_PROGRESS_MONITORING, status.STATUS_RUNNING)
max_retries = 5
num_retries = 0
run_step = 0
job_id = "<undefined>"
while execution.isNotComplete() or run_step == 0:
if num_retries >= max_retries:
raise Exception("Could not read status document after {} retries. Giving up.".format(max_retries))
try:
execution = check_wps_status(location=execution.statusLocation, verify=self.verify,
sleep_secs=wait_secs(run_step))
job_id = execution.statusLocation.replace(".xml", "").split("/")[-1]
LOGGER.debug(get_log_monitor_msg(job_id, status.map_status(execution.getStatus()),
execution.percentCompleted, execution.statusMessage,
execution.statusLocation))
self.update_status(get_job_log_msg(status=status.map_status(execution.getStatus()),
message=execution.statusMessage,
progress=execution.percentCompleted,
duration=None), # get if available
map_progress(execution.percentCompleted,
REMOTE_JOB_PROGRESS_MONITORING, REMOTE_JOB_PROGRESS_FETCH_OUT),
status.STATUS_RUNNING)
except Exception as exc:
num_retries += 1
LOGGER.debug("Exception raised: %r", exc)
sleep(1)
else:
num_retries = 0
run_step += 1
if not execution.isSucceded():
exec_msg = execution.statusMessage or "Job failed."
LOGGER.debug(get_log_monitor_msg(job_id, status.map_status(execution.getStatus()),
execution.percentCompleted, exec_msg, execution.statusLocation))
raise Exception(execution.statusMessage or "Job failed.")
self.update_status("Fetching job outputs from remote WPS1 provider.",
REMOTE_JOB_PROGRESS_FETCH_OUT, status.STATUS_RUNNING)
results = [ows2json_output(output, process) for output in execution.processOutputs]
for result in results:
result_id = get_any_id(result)
result_val = get_any_value(result)
if result_id in expected_outputs:
# This is where cwl expect the output file to be written
# TODO We will probably need to handle multiple output value...
dst_fn = "/".join([out_dir.rstrip("/"), expected_outputs[result_id]])
# TODO Should we handle other type than File reference?
resp = request_extra("get", result_val, allow_redirects=True, settings=self.settings)
LOGGER.debug("Fetching result output from [%s] to cwl output destination: [%s]", result_val, dst_fn)
with open(dst_fn, mode="wb") as dst_fh:
dst_fh.write(resp.content)
except Exception as exc:
exception_class = "{}.{}".format(type(exc).__module__, type(exc).__name__)
errors = "{0}: {1!s}".format(exception_class, exc)
LOGGER.exception(exc)
raise Exception(errors)
self.update_status("Execution on remote WPS1 provider completed.",
REMOTE_JOB_PROGRESS_COMPLETED, status.STATUS_SUCCEEDED)
def execute_process(self, job_id, url, headers=None):
from weaver.wps.service import get_pywps_service
LOGGER.debug("Job execute process called.")
settings = get_settings(app)
task_logger = get_task_logger(__name__)
load_pywps_config(settings)
task_logger.debug("Job task setup.")
# reset the connection because we are in a forked celery process
db = get_db(app, reset_connection=True)
store = db.get_store(StoreJobs)
job = store.fetch_by_id(job_id)
job.task_id = self.request.id
job.progress = JOB_PROGRESS_SETUP
job.save_log(logger=task_logger, message="Job task setup completed.")
job = store.update_job(job)
try:
try:
job.progress = JOB_PROGRESS_DESCRIBE
job.save_log(
logger=task_logger,
message="Execute WPS request for process [{!s}]".format(
job.process))
ssl_verify = get_ssl_verify_option("get", url, settings=settings)
wps = WebProcessingService(url=url,
headers=get_cookie_headers(headers),
verify=ssl_verify)
set_wps_language(wps, accept_language=job.accept_language)
raise_on_xml_exception(wps._capabilities) # noqa
except Exception as ex:
raise OWSNoApplicableCode(
"Failed to retrieve WPS capabilities. Error: [{}].".format(
str(ex)))
try:
process = wps.describeprocess(job.process)
except Exception as ex:
raise OWSNoApplicableCode(
"Failed to retrieve WPS process description. Error: [{}].".
format(str(ex)))
# prepare inputs
job.progress = JOB_PROGRESS_GET_INPUTS
job.save_log(logger=task_logger,
message="Fetching job input definitions.")
complex_inputs = []
for process_input in process.dataInputs:
if WPS_COMPLEX_DATA in process_input.dataType:
complex_inputs.append(process_input.identifier)
try:
wps_inputs = list()
for process_input in job.inputs:
input_id = get_any_id(process_input)
process_value = get_any_value(process_input)
# in case of array inputs, must repeat (id,value)
input_values = process_value if isinstance(
process_value, list) else [process_value]
# we need to support file:// scheme but PyWPS doesn't like them so remove the scheme file://
input_values = [
val[7:] if str(val).startswith("file://") else val
for val in input_values
]
# need to use ComplexDataInput structure for complex input
# need to use literal String for anything else than complex
# TODO: BoundingBox not supported
wps_inputs.extend([
(input_id, ComplexDataInput(input_value)
if input_id in complex_inputs else str(input_value))
for input_value in input_values
])
except KeyError:
wps_inputs = []
# prepare outputs
job.progress = JOB_PROGRESS_GET_OUTPUTS
job.save_log(logger=task_logger,
message="Fetching job output definitions.")
wps_outputs = [(o.identifier, o.dataType == WPS_COMPLEX_DATA)
for o in process.processOutputs]
mode = EXECUTE_MODE_ASYNC if job.execute_async else EXECUTE_MODE_SYNC
job.progress = JOB_PROGRESS_EXECUTE_REQUEST
job.save_log(logger=task_logger,
message="Starting job process execution.")
job.save_log(
logger=task_logger,
message=
"Following updates could take a while until the Application Package answers..."
)
wps_worker = get_pywps_service(environ=settings, is_worker=True)
execution = wps_worker.execute_job(job.process,
wps_inputs=wps_inputs,
wps_outputs=wps_outputs,
mode=mode,
job_uuid=job.id)
if not execution.process and execution.errors:
raise execution.errors[0]
# adjust status location
wps_status_path = get_wps_local_status_location(
execution.statusLocation, settings)
job.progress = JOB_PROGRESS_EXECUTE_STATUS_LOCATION
LOGGER.debug("WPS status location that will be queried: [%s]",
wps_status_path)
if not wps_status_path.startswith("http") and not os.path.isfile(
wps_status_path):
LOGGER.warning(
"WPS status location not resolved to local path: [%s]",
wps_status_path)
job.save_log(logger=task_logger,
level=logging.DEBUG,
message="Updated job status location: [{}].".format(
wps_status_path))
job.status = map_status(STATUS_STARTED)
job.status_message = execution.statusMessage or "{} initiation done.".format(
str(job))
job.status_location = wps_status_path
job.request = execution.request
job.response = execution.response
job.progress = JOB_PROGRESS_EXECUTE_MONITOR_START
job.save_log(logger=task_logger,
message="Starting monitoring of job execution.")
job = store.update_job(job)
max_retries = 5
num_retries = 0
run_step = 0
while execution.isNotComplete() or run_step == 0:
if num_retries >= max_retries:
raise Exception(
"Could not read status document after {} retries. Giving up."
.format(max_retries))
try:
# NOTE:
# Don't actually log anything here until process is completed (success or fail) so that underlying
# WPS execution logs can be inserted within the current job log and appear continuously.
# Only update internal job fields in case they get referenced elsewhere.
job.progress = JOB_PROGRESS_EXECUTE_MONITOR_LOOP
execution = check_wps_status(location=wps_status_path,
settings=settings,
sleep_secs=wait_secs(run_step))
job_msg = (execution.statusMessage or "").strip()
job.response = execution.response
job.status = map_status(execution.getStatus())
job.status_message = "Job execution monitoring (progress: {}%, status: {})."\
.format(execution.percentCompleted, job_msg or "n/a")
# job.save_log(logger=task_logger)
# job = store.update_job(job)
if execution.isComplete():
job.mark_finished()
job.progress = JOB_PROGRESS_EXECUTE_MONITOR_END
msg_progress = " (status: {})".format(
job_msg) if job_msg else ""
if execution.isSucceded():
job.status = map_status(STATUS_SUCCEEDED)
job.status_message = "Job succeeded{}.".format(
msg_progress)
wps_package.retrieve_package_job_log(execution, job)
job.save_log(logger=task_logger)
job_results = [
ows2json_output(output, process, settings)
for output in execution.processOutputs
]
job.results = make_results_relative(
job_results, settings)
else:
task_logger.debug("Job failed.")
job.status_message = "Job failed{}.".format(
msg_progress)
wps_package.retrieve_package_job_log(execution, job)
job.save_log(errors=execution.errors,
logger=task_logger)
task_logger.debug(
"Mapping Job references with generated WPS locations.")
map_locations(job, settings)
except Exception as exc:
num_retries += 1
task_logger.debug("Exception raised: %s", repr(exc))
job.status_message = "Could not read status XML document for {!s}. Trying again...".format(
job)
job.save_log(errors=execution.errors, logger=task_logger)
sleep(1)
else:
# job.status_message = "Update {}...".format(str(job))
# job.save_log(logger=task_logger)
num_retries = 0
run_step += 1
finally:
job = store.update_job(job)
except Exception as exc:
LOGGER.exception("Failed running [%s]", job)
job.status = map_status(STATUS_FAILED)
job.status_message = "Failed to run {!s}.".format(job)
job.progress = JOB_PROGRESS_EXECUTE_MONITOR_ERROR
exception_class = "{}.{}".format(
type(exc).__module__,
type(exc).__name__)
errors = "{0}: {1!s}".format(exception_class, exc)
job.save_log(errors=errors, logger=task_logger)
finally:
job.progress = JOB_PROGRESS_EXECUTE_MONITOR_END
job.status_message = "Job {}.".format(job.status)
job.save_log(logger=task_logger)
# Send email if requested
if job.notification_email is not None:
job.progress = JOB_PROGRESS_NOTIFY
try:
notify_job_complete(job, job.notification_email, settings)
message = "Notification email sent successfully."
job.save_log(logger=task_logger, message=message)
except Exception as exc:
exception_class = "{}.{}".format(
type(exc).__module__,
type(exc).__name__)
exception = "{0}: {1!s}".format(exception_class, exc)
message = "Couldn't send notification email ({})".format(
exception)
job.save_log(errors=message,
logger=task_logger,
message=message)
job.progress = JOB_PROGRESS_DONE
job.save_log(logger=task_logger, message="Job task complete.")
job = store.update_job(job)
return job.status