def test_set_wps_language():
wps = mock.Mock()
languages = mock.Mock()
wps.languages = languages
languages.default = ACCEPT_LANGUAGE_EN_US
languages.supported = [ACCEPT_LANGUAGE_EN_US, ACCEPT_LANGUAGE_FR_CA]
set_wps_language(wps, "ru, fr;q=0.5")
assert wps.language == ACCEPT_LANGUAGE_FR_CA
def test_set_wps_language():
wps = mock.Mock()
languages = mock.Mock()
wps.languages = languages
languages.default = AcceptLanguage.EN_US
languages.supported = [AcceptLanguage.EN_US, AcceptLanguage.FR_CA]
set_wps_language(wps, "ru, fr;q=0.5")
assert wps.language == AcceptLanguage.FR_CA
def test_set_wps_language():
wps = mock.Mock()
languages = mock.Mock()
wps.languages = languages
languages.default = "en-US"
languages.supported = ["en-US", "fr-CA"]
set_wps_language(wps, "ru, fr;q=0.5")
assert wps.language == "fr-CA"
def list_remote_processes(service, request):
# type: (Service, Request) -> List[Process]
"""
Obtains a list of remote service processes in a compatible :class:`weaver.datatype.Process` format.
Note: remote processes won't be stored to the local process storage.
"""
wps = WebProcessingService(url=service.url,
headers=get_cookie_headers(request.headers))
set_wps_language(wps, request=request)
settings = get_settings(request)
return [
Process.from_ows(service, process, settings)
for process in wps.processes
]
def describe_provider_process(request):
# type: (Request) -> Process
"""
Obtains a remote service process description in a compatible local process format.
Note: this processes won't be stored to the local process storage.
"""
provider_id = request.matchdict.get("provider_id")
process_id = request.matchdict.get("process_id")
store = get_db(request).get_store(StoreServices)
service = store.fetch_by_name(provider_id)
wps = WebProcessingService(url=service.url,
headers=get_cookie_headers(request.headers))
set_wps_language(wps, request=request)
process = wps.describeprocess(process_id)
return Process.from_ows(service, process, get_settings(request))
def execute_process(self, job_id, url, headers=None):
from weaver.wps.service import get_pywps_service
LOGGER.debug("Job execute process called.")
settings = get_settings(app)
task_logger = get_task_logger(__name__)
load_pywps_config(settings)
task_logger.debug("Job task setup.")
# reset the connection because we are in a forked celery process
db = get_db(app, reset_connection=True)
store = db.get_store(StoreJobs)
job = store.fetch_by_id(job_id)
job.task_id = self.request.id
job.progress = JOB_PROGRESS_SETUP
job.save_log(logger=task_logger, message="Job task setup completed.")
job = store.update_job(job)
try:
try:
job.progress = JOB_PROGRESS_DESCRIBE
job.save_log(
logger=task_logger,
message="Execute WPS request for process [{!s}]".format(
job.process))
ssl_verify = get_ssl_verify_option("get", url, settings=settings)
wps = WebProcessingService(url=url,
headers=get_cookie_headers(headers),
verify=ssl_verify)
set_wps_language(wps, accept_language=job.accept_language)
raise_on_xml_exception(wps._capabilities) # noqa
except Exception as ex:
raise OWSNoApplicableCode(
"Failed to retrieve WPS capabilities. Error: [{}].".format(
str(ex)))
try:
process = wps.describeprocess(job.process)
except Exception as ex:
raise OWSNoApplicableCode(
"Failed to retrieve WPS process description. Error: [{}].".
format(str(ex)))
# prepare inputs
job.progress = JOB_PROGRESS_GET_INPUTS
job.save_log(logger=task_logger,
message="Fetching job input definitions.")
complex_inputs = []
for process_input in process.dataInputs:
if WPS_COMPLEX_DATA in process_input.dataType:
complex_inputs.append(process_input.identifier)
try:
wps_inputs = list()
for process_input in job.inputs:
input_id = get_any_id(process_input)
process_value = get_any_value(process_input)
# in case of array inputs, must repeat (id,value)
input_values = process_value if isinstance(
process_value, list) else [process_value]
# we need to support file:// scheme but PyWPS doesn't like them so remove the scheme file://
input_values = [
val[7:] if str(val).startswith("file://") else val
for val in input_values
]
# need to use ComplexDataInput structure for complex input
# need to use literal String for anything else than complex
# TODO: BoundingBox not supported
wps_inputs.extend([
(input_id, ComplexDataInput(input_value)
if input_id in complex_inputs else str(input_value))
for input_value in input_values
])
except KeyError:
wps_inputs = []
# prepare outputs
job.progress = JOB_PROGRESS_GET_OUTPUTS
job.save_log(logger=task_logger,
message="Fetching job output definitions.")
wps_outputs = [(o.identifier, o.dataType == WPS_COMPLEX_DATA)
for o in process.processOutputs]
mode = EXECUTE_MODE_ASYNC if job.execute_async else EXECUTE_MODE_SYNC
job.progress = JOB_PROGRESS_EXECUTE_REQUEST
job.save_log(logger=task_logger,
message="Starting job process execution.")
job.save_log(
logger=task_logger,
message=
"Following updates could take a while until the Application Package answers..."
)
wps_worker = get_pywps_service(environ=settings, is_worker=True)
execution = wps_worker.execute_job(job.process,
wps_inputs=wps_inputs,
wps_outputs=wps_outputs,
mode=mode,
job_uuid=job.id)
if not execution.process and execution.errors:
raise execution.errors[0]
# adjust status location
wps_status_path = get_wps_local_status_location(
execution.statusLocation, settings)
job.progress = JOB_PROGRESS_EXECUTE_STATUS_LOCATION
LOGGER.debug("WPS status location that will be queried: [%s]",
wps_status_path)
if not wps_status_path.startswith("http") and not os.path.isfile(
wps_status_path):
LOGGER.warning(
"WPS status location not resolved to local path: [%s]",
wps_status_path)
job.save_log(logger=task_logger,
level=logging.DEBUG,
message="Updated job status location: [{}].".format(
wps_status_path))
job.status = map_status(STATUS_STARTED)
job.status_message = execution.statusMessage or "{} initiation done.".format(
str(job))
job.status_location = wps_status_path
job.request = execution.request
job.response = execution.response
job.progress = JOB_PROGRESS_EXECUTE_MONITOR_START
job.save_log(logger=task_logger,
message="Starting monitoring of job execution.")
job = store.update_job(job)
max_retries = 5
num_retries = 0
run_step = 0
while execution.isNotComplete() or run_step == 0:
if num_retries >= max_retries:
raise Exception(
"Could not read status document after {} retries. Giving up."
.format(max_retries))
try:
# NOTE:
# Don't actually log anything here until process is completed (success or fail) so that underlying
# WPS execution logs can be inserted within the current job log and appear continuously.
# Only update internal job fields in case they get referenced elsewhere.
job.progress = JOB_PROGRESS_EXECUTE_MONITOR_LOOP
execution = check_wps_status(location=wps_status_path,
settings=settings,
sleep_secs=wait_secs(run_step))
job_msg = (execution.statusMessage or "").strip()
job.response = execution.response
job.status = map_status(execution.getStatus())
job.status_message = "Job execution monitoring (progress: {}%, status: {})."\
.format(execution.percentCompleted, job_msg or "n/a")
# job.save_log(logger=task_logger)
# job = store.update_job(job)
if execution.isComplete():
job.mark_finished()
job.progress = JOB_PROGRESS_EXECUTE_MONITOR_END
msg_progress = " (status: {})".format(
job_msg) if job_msg else ""
if execution.isSucceded():
job.status = map_status(STATUS_SUCCEEDED)
job.status_message = "Job succeeded{}.".format(
msg_progress)
wps_package.retrieve_package_job_log(execution, job)
job.save_log(logger=task_logger)
job_results = [
ows2json_output(output, process, settings)
for output in execution.processOutputs
]
job.results = make_results_relative(
job_results, settings)
else:
task_logger.debug("Job failed.")
job.status_message = "Job failed{}.".format(
msg_progress)
wps_package.retrieve_package_job_log(execution, job)
job.save_log(errors=execution.errors,
logger=task_logger)
task_logger.debug(
"Mapping Job references with generated WPS locations.")
map_locations(job, settings)
except Exception as exc:
num_retries += 1
task_logger.debug("Exception raised: %s", repr(exc))
job.status_message = "Could not read status XML document for {!s}. Trying again...".format(
job)
job.save_log(errors=execution.errors, logger=task_logger)
sleep(1)
else:
# job.status_message = "Update {}...".format(str(job))
# job.save_log(logger=task_logger)
num_retries = 0
run_step += 1
finally:
job = store.update_job(job)
except Exception as exc:
LOGGER.exception("Failed running [%s]", job)
job.status = map_status(STATUS_FAILED)
job.status_message = "Failed to run {!s}.".format(job)
job.progress = JOB_PROGRESS_EXECUTE_MONITOR_ERROR
exception_class = "{}.{}".format(
type(exc).__module__,
type(exc).__name__)
errors = "{0}: {1!s}".format(exception_class, exc)
job.save_log(errors=errors, logger=task_logger)
finally:
job.progress = JOB_PROGRESS_EXECUTE_MONITOR_END
job.status_message = "Job {}.".format(job.status)
job.save_log(logger=task_logger)
# Send email if requested
if job.notification_email is not None:
job.progress = JOB_PROGRESS_NOTIFY
try:
notify_job_complete(job, job.notification_email, settings)
message = "Notification email sent successfully."
job.save_log(logger=task_logger, message=message)
except Exception as exc:
exception_class = "{}.{}".format(
type(exc).__module__,
type(exc).__name__)
exception = "{0}: {1!s}".format(exception_class, exc)
message = "Couldn't send notification email ({})".format(
exception)
job.save_log(errors=message,
logger=task_logger,
message=message)
job.progress = JOB_PROGRESS_DONE
job.save_log(logger=task_logger, message="Job task complete.")
job = store.update_job(job)
return job.status