def register_builtin_processes(container):
# type: (AnyRegistryContainer) -> None
"""
Registers every ``builtin`` CWL package to the processes database.
CWL definitions must be located within the :mod:`weaver.processes.builtin` module.
"""
restapi_url = get_wps_restapi_base_url(container)
builtin_apps_mapping = _get_builtin_reference_mapping(
os.path.abspath(os.path.dirname(__file__)))
builtin_processes = []
for process_id, process_path in builtin_apps_mapping.items():
process_info = get_process_definition({},
package=None,
reference=process_path)
process_url = "/".join([restapi_url, "processes", process_id])
process_package = _get_builtin_package(process_id,
process_info["package"])
process_abstract = _get_builtin_metadata(process_id,
process_path,
"__doc__",
clean=True)
process_version = _get_builtin_metadata(process_id, process_path,
"__version__")
process_title = _get_builtin_metadata(process_id, process_path,
"__title__")
process_payload = {
"processDescription": {
"process": {
"id": process_id,
"type": ProcessType.BUILTIN,
"title": process_title,
"version": process_version,
"abstract": process_abstract,
}
},
"deploymentProfileName":
"http://www.opengis.net/profiles/eoc/builtinApplication",
"executionUnit": [{
"unit": process_package
}],
}
process_payload["processDescription"]["process"].update(
ows_context_href(process_url))
builtin_processes.append(
Process(
id=process_id,
type=ProcessType.BUILTIN,
title=process_title,
version=process_version,
abstract=process_abstract,
payload=process_payload,
package=process_package,
inputs=process_info["inputs"],
outputs=process_info["outputs"],
processDescriptionURL=process_url,
processEndpointWPS1=get_wps_url(container),
executeEndpoint="/".join([process_url, "jobs"]),
jobControlOptions=ExecuteControlOption.values(),
visibility=Visibility.PUBLIC,
))
# registration of missing/updated apps automatically applied with 'default_processes'
get_db(container).get_store(StoreProcesses,
default_processes=builtin_processes)
def deploy_process_from_payload(payload, container, overwrite=False):
# type: (JSON, AnyContainer, bool) -> HTTPException
"""
Deploy the process after resolution of all references and validation of the parameters from payload definition.
Adds a :class:`weaver.datatype.Process` instance to storage using the provided JSON ``payload``
matching :class:`weaver.wps_restapi.swagger_definitions.ProcessDescription`.
:param payload: JSON payload that was specified during the process deployment request.
:param container: container to retrieve application settings.
:param overwrite: whether to allow override of an existing process definition if conflict occurs.
:returns: HTTPOk if the process registration was successful.
:raises HTTPException: for any invalid process deployment step.
"""
# use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload
payload_copy = deepcopy(payload)
payload = _check_deploy(payload)
# validate identifier naming for unsupported characters
process_description = payload.get("processDescription")
process_info = process_description.get("process", {})
process_href = process_description.pop("href", None)
# retrieve CWL package definition, either via "href" (WPS-1/2), "owsContext" or "executionUnit" (package/reference)
deployment_profile_name = payload.get("deploymentProfileName", "").lower()
ows_context = process_info.pop("owsContext", None)
reference = None
package = None
if process_href:
reference = process_href # reference type handled downstream
elif isinstance(ows_context, dict):
offering = ows_context.get("offering")
if not isinstance(offering, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering'."
)
content = offering.get("content")
if not isinstance(content, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering.content'."
)
package = None
reference = content.get("href")
elif deployment_profile_name:
if not any(
deployment_profile_name.endswith(typ)
for typ in [PROCESS_APPLICATION, PROCESS_WORKFLOW]):
raise HTTPBadRequest(
"Invalid value for parameter 'deploymentProfileName'.")
execution_units = payload.get("executionUnit")
if not isinstance(execution_units, list):
raise HTTPUnprocessableEntity("Invalid parameter 'executionUnit'.")
for execution_unit in execution_units:
if not isinstance(execution_unit, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'executionUnit'.")
package = execution_unit.get("unit")
reference = execution_unit.get("href")
# stop on first package/reference found, simultaneous usage will raise during package retrieval
if package or reference:
break
else:
raise HTTPBadRequest(
"Missing one of required parameters [href, owsContext, deploymentProfileName]."
)
if process_info.get("type", "") == PROCESS_BUILTIN:
raise HTTPBadRequest(
"Invalid process type resolved from package: [{0}]. Deployment of {0} process is not allowed."
.format(PROCESS_BUILTIN))
# update and validate process information using WPS process offering, CWL/WPS reference or CWL package definition
settings = get_settings(container)
headers = getattr(
container, "headers", {}
) # container is any request (as when called from API Deploy request)
process_info = _validate_deploy_process_info(process_info, reference,
package, settings, headers)
restapi_url = get_wps_restapi_base_url(settings)
description_url = "/".join(
[restapi_url, "processes", process_info["identifier"]])
execute_endpoint = "/".join([description_url, "jobs"])
# ensure that required "processEndpointWPS1" in db is added,
# will be auto-fixed to localhost if not specified in body
process_info["processEndpointWPS1"] = process_description.get(
"processEndpointWPS1")
process_info["executeEndpoint"] = execute_endpoint
process_info["payload"] = payload_copy
process_info["jobControlOptions"] = process_description.get(
"jobControlOptions", [])
process_info["outputTransmission"] = process_description.get(
"outputTransmission", [])
process_info["processDescriptionURL"] = description_url
# insert the "resolved" context using details retrieved from "executionUnit"/"href" or directly with "owsContext"
if "owsContext" not in process_info and reference:
process_info["owsContext"] = {
"offering": {
"content": {
"href": str(reference)
}
}
}
elif isinstance(ows_context, dict):
process_info["owsContext"] = ows_context
# bw-compat abstract/description (see: ProcessDeployment schema)
if "description" not in process_info or not process_info["description"]:
process_info["description"] = process_info.get("abstract", "")
# FIXME: handle colander invalid directly in tween (https://github.com/crim-ca/weaver/issues/112)
try:
store = get_db(container).get_store(StoreProcesses)
process = Process(process_info)
sd.ProcessSummary().deserialize(
process) # make if fail before save if invalid
store.save_process(process, overwrite=overwrite)
process_summary = process.summary()
except ProcessRegistrationError as ex:
raise HTTPConflict(detail=str(ex))
except (ValueError, colander.Invalid) as ex:
# raised on invalid process name
raise HTTPBadRequest(detail=str(ex))
return HTTPCreated(
json={
"description": sd.OkPostProcessesResponse.description,
"processSummary": process_summary,
"deploymentDone": True
})
def deploy_process_from_payload(payload, container):
# type: (JSON, AnyContainer) -> HTTPException
"""
Adds a :class:`weaver.datatype.Process` instance to storage using the provided JSON ``payload`` matching
:class:`weaver.wps_restapi.swagger_definitions.ProcessDescription`.
:returns: HTTPOk if the process registration was successful
:raises HTTPException: otherwise
"""
_check_deploy(payload)
# use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload
payload_copy = deepcopy(payload)
# validate identifier naming for unsupported characters
process_description = payload.get("processDescription")
process_info = process_description.get("process", {})
process_href = process_description.pop("href", None)
# retrieve CWL package definition, either via "href" (WPS-1/2), "owsContext" or "executionUnit" (package/reference)
deployment_profile_name = payload.get("deploymentProfileName", "").lower()
ows_context = process_info.pop("owsContext", None)
reference = None
package = None
if process_href:
reference = process_href # reference type handled downstream
elif isinstance(ows_context, dict):
offering = ows_context.get("offering")
if not isinstance(offering, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering'."
)
content = offering.get("content")
if not isinstance(content, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering.content'."
)
package = None
reference = content.get("href")
elif deployment_profile_name:
if not any(
deployment_profile_name.endswith(typ)
for typ in [PROCESS_APPLICATION, PROCESS_WORKFLOW]):
raise HTTPBadRequest(
"Invalid value for parameter 'deploymentProfileName'.")
execution_units = payload.get("executionUnit")
if not isinstance(execution_units, list):
raise HTTPUnprocessableEntity("Invalid parameter 'executionUnit'.")
for execution_unit in execution_units:
if not isinstance(execution_unit, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'executionUnit'.")
package = execution_unit.get("unit")
reference = execution_unit.get("href")
# stop on first package/reference found, simultaneous usage will raise during package retrieval
if package or reference:
break
else:
raise HTTPBadRequest(
"Missing one of required parameters [href, owsContext, deploymentProfileName]."
)
# obtain updated process information using WPS process offering, CWL/WPS reference or CWL package definition
process_info = _get_deploy_process_info(process_info, reference, package)
# validate process type against weaver configuration
settings = get_settings(container)
process_type = process_info["type"]
if process_type == PROCESS_WORKFLOW:
weaver_config = get_weaver_configuration(settings)
if weaver_config != WEAVER_CONFIGURATION_EMS:
raise HTTPBadRequest(
"Invalid [{0}] package deployment on [{1}].".format(
process_type, weaver_config))
restapi_url = get_wps_restapi_base_url(settings)
description_url = "/".join(
[restapi_url, "processes", process_info["identifier"]])
execute_endpoint = "/".join([description_url, "jobs"])
# ensure that required "processEndpointWPS1" in db is added,
# will be auto-fixed to localhost if not specified in body
process_info["processEndpointWPS1"] = process_description.get(
"processEndpointWPS1")
process_info["executeEndpoint"] = execute_endpoint
process_info["payload"] = payload_copy
process_info["jobControlOptions"] = process_description.get(
"jobControlOptions", [])
process_info["outputTransmission"] = process_description.get(
"outputTransmission", [])
process_info["processDescriptionURL"] = description_url
# insert the "resolved" context using details retrieved from "executionUnit"/"href" or directly with "owsContext"
if "owsContext" not in process_info and reference:
process_info["owsContext"] = {
"offering": {
"content": {
"href": str(reference)
}
}
}
elif isinstance(ows_context, dict):
process_info["owsContext"] = ows_context
try:
store = get_db(container).get_store(StoreProcesses)
saved_process = store.save_process(Process(process_info),
overwrite=False)
except ProcessRegistrationError as ex:
raise HTTPConflict(detail=str(ex))
except ValueError as ex:
# raised on invalid process name
raise HTTPBadRequest(detail=str(ex))
json_response = {
"processSummary": saved_process.process_summary(),
"deploymentDone": True
}
return HTTPOk(
json=json_response
) # FIXME: should be 201 (created), update swagger accordingly
def test_process_split_version(process_id, result):
assert Process.split_version(process_id) == result
def execute_process(task, job_id, wps_url, headers=None):
# type: (Task, UUID, str, Optional[HeadersType]) -> StatusType
"""
Celery task that executes the WPS process job monitoring as status updates (local and remote).
"""
from weaver.wps.service import get_pywps_service
LOGGER.debug("Job execute process called.")
task_process = get_celery_process()
rss_start = task_process.memory_info().rss
registry = get_registry(
None) # local thread, whether locally or dispatched celery
settings = get_settings(registry)
db = get_db(
registry, reset_connection=True
) # reset the connection because we are in a forked celery process
store = db.get_store(StoreJobs)
job = store.fetch_by_id(job_id)
job.started = now()
job.status = Status.STARTED # will be mapped to 'RUNNING'
job.status_message = f"Job {Status.STARTED}." # will preserve detail of STARTED vs RUNNING
job.save_log(message=job.status_message)
task_logger = get_task_logger(__name__)
job.save_log(logger=task_logger, message="Job task setup initiated.")
load_pywps_config(settings)
job.progress = JobProgress.SETUP
job.task_id = task.request.id
job.save_log(logger=task_logger, message="Job task setup completed.")
job = store.update_job(job)
# Flag to keep track if job is running in background (remote-WPS, CWL app, etc.).
# If terminate signal is sent to worker task via API dismiss request while still running in background,
# the raised exception within the task will switch the job to Status.FAILED, but this will not raise an
# exception here. Since the task execution 'succeeds' without raising, it skips directly to the last 'finally'.
# Patch it back to Status.DISMISSED in this case.
task_terminated = True
try:
job.progress = JobProgress.DESCRIBE
job.save_log(logger=task_logger,
message=f"Employed WPS URL: [{wps_url!s}]",
level=logging.DEBUG)
job.save_log(
logger=task_logger,
message=f"Execute WPS request for process [{job.process!s}]")
wps_process = fetch_wps_process(job, wps_url, headers, settings)
# prepare inputs
job.progress = JobProgress.GET_INPUTS
job.save_log(logger=task_logger,
message="Fetching job input definitions.")
wps_inputs = parse_wps_inputs(wps_process, job)
# prepare outputs
job.progress = JobProgress.GET_OUTPUTS
job.save_log(logger=task_logger,
message="Fetching job output definitions.")
wps_outputs = [(o.identifier, o.dataType == WPS_COMPLEX_DATA)
for o in wps_process.processOutputs]
# if process refers to a remote WPS provider, pass it down to avoid unnecessary re-fetch request
if job.is_local:
process = None # already got all the information needed pre-loaded in PyWPS service
else:
service = Service(name=job.service, url=wps_url)
process = Process.from_ows(wps_process, service, settings)
job.progress = JobProgress.EXECUTE_REQUEST
job.save_log(logger=task_logger,
message="Starting job process execution.")
job.save_log(
logger=task_logger,
message=
"Following updates could take a while until the Application Package answers..."
)
wps_worker = get_pywps_service(environ=settings, is_worker=True)
execution = wps_worker.execute_job(job,
wps_inputs=wps_inputs,
wps_outputs=wps_outputs,
remote_process=process,
headers=headers)
if not execution.process and execution.errors:
raise execution.errors[0]
# adjust status location
wps_status_path = get_wps_local_status_location(
execution.statusLocation, settings)
job.progress = JobProgress.EXECUTE_STATUS_LOCATION
LOGGER.debug("WPS status location that will be queried: [%s]",
wps_status_path)
if not wps_status_path.startswith("http") and not os.path.isfile(
wps_status_path):
LOGGER.warning(
"WPS status location not resolved to local path: [%s]",
wps_status_path)
job.save_log(
logger=task_logger,
level=logging.DEBUG,
message=f"Updated job status location: [{wps_status_path}].")
job.status = Status.RUNNING
job.status_message = execution.statusMessage or f"{job!s} initiation done."
job.status_location = wps_status_path
job.request = execution.request
job.response = execution.response
job.progress = JobProgress.EXECUTE_MONITOR_START
job.save_log(logger=task_logger,
message="Starting monitoring of job execution.")
job = store.update_job(job)
max_retries = 5
num_retries = 0
run_step = 0
while execution.isNotComplete() or run_step == 0:
if num_retries >= max_retries:
job.save_log(errors=execution.errors, logger=task_logger)
job = store.update_job(job)
raise Exception(
f"Could not read status document after {max_retries} retries. Giving up."
)
try:
# NOTE:
# Don't actually log anything here until process is completed (success or fail) so that underlying
# WPS execution logs can be inserted within the current job log and appear continuously.
# Only update internal job fields in case they get referenced elsewhere.
progress_min = JobProgress.EXECUTE_MONITOR_LOOP
progress_max = JobProgress.EXECUTE_MONITOR_DONE
job.progress = progress_min
run_delay = wait_secs(run_step)
execution = check_wps_status(location=wps_status_path,
settings=settings,
sleep_secs=run_delay)
job_msg = (execution.statusMessage or "").strip()
job.response = execution.response
job.status = map_status(execution.getStatus())
job_status_msg = job_msg or "n/a"
job_percent = execution.percentCompleted
job.status_message = f"Job execution monitoring (progress: {job_percent}%, status: {job_status_msg})."
if execution.isComplete():
msg_progress = f" (status: {job_msg})" if job_msg else ""
if execution.isSucceded():
wps_package.retrieve_package_job_log(
execution, job, progress_min, progress_max)
job.status = map_status(Status.SUCCEEDED)
job.status_message = f"Job succeeded{msg_progress}."
job.progress = progress_max
job.save_log(logger=task_logger)
job_results = [
ows2json_output_data(output, process, settings)
for output in execution.processOutputs
]
job.results = make_results_relative(
job_results, settings)
else:
task_logger.debug("Job failed.")
wps_package.retrieve_package_job_log(
execution, job, progress_min, progress_max)
job.status_message = f"Job failed{msg_progress}."
job.progress = progress_max
job.save_log(errors=execution.errors,
logger=task_logger)
task_logger.debug(
"Mapping Job references with generated WPS locations.")
map_locations(job, settings)
job = store.update_job(job)
except Exception as exc:
num_retries += 1
task_logger.debug("Exception raised: %s", repr(exc))
job.status_message = f"Could not read status XML document for {job!s}. Trying again..."
job.save_log(errors=execution.errors, logger=task_logger)
job = store.update_job(job)
sleep(1)
else:
num_retries = 0
run_step += 1
finally:
task_terminated = False # reached only if WPS execution completed (worker not terminated beforehand)
job = store.update_job(job)
except Exception as exc:
# if 'execute_job' finishes quickly before even reaching the 'monitoring loop'
# consider WPS execution produced an error (therefore Celery worker not terminated)
task_terminated = False
LOGGER.exception("Failed running [%s]", job)
LOGGER.debug("Failed job [%s] raised an exception.", job, exc_info=exc)
# note: don't update the progress here to preserve last one that was set
job.status = map_status(Status.FAILED)
job.status_message = f"Failed to run {job!s}."
errors = f"{fully_qualified_name(exc)}: {exc!s}"
job.save_log(errors=errors, logger=task_logger)
job = store.update_job(job)
finally:
# if task worker terminated, local 'job' is out of date compared to remote/background runner last update
job = store.fetch_by_id(job.id)
if task_terminated and map_status(job.status) == Status.FAILED:
job.status = Status.DISMISSED
task_success = map_status(
job.status) not in JOB_STATUS_CATEGORIES[StatusCategory.FAILED]
collect_statistics(task_process, settings, job, rss_start)
if task_success:
job.progress = JobProgress.EXECUTE_MONITOR_END
job.status_message = f"Job {job.status}."
job.save_log(logger=task_logger)
if task_success:
job.progress = JobProgress.NOTIFY
send_job_complete_notification_email(job, task_logger, settings)
if job.status not in JOB_STATUS_CATEGORIES[StatusCategory.FINISHED]:
job.status = Status.SUCCEEDED
job.status_message = f"Job {job.status}."
job.mark_finished()
if task_success:
job.progress = JobProgress.DONE
job.save_log(logger=task_logger, message="Job task complete.")
job = store.update_job(job)
return job.status
def list_processes(
self,
visibility=None, # type: Optional[str]
page=None, # type: Optional[int]
limit=None, # type: Optional[int]
sort=None, # type: Optional[str]
total=False, # type: bool
): # type: (...) -> Union[List[Process], Tuple[List[Process], int]]
"""
Lists all processes in database, optionally filtered by `visibility`.
:param visibility: One value amongst `weaver.visibility`.
:param page: page number to return when using result paging.
:param limit: number of processes per page when using result paging.
:param sort: field which is used for sorting results (default: process ID, descending).
:param total: request the total number of processes to be calculated (ignoring paging).
:returns:
List of sorted, and possibly page-filtered, processes matching queries.
If ``total`` was requested, return a tuple of this list and the number of processes.
"""
search_filters = {}
if visibility is None:
visibility = VISIBILITY_VALUES
if isinstance(visibility, str):
visibility = [visibility]
for v in visibility:
if v not in VISIBILITY_VALUES:
raise ValueError(
"Invalid visibility value '{0!s}' is not one of {1!s}".
format(v, list(VISIBILITY_VALUES)))
search_filters["visibility"] = {"$in": list(visibility)}
# processes do not have 'created', but ObjectID in '_id' has the particularity of embedding creation time
if sort == SORT_CREATED:
sort = "_id"
# replace equivalent aliases to corresponding fields in db
if sort in [SORT_ID, SORT_PROCESS]:
sort = SORT_ID_LONG
sort_allowed = list(PROCESS_SORT_VALUES) + ["_id"]
sort_method = {
"$sort": self._apply_sort_method(sort, SORT_ID_LONG, sort_allowed)
}
search_pipeline = [{"$match": search_filters}, sort_method]
paging_pipeline = []
if page is not None and limit is not None:
paging_pipeline = self._apply_paging_pipeline(page, limit)
if total:
pipeline = self._apply_total_result(search_pipeline,
paging_pipeline)
else:
pipeline = search_pipeline + paging_pipeline
LOGGER.debug("Process listing pipeline:\n%s",
repr_json(pipeline, indent=2))
found = list(
self.collection.aggregate(pipeline,
collation=Collation(locale="en")))
if total:
items = [Process(item) for item in found[0]["items"]]
total = found[0]["total"]
return items, total
return [Process(item) for item in found]
def test_stdout_stderr_logging_for_commandline_tool_failure():
"""
Execute a process and assert that stderr is correctly logged to log file.
"""
process = Process({
"title": "test-stdout-stderr",
"id": "test-stdout-stderr",
"package": {
"cwlVersion": "v1.0",
"class": "CommandLineTool",
"baseCommand": "not_existing_command",
"inputs": {
"message": {
"type": "string",
"inputBinding": {
"position": 1
}
}
},
"outputs": {}
}
})
payload = process
package = process["package"]
title = process["title"]
identifier = process["id"]
# WPSPackage._handle()
log_file = tempfile.NamedTemporaryFile()
status_location = log_file.name
workdir = tempfile.TemporaryDirectory()
class TestWpsPackage(WpsPackage):
@property
def status_location(self):
return status_location
wps_package_instance = TestWpsPackage(identifier=identifier,
title=title,
payload=payload,
package=package)
wps_package_instance.set_workdir(workdir.name)
# WPSRequest mock
wps_request = WPSRequest()
wps_request.json = {
"identifier": "test-stdout-stderr",
"operation": "execute",
"version": "1.0.0",
"language": "null",
"identifiers": "null",
"store_execute": "true",
"status": "true",
"lineage": "true",
"raw": "false",
"inputs": {
"message": [{
"identifier": "message",
"title": "A dummy message",
"type": "literal",
"data_type": "string",
"data": "Dummy message",
"allowed_values": [],
}]
},
"outputs": {}
}
# ExecuteResponse mock
wps_response = type("", (object, ), {
"_update_status": lambda *_, **__: 1
})()
from weaver.exceptions import PackageExecutionError
try:
wps_package_instance._handler(wps_request, wps_response)
except PackageExecutionError as exception:
assert "Completed permanentFail" in exception.args[0]
else:
fail(
"\"wps_package._handler()\" was expected to throw \"PackageExecutionError\" exception"
)
def test_stdout_stderr_logging_for_commandline_tool_success():
"""
Execute a process and assert that stdout is correctly logged to log file.
"""
process = Process({
"title": "test-stdout-stderr",
"id": "test-stdout-stderr",
"package": {
"cwlVersion": "v1.0",
"class": "CommandLineTool",
"baseCommand": "echo",
"inputs": {
"message": {
"type": "string",
"inputBinding": {
"position": 1
}
}
},
"outputs": {}
}
})
payload = process
package = process["package"]
title = process["title"]
identifier = process["id"]
# WPSPackage._handle()
log_file = tempfile.NamedTemporaryFile()
status_location = log_file.name
workdir = tempfile.TemporaryDirectory()
class TestWpsPackage(WpsPackage):
@property
def status_location(self):
return status_location
wps_package_instance = TestWpsPackage(identifier=identifier,
title=title,
payload=payload,
package=package)
wps_package_instance.set_workdir(workdir.name)
# WPSRequest mock
wps_request = WPSRequest()
wps_request.json = {
"identifier": "test-stdout-stderr",
"operation": "execute",
"version": "1.0.0",
"language": "null",
"identifiers": "null",
"store_execute": "true",
"status": "true",
"lineage": "true",
"raw": "false",
"inputs": {
"message": [{
"identifier": "message",
"title": "A dummy message",
"type": "literal",
"data_type": "string",
"data": "Dummy message",
"allowed_values": [],
}]
},
"outputs": {}
}
# ExecuteResponse mock
wps_response = type("", (object, ), {
"_update_status": lambda *_, **__: 1
})()
wps_package_instance._handler(wps_request, wps_response)
# log assertions
with open(status_location + ".log", "r") as file:
log_data = file.read()
assert "Dummy message" in log_data
def get_job(request):
# type: (PyramidRequest) -> Job
"""
Obtain a :term:`Job` from request parameters.
.. versionchanged:: 4.20
When looking for :term:`Job` that refers to a local :term:`Process`, allow implicit resolution of the
unspecified ``version`` portion to automatically resolve the identifier. Consider that validation of
the expected :term:`Process` for this :term:`Job` is "good enough", since the specific ID is not actually
required to obtain the :term:`Job` (could be queried by ID only on the ``/jobs/{jobId}`` endpoint.
If the ``version`` is provided though (either query parameter or tagged representation), the validation
will ensure that it matches explicitly.
:param request: Request with path and query parameters to retrieve the desired job.
:returns: Job information if found.
:raise HTTPNotFound: with JSON body details on missing/non-matching job, process, provider IDs.
"""
job_id = request.matchdict.get("job_id")
try:
if not is_uuid(job_id):
raise JobInvalidParameter
store = get_db(request).get_store(StoreJobs)
job = store.fetch_by_id(job_id)
except (JobInvalidParameter, JobNotFound) as exc:
exception = type(exc)
if exception is JobInvalidParameter:
desc = "Invalid job reference is not a valid UUID."
else:
desc = "Could not find job with specified reference."
title = "NoSuchJob"
raise exception(
# new format: https://docs.ogc.org/is/18-062r2/18-062r2.html#req_core_job-exception-no-such-job
json={
"title": title,
"type":
"http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-job",
"detail": desc,
"status": exception.code,
"cause": str(job_id)
},
code=title,
locator="JobID",
description=desc # old format
)
provider_id = request.matchdict.get("provider_id", job.service)
process_tag = request.matchdict.get("process_id")
if process_tag:
process_tag = resolve_process_tag(
request) # find version if available as well
else:
process_tag = job.process
if provider_id:
forbid_local_only(request)
if job.service != provider_id:
title = "NoSuchProvider"
desc = "Could not find job reference corresponding to specified provider reference."
raise OWSNotFound(
# new format: https://docs.ogc.org/is/18-062r2/18-062r2.html#req_core_job-exception-no-such-job
json={
"title": title,
"type":
"http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-job",
"detail": desc,
"status": OWSNotFound.code,
"cause": str(provider_id)
},
code=title,
locator="provider",
description=desc # old format
)
process_id = Process.split_version(process_tag)[0]
if job.process not in [process_id, process_tag]:
title = "NoSuchProcess"
desc = "Could not find job reference corresponding to specified process reference."
raise OWSNotFound(
# new format: https://docs.ogc.org/is/18-062r2/18-062r2.html#req_core_job-exception-no-such-job
# note: although 'no-such-process' error, return 'no-such-job' because process could exist, only mismatches
json={
"title": title,
"type":
"http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-job",
"detail": desc,
"status": OWSNotFound.code,
"cause": str(process_tag)
},
code=title,
locator="process",
description=desc # old format
)
return job
def get_opensearch_process():
return Process(load_json_test_file("opensearch_process.json"))
