def register_builtin_processes(container):
# type: (AnySettingsContainer) -> None
"""
Registers every ``builtin`` CWL package to the processes database.
CWL definitions must be located within the :mod:`weaver.processes.builtin` module.
"""
restapi_url = get_wps_restapi_base_url(container)
builtin_apps_mapping = _get_builtin_reference_mapping(os.path.abspath(os.path.dirname(__file__)))
builtin_processes = []
for process_id, process_path in builtin_apps_mapping.items():
process_info = get_process_definition({}, package=None, reference=process_path)
process_url = "/".join([restapi_url, "processes", process_id])
process_package = _get_builtin_package(process_id, process_info["package"])
process_abstract = _get_builtin_metadata(process_id, process_path, "__doc__", clean=True)
process_version = _get_builtin_metadata(process_id, process_path, "__version__")
process_title = _get_builtin_metadata(process_id, process_path, "__title__")
process_payload = {
"processDescription": {
"process": {
"id": process_id,
"type": PROCESS_BUILTIN,
"title": process_title,
"version": process_version,
"abstract": process_abstract,
}
},
"deploymentProfileName": "http://www.opengis.net/profiles/eoc/builtinApplication",
"executionUnit": [{"unit": process_package}],
}
process_payload["processDescription"]["process"].update(ows_context_href(process_url))
builtin_processes.append(Process(
id=process_id,
type=PROCESS_BUILTIN,
title=process_title,
version=process_version,
abstract=process_abstract,
payload=process_payload,
package=process_package,
inputs=process_info["inputs"],
outputs=process_info["outputs"],
processDescriptionURL=process_url,
processEndpointWPS1=get_wps_url(container),
executeEndpoint="/".join([process_url, "jobs"]),
visibility=VISIBILITY_PUBLIC,
))
# registration of missing/updated apps automatically applied with 'default_processes'
get_db(container).get_store(StoreProcesses, default_processes=builtin_processes)
def set_process_visibility(request):
"""
Set the visibility of a registered local process.
"""
visibility_value = request.json.get("value")
process_id = request.matchdict.get("process_id")
if not isinstance(process_id, str):
raise HTTPUnprocessableEntity("Invalid process identifier.")
if not isinstance(visibility_value, str):
raise HTTPUnprocessableEntity(
"Invalid visibility value specified. String expected.")
if visibility_value not in VISIBILITY_VALUES:
raise HTTPBadRequest("Invalid visibility value specified: {!s}".format(
visibility_value))
try:
store = get_db(request).get_store(StoreProcesses)
process = store.fetch_by_id(process_id)
if process.type == PROCESS_BUILTIN:
raise HTTPForbidden(
"Cannot change the visibility of builtin process.")
store.set_visibility(process_id, visibility_value)
return HTTPOk(json={u"value": visibility_value})
except TypeError:
raise HTTPBadRequest("Value of visibility must be a string.")
except ValueError:
raise HTTPUnprocessableEntity(
"Value of visibility must be one of : {!s}".format(
list(VISIBILITY_VALUES)))
except ProcessNotFound as ex:
raise HTTPNotFound(str(ex))
def cancel_job_batch(request):
# type: (PyramidRequest) -> AnyResponseType
"""
Dismiss operation for multiple jobs.
Note:
Will only stop tracking jobs when underlying remote provider services do not support cancel operation.
"""
try:
body = sd.DeleteJobsBodySchema().deserialize(request.json)
jobs = body["jobs"]
except Invalid as exc:
raise HTTPUnprocessableEntity(json={"code": Invalid.__name__, "description": str(exc)})
except Exception as exc:
raise HTTPBadRequest(json={"code": "Could not parse request body.", "description": str(exc)})
store = get_db(request).get_store(StoreJobs)
found_jobs = []
for job_id in jobs:
try:
job = store.fetch_by_id(job_id)
except JobNotFound as exc:
LOGGER.debug("Job [%s] not found, cannot be dismissed: [%s]", job_id, exc)
continue
found_jobs.append(job.id)
try:
dismiss_job_task(job, request)
except JobNotFound as exc:
LOGGER.debug("Job [%s] cannot be dismissed: %s.", job_id, exc.description)
body["description"] = "Following jobs have been successfully dismissed."
body = sd.BatchDismissJobsBodySchema().deserialize({"jobs": found_jobs})
return HTTPOk(json=body)
def delete_local_process(request):
# type: (PyramidRequest) -> AnyViewResponse
"""
Unregister a local process.
"""
db = get_db(request)
proc_store = db.get_store(StoreProcesses)
process = get_process(request=request)
process_id = process.id
if not process.mutable:
raise HTTPForbidden(json={
"title": "Process immutable.",
"type": "ProcessImmutable",
"detail": "Cannot delete an immutable process.",
"status": HTTPForbidden.code,
"cause": {"mutable": False}
})
job_store = db.get_store(StoreJobs)
jobs, total = job_store.find_jobs(process=process_id, status=Status.RUNNING, page=None, limit=None)
if total != 0:
raise HTTPForbidden(json={
"title": "ProcessBusy",
"type": "ProcessBusy",
"detail": "Process with specified identifier is in use by a least one job and cannot be undeployed.",
"status": HTTPForbidden.code,
"cause": {"jobs": [str(job.id) for job in jobs]}
})
if proc_store.delete_process(process_id, visibility=Visibility.PUBLIC):
return HTTPOk(json={
"description": sd.OkDeleteProcessResponse.description,
"identifier": process_id,
"undeploymentDone": True,
})
LOGGER.error("Existing process [%s] should have been deleted with success status.", process_id)
raise HTTPForbidden("Deletion of process has been refused by the database or could not have been validated.")
def set_process_visibility(request):
# type: (PyramidRequest) -> AnyViewResponse
"""
Set the visibility of a registered local process.
"""
visibility = Visibility.get(request.json.get("value"))
process_id = request.matchdict.get("process_id")
if not isinstance(process_id, str):
raise HTTPUnprocessableEntity("Invalid process identifier.")
if visibility not in Visibility:
raise HTTPBadRequest(f"Invalid visibility value specified: {visibility!s}")
try:
store = get_db(request).get_store(StoreProcesses)
process = store.fetch_by_id(process_id)
if not process.mutable:
raise HTTPForbidden("Cannot change the visibility of builtin process.")
store.set_visibility(process_id, visibility)
return HTTPOk(json={u"value": visibility})
except TypeError:
raise HTTPBadRequest("Value of visibility must be a string.")
except ValueError:
raise HTTPUnprocessableEntity(f"Value of visibility must be one of : {Visibility.values()!s}")
except ProcessNotFound as ex:
raise HTTPNotFound(str(ex))
def get_pywps_service(environ=None, is_worker=False):
# type: (SettingsType, bool) -> WorkerService
"""
Generates the PyWPS Service that provides WPS-1/2 XML endpoint.
"""
environ = environ or {}
try:
# get config file
registry = get_registry()
settings = get_settings(registry)
pywps_cfg = environ.get("PYWPS_CFG") or settings.get(
"PYWPS_CFG") or os.getenv("PYWPS_CFG")
if not isinstance(
pywps_cfg,
ConfigParser) or not settings.get("weaver.wps_configured"):
load_pywps_config(settings, config=pywps_cfg)
# call pywps application with processes filtered according to the adapter's definition
process_store = get_db(registry).get_store(
StoreProcesses) # type: StoreProcesses
processes_wps = [
process.wps() for process in process_store.list_processes(
visibility=Visibility.PUBLIC)
]
service = WorkerService(processes_wps,
is_worker=is_worker,
settings=settings)
except Exception as ex:
LOGGER.exception(
"Error occurred during PyWPS Service and/or Processes setup.")
raise OWSNoApplicableCode(
f"Failed setup of PyWPS Service and/or Processes. Error [{ex!r}]")
return service
def get_process(process_id=None, request=None, settings=None, store=None):
# type: (Optional[str], Optional[Request], Optional[SettingsType], Optional[StoreProcesses]) -> Process
"""
Obtain the specified process and validate information, returning appropriate HTTP error if invalid.
Process identifier must be provided from either the request path definition or literal ID.
Database must be retrievable from either the request, underlying settings, or direct store reference.
Different parameter combinations are intended to be used as needed or more appropriate, such that redundant
operations can be reduced where some objects are already fetched from previous operations.
"""
if process_id is None and request is not None:
process_id = request.matchdict.get("process_id")
if store is None:
store = get_db(settings or request).get_store(StoreProcesses)
try:
process = store.fetch_by_id(process_id, visibility=VISIBILITY_PUBLIC)
return process
except (InvalidIdentifierValue, MissingIdentifierValue) as ex:
raise HTTPBadRequest(str(ex))
except ProcessNotAccessible:
raise HTTPForbidden(
"Process with ID '{!s}' is not accessible.".format(process_id))
except ProcessNotFound:
raise HTTPNotFound(
"Process with ID '{!s}' does not exist.".format(process_id))
except colander.Invalid as ex:
raise HTTPBadRequest("Invalid schema:\n[{0!r}].".format(ex))
def get_bill_list(request):
"""
Get list of bills IDs.
"""
store = get_db(request).get_store(StoreBills)
bills = store.list_bills()
return HTTPOk(json={"bills": [b.id for b in bills]})
def get_quote_list(request):
"""
Get list of quotes IDs.
"""
page = int(request.params.get("page", "0"))
limit = int(request.params.get("limit", "10"))
filters = {
"process_id":
request.params.get("process", None)
or request.matchdict.get("process_id", None),
"page":
page,
"limit":
limit,
"sort":
request.params.get("sort", sort.SORT_CREATED),
}
store = get_db(request).get_store(StoreQuotes)
items, count = store.find_quotes(**filters)
return HTTPOk(
json={
"count": count,
"page": page,
"limit": limit,
"quotes": [quote.id for quote in items]
})
def get_providers(request):
"""
Lists registered providers.
"""
store = get_db(request).get_store(StoreServices)
providers = []
for service in store.list_services():
try:
if service.type.lower() != "wps":
continue
wps = WebProcessingService(url=service.url,
headers=get_cookie_headers(
request.headers))
providers.append(
dict(id=service.name,
title=getattr(wps.identification, "title", ""),
abstract=getattr(wps.identification, "abstract", ""),
url="{base_url}/providers/{provider_id}".format(
base_url=get_wps_restapi_base_url(
get_settings(request)),
provider_id=service.name),
public=service.public))
except Exception as exc:
warnings.warn(
"Exception occurred while fetching wps {0} : {1!r}".format(
service.url, exc), NonBreakingExceptionWarning)
return HTTPOk(json={"providers": providers})
def add_provider(request):
"""
Add a provider.
"""
store = get_db(request).get_store(StoreServices)
try:
new_service = Service(url=request.json["url"],
name=get_any_id(request.json))
except KeyError as exc:
raise OWSMissingParameterValue(
"Missing json parameter '{!s}'.".format(exc), value=exc)
if "public" in request.json:
new_service["public"] = request.json["public"]
if "auth" in request.json:
new_service["auth"] = request.json["auth"]
try:
store.save_service(new_service)
except NotImplementedError:
raise OWSNotImplemented(
sd.NotImplementedPostProviderResponse.description,
value=new_service)
return HTTPCreated(json=get_capabilities(new_service, request))
def submit_job_handler(payload, # type: JSON
settings, # type: SettingsType
service_url, # type: str
provider_id=None, # type: Optional[str]
process_id=None, # type: str
is_workflow=False, # type: bool
is_local=True, # type: bool
visibility=None, # type: Optional[str]
language=None, # type: Optional[str]
auth=None, # type: Optional[HeaderCookiesType]
tags=None, # type: Optional[List[str]]
user=None, # type: Optional[int]
context=None, # type: Optional[str]
): # type: (...) -> JSON
"""
Submits the job to the Celery worker with provided parameters.
Assumes that parameters have been pre-fetched and validated, except for the input payload.
"""
try:
json_body = sd.Execute().deserialize(payload)
except colander.Invalid as ex:
raise HTTPBadRequest("Invalid schema: [{}]".format(str(ex)))
# TODO: remove when all parameter variations are supported
# FIXME:
# - support 'sync' and 'Prefer' header variants (https://github.com/crim-ca/weaver/issues/247)
# - support 'response: raw' (https://github.com/crim-ca/weaver/issues/376)
# - allow omitting 'outputs' (https://github.com/crim-ca/weaver/issues/375)
_validate_job_parameters(json_body)
is_execute_async = json_body["mode"] != EXECUTE_MODE_SYNC # convert auto to async
notification_email = json_body.get("notification_email")
encrypted_email = encrypt_email(notification_email, settings) if notification_email else None
store = get_db(settings).get_store(StoreJobs)
job = store.save_job(task_id=STATUS_ACCEPTED, process=process_id, service=provider_id,
inputs=json_body.get("inputs"), is_local=is_local, is_workflow=is_workflow,
access=visibility, user_id=user, execute_async=is_execute_async, custom_tags=tags,
notification_email=encrypted_email, accept_language=language, context=context)
job.save_log(logger=LOGGER, message="Job task submitted for execution.", status=STATUS_ACCEPTED, progress=0)
job = store.update_job(job)
result = execute_process.delay(job_id=job.id, wps_url=clean_ows_url(service_url), headers=auth)
LOGGER.debug("Celery pending task [%s] for job [%s].", result.id, job.id)
# local/provider process location
location_base = "/providers/{provider_id}".format(provider_id=provider_id) if provider_id else ""
location = "{base_url}{location_base}/processes/{process_id}/jobs/{job_id}".format(
base_url=get_wps_restapi_base_url(settings),
location_base=location_base,
process_id=process_id,
job_id=job.id)
body_data = {
"jobID": job.id,
"processID": job.process,
"providerID": provider_id, # dropped by validator if not applicable
"status": map_status(STATUS_ACCEPTED),
"location": location
}
return body_data
def setup_mongodb_jobstore(config=None):
# type: (Optional[Configurator]) -> MongodbJobStore
"""Setup store using mongodb, will be enforced if not configured properly."""
config = setup_config_with_mongodb(config)
store = get_db(config).get_store(MongodbJobStore)
store.clear_jobs()
return store
def get_pywps_service(environ=None, is_worker=False):
"""
Generates the PyWPS Service that provides *older* WPS-1/2 XML endpoint.
"""
environ = environ or {}
try:
# get config file
settings = get_settings(app)
pywps_cfg = environ.get("PYWPS_CFG") or settings.get(
"PYWPS_CFG") or os.getenv("PYWPS_CFG")
if not isinstance(
pywps_cfg,
ConfigParser) or not settings.get("weaver.wps_configured"):
load_pywps_config(app, config=pywps_cfg)
# call pywps application with processes filtered according to the adapter's definition
process_store = get_db(app).get_store(StoreProcesses)
processes_wps = [
process.wps() for process in process_store.list_processes(
visibility=VISIBILITY_PUBLIC)
]
service = WorkerService(processes_wps,
is_worker=is_worker,
settings=settings)
except Exception as ex:
LOGGER.exception(
"Error occurred during PyWPS Service and/or Processes setup.")
raise OWSNoApplicableCode(
"Failed setup of PyWPS Service and/or Processes. Error [{!r}]".
format(ex))
return service
def main(global_config, **settings):
"""
Creates a Pyramid WSGI application for Weaver.
"""
setup_loggers(settings)
LOGGER.info("Initiating weaver application")
# validate and fix configuration
weaver_config = get_weaver_configuration(settings)
settings.update({"weaver.configuration": weaver_config})
# Parse extra_options and add each of them in the settings dict
LOGGER.info("Parsing extra options...")
settings.update(
parse_extra_options(settings.get("weaver.extra_options", "")))
# load requests options if found, otherwise skip
LOGGER.info("Checking for request options file...")
req_file = get_weaver_config_file(settings.get("weaver.request_options",
""),
WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG,
generate_default_from_example=False)
if req_file:
LOGGER.info("Loading request options...")
with open(req_file, "r") as f:
settings.update({"weaver.request_options": yaml.safe_load(f)})
else:
LOGGER.warning("No request options found.")
# add default caching regions if they were omitted in config file
if settings.get("weaver.celery", False):
LOGGER.info("Celery runner detected. Skipping cache options setup.")
else:
LOGGER.info("Adding default caching options...")
setup_cache(settings)
LOGGER.info("Setup celery configuration...")
local_config = Configurator(settings=settings)
if global_config.get("__file__") is not None:
local_config.include("pyramid_celery")
local_config.configure_celery(global_config["__file__"])
local_config.include("weaver")
LOGGER.info("Running database migration...")
db = get_db(settings)
db.run_migration()
if settings.get("weaver.celery", False):
LOGGER.info("Celery runner detected. Skipping process registration.")
else:
LOGGER.info("Registering builtin processes...")
register_builtin_processes(local_config)
LOGGER.info("Registering WPS-1 processes from configuration file...")
wps_processes_file = get_settings(local_config).get(
"weaver.wps_processes_file")
register_wps_processes_from_config(wps_processes_file, local_config)
return local_config.make_wsgi_app()
def add_provider(request):
# type: (PyramidRequest) -> AnyViewResponse
"""
Register a new service provider.
"""
schema = sd.CreateProviderRequestBody()
schema_ref = get_schema_ref(schema, request)
try:
body = schema.deserialize(request.json)
except colander.Invalid as invalid:
data = {
"description": f"Invalid schema: [{invalid!s}]",
"value": invalid.value
}
data.update(schema_ref)
raise HTTPBadRequest(json=data)
store = get_db(request).get_store(StoreServices)
prov_id = get_any_id(body)
try:
store.fetch_by_name(prov_id)
except ServiceNotFound:
pass
else:
raise HTTPConflict(f"Provider [{prov_id}] already exists.")
try:
new_service = Service(url=body["url"], name=prov_id)
except KeyError as exc:
raise OWSMissingParameterValue(f"Missing JSON parameter '{exc!s}'.",
value=exc)
if "public" in body:
new_service["public"] = body["public"]
if "auth" in body:
new_service["auth"] = body["auth"]
try:
# validate that metadata or any pre-fetch operation can be resolved
service = new_service.summary(request, fetch=True, ignore=False)
if not service:
raise colander.Invalid(None, value=body)
store.save_service(new_service)
except NotImplementedError: # raised when supported service types / conversion
raise OWSNotImplemented(
sd.NotImplementedPostProviderResponse.description,
value=new_service)
except ServiceParsingError: # derives from HTTPUnprocessableEntity with relevant error message
raise
except colander.Invalid as invalid:
data = {
"description":
"Provider properties could not be parsed correctly.",
"value": invalid.value
}
data.update(schema_ref)
raise HTTPUnprocessableEntity(json=data)
data = get_schema_ref(sd.ProviderSummarySchema, request)
data.update(service)
return HTTPCreated(json=data)
def get_bill_list(request):
# type: (PyramidRequest) -> AnyViewResponse
"""
Get list of bills IDs.
"""
store = get_db(request).get_store(StoreBills)
bills = store.list_bills()
return HTTPOk(json={"bills": [b.id for b in bills]})
def dismiss_job_task(job, container):
# type: (Job, AnySettingsContainer) -> Job
"""
Cancels any pending or running :mod:`Celery` task and removes completed job artifacts.
.. note::
The :term:`Job` object itself is not deleted, only its artifacts.
Therefore, its inputs, outputs, logs, exceptions, etc. are still available in the database,
but corresponding files that would be exposed by ``weaver.wps_output`` configurations are removed.
:param job: Job to cancel or cleanup.
:param container: Application settings.
:return: Updated and dismissed job.
"""
raise_job_dismissed(job, container)
if job.status in JOB_STATUS_CATEGORIES[StatusCategory.RUNNING]:
# signal to stop celery task. Up to it to terminate remote if any.
LOGGER.debug("Job [%s] dismiss operation: Canceling task [%s]", job.id,
job.task_id)
celery_app.control.revoke(job.task_id, terminate=True)
wps_out_dir = get_wps_output_dir(container)
job_out_dir = os.path.join(wps_out_dir, str(job.id))
job_out_log = os.path.join(wps_out_dir, str(job.id) + ".log")
job_out_xml = os.path.join(wps_out_dir, str(job.id) + ".xml")
if os.path.isdir(job_out_dir):
LOGGER.debug("Job [%s] dismiss operation: Removing output results.",
job.id)
shutil.rmtree(
job_out_dir,
onerror=lambda func, path, _exc: LOGGER.warning(
"Job [%s] dismiss operation: Failed to delete [%s] due to [%s]",
job.id, job_out_dir, _exc))
if os.path.isfile(job_out_log):
LOGGER.debug("Job [%s] dismiss operation: Removing output logs.",
job.id)
try:
os.remove(job_out_log)
except OSError as exc:
LOGGER.warning(
"Job [%s] dismiss operation: Failed to delete [%s] due to [%s]",
job.id, job_out_log, exc)
if os.path.isfile(job_out_xml):
LOGGER.debug("Job [%s] dismiss operation: Removing output WPS status.",
job.id)
try:
os.remove(job_out_xml)
except OSError as exc:
LOGGER.warning(
"Job [%s] dismiss operation: Failed to delete [%s] due to [%s]",
job.id, job_out_xml, exc)
LOGGER.debug("Job [%s] dismiss operation: Updating job status.")
store = get_db(container).get_store(StoreJobs)
job.status_message = f"Job {Status.DISMISSED}."
job.status = map_status(Status.DISMISSED)
job = store.update_job(job)
return job
def _validate_process(self):
try:
registry = get_registry()
store = get_db(registry).get_store(StoreProcesses)
process = store.fetch_by_id(self.process) # raise if not found
except (ProcessNotAccessible, ProcessNotFound):
raise PackageNotFound("Cannot find '{}' package for process '{}'".format(PROCESS_BUILTIN, self.process))
if process.type != PROCESS_BUILTIN:
raise PackageExecutionError("Invalid package is not of type '{}'".format(PROCESS_BUILTIN))
def submit_provider_job(request):
"""
Execute a remote provider process.
"""
store = get_db(request).get_store(StoreServices)
provider_id = request.matchdict.get("provider_id")
service = store.fetch_by_name(provider_id)
body = submit_job(request, service, tags=["wps-rest"])
return get_job_submission_response(body)
def get_provider_processes(request):
"""
Retrieve available provider processes (GetCapabilities).
"""
provider_id = request.matchdict.get("provider_id")
store = get_db(request).get_store(StoreServices)
service = store.fetch_by_name(provider_id)
processes = list_remote_processes(service, request)
return HTTPOk(json={"processes": [p.process_summary() for p in processes]})
def get_bill_info(request):
"""
Get bill information.
"""
bill_id = request.matchdict.get("bill_id")
store = get_db(request).get_store(StoreBills)
try:
bill = store.fetch_by_id(bill_id)
except BillNotFound:
raise HTTPNotFound("Could not find bill with specified 'bill_id'.")
return HTTPOk(json={"bill": bill.json()})
def get_quote_info(request):
"""
Get quote information.
"""
quote_id = request.matchdict.get("quote_id")
store = get_db(request).get_store(StoreQuotes)
try:
quote = store.fetch_by_id(quote_id)
except QuoteNotFound:
raise HTTPNotFound("Could not find quote with specified 'quote_id'.")
return HTTPOk(json={"quote": quote.json()})
def get_service(request):
"""
Get the request service using provider_id from the service store.
"""
store = get_db(request).get_store(StoreServices)
provider_id = request.matchdict.get("provider_id")
try:
service = store.fetch_by_name(provider_id)
except ServiceNotFound:
raise HTTPNotFound("Provider {0} cannot be found.".format(provider_id))
return service, store
def submit_provider_job(request):
# type: (PyramidRequest) -> AnyViewResponse
"""
Execute a remote provider process.
"""
from weaver.processes.execution import submit_job # isort:skip # noqa: E402 # pylint: disable=C0413
store = get_db(request).get_store(StoreServices)
provider_id = request.matchdict.get("provider_id")
service = store.fetch_by_name(provider_id)
return submit_job(request, service, tags=["wps-rest"])
def setUpClass(cls):
config = setup_config_with_mongodb(settings=cls.settings)
config = setup_config_with_pywps(config)
config = setup_config_with_celery(config)
config = get_test_weaver_config(config)
cls.process_store = setup_mongodb_processstore(config) # force reset
cls.job_store = setup_mongodb_jobstore(config)
cls.app = get_test_weaver_app(config=config, settings=cls.settings)
cls.db = get_db(config)
cls.config = config
cls.settings.update(cls.config.registry.settings) # back propagate changes
def setup_mongodb_processstore(config=None):
# type: (Optional[Configurator]) -> MongodbProcessStore
"""Setup store using mongodb, will be enforced if not configured properly."""
config = setup_config_with_mongodb(config)
db = get_db(config)
store = db.get_store(MongodbProcessStore)
store.clear_processes()
# store must be recreated after clear because processes are added automatically on __init__
db.reset_store(MongodbProcessStore.type)
store = db.get_store(MongodbProcessStore)
return store
def dismiss_job_task(job, container):
# type: (Job, AnySettingsContainer) -> Job
"""
Cancels any pending or running :mod:`Celery` task and removes completed job artifacts.
:param job: job to cancel or cleanup.
:param container:
:return:
"""
raise_job_dismissed(job, container)
if job.status in status.JOB_STATUS_CATEGORIES[
status.JOB_STATUS_CATEGORY_RUNNING]:
# signal to stop celery task. Up to it to terminate remote if any.
LOGGER.debug("Job [%s] dismiss operation: Canceling task [%s]", job.id,
job.task_id)
celery_app.control.revoke(job.task_id, terminate=True)
wps_out_dir = get_wps_output_dir(container)
job_out_dir = os.path.join(wps_out_dir, str(job.id))
job_out_log = os.path.join(wps_out_dir, str(job.id) + ".log")
job_out_xml = os.path.join(wps_out_dir, str(job.id) + ".xml")
if os.path.isdir(job_out_dir):
LOGGER.debug("Job [%s] dismiss operation: Removing output results.",
job.id)
shutil.rmtree(
job_out_dir,
onerror=lambda func, path, _exc: LOGGER.warning(
"Job [%s] dismiss operation: Failed to delete [%s] due to [%s]",
job.id, job_out_dir, _exc))
if os.path.isfile(job_out_log):
LOGGER.debug("Job [%s] dismiss operation: Removing output logs.",
job.id)
try:
os.remove(job_out_log)
except OSError as exc:
LOGGER.warning(
"Job [%s] dismiss operation: Failed to delete [%s] due to [%s]",
job.id, job_out_log, exc)
if os.path.isfile(job_out_xml):
LOGGER.debug("Job [%s] dismiss operation: Removing output WPS status.",
job.id)
try:
os.remove(job_out_xml)
except OSError as exc:
LOGGER.warning(
"Job [%s] dismiss operation: Failed to delete [%s] due to [%s]",
job.id, job_out_xml, exc)
LOGGER.debug("Job [%s] dismiss operation: Updating job status.")
store = get_db(container).get_store(StoreJobs)
job.status_message = "Job {}.".format(status.STATUS_DISMISSED)
job.status = status.map_status(status.STATUS_DISMISSED)
job = store.update_job(job)
return job
def get_service(request):
# type: (AnyRequestType) -> Tuple[Service, StoreServices]
"""
Get the request service using provider_id from the service store.
"""
store = get_db(request).get_store(StoreServices)
provider_id = request.matchdict.get("provider_id")
try:
service = store.fetch_by_name(provider_id)
except ServiceNotFound:
raise HTTPNotFound(f"Provider {provider_id} cannot be found.")
return service, store
def get_provider_processes(request):
"""
Retrieve available provider processes (GetCapabilities).
"""
detail = asbool(request.params.get("detail", True))
provider_id = request.matchdict.get("provider_id")
store = get_db(request).get_store(StoreServices)
service = store.fetch_by_name(provider_id)
processes = service.processes(request)
processes = [p.summary() if detail else p.id for p in processes]
links = get_process_list_links(request, paging={}, total=None, provider=service)
return HTTPOk(json={"processes": processes, "links": links})