def from_ows(service, process, container):
# type: (Service, ProcessWPS, AnySettingsContainer) -> Process
"""
Converts a :mod:`owslib.wps` Process to local storage :class:`weaver.datatype.Process`.
"""
wps_url = get_wps_restapi_base_url(container)
if wps_url == service.url:
provider_url = wps_url # local weaver process, using WPS-XML endpoint
else:
provider_url = "{}/providers/{}".format(wps_url, service.get("name"))
describe_process_url = "{}/processes/{}".format(provider_url, process.identifier)
execute_process_url = "{describe_url}/jobs".format(describe_url=describe_process_url)
# FIXME: should use common function
inputs = [ows2json_io_FIXME(ows_input) for ows_input in get_field(process, "dataInputs", default=[])]
outputs = [ows2json_io(ows_output) for ows_output in get_field(process, "processOutputs", default=[])]
return Process(
id=process.identifier,
title=get_field(process, "title", default=process.identifier, search_variations=True),
abstract=get_field(process, "abstract", default=None, search_variations=True),
inputs=inputs,
outputs=outputs,
url=describe_process_url,
processEndpointWPS1=service.get("url"),
processDescriptionURL=describe_process_url,
executeEndpoint=execute_process_url,
package=None,
)
def test_execute_rest_xml_post_resp_json(self):
"""
Test :term:`XML` content using :term:`WPS` format submitted to REST endpoint gets redirected automatically.
"""
base = get_wps_restapi_base_url(self.settings)
url = f"{base}/processes/{self.process_id}/execution"
self.wps_execute("2.0.0", ContentType.APP_JSON, url=url)
def get_providers(request):
"""
Lists registered providers.
"""
store = get_db(request).get_store(StoreServices)
providers = []
for service in store.list_services():
try:
if service.type.lower() != "wps":
continue
wps = WebProcessingService(url=service.url,
headers=get_cookie_headers(
request.headers))
providers.append(
dict(id=service.name,
title=getattr(wps.identification, "title", ""),
abstract=getattr(wps.identification, "abstract", ""),
url="{base_url}/providers/{provider_id}".format(
base_url=get_wps_restapi_base_url(
get_settings(request)),
provider_id=service.name),
public=service.public))
except Exception as exc:
warnings.warn(
"Exception occurred while fetching wps {0} : {1!r}".format(
service.url, exc), NonBreakingExceptionWarning)
return HTTPOk(json={"providers": providers})
def submit_job_handler(payload, # type: JSON
settings, # type: SettingsType
service_url, # type: str
provider_id=None, # type: Optional[str]
process_id=None, # type: str
is_workflow=False, # type: bool
is_local=True, # type: bool
visibility=None, # type: Optional[str]
language=None, # type: Optional[str]
auth=None, # type: Optional[HeaderCookiesType]
tags=None, # type: Optional[List[str]]
user=None, # type: Optional[int]
context=None, # type: Optional[str]
): # type: (...) -> JSON
"""
Submits the job to the Celery worker with provided parameters.
Assumes that parameters have been pre-fetched and validated, except for the input payload.
"""
try:
json_body = sd.Execute().deserialize(payload)
except colander.Invalid as ex:
raise HTTPBadRequest("Invalid schema: [{}]".format(str(ex)))
# TODO: remove when all parameter variations are supported
# FIXME:
# - support 'sync' and 'Prefer' header variants (https://github.com/crim-ca/weaver/issues/247)
# - support 'response: raw' (https://github.com/crim-ca/weaver/issues/376)
# - allow omitting 'outputs' (https://github.com/crim-ca/weaver/issues/375)
_validate_job_parameters(json_body)
is_execute_async = json_body["mode"] != EXECUTE_MODE_SYNC # convert auto to async
notification_email = json_body.get("notification_email")
encrypted_email = encrypt_email(notification_email, settings) if notification_email else None
store = get_db(settings).get_store(StoreJobs)
job = store.save_job(task_id=STATUS_ACCEPTED, process=process_id, service=provider_id,
inputs=json_body.get("inputs"), is_local=is_local, is_workflow=is_workflow,
access=visibility, user_id=user, execute_async=is_execute_async, custom_tags=tags,
notification_email=encrypted_email, accept_language=language, context=context)
job.save_log(logger=LOGGER, message="Job task submitted for execution.", status=STATUS_ACCEPTED, progress=0)
job = store.update_job(job)
result = execute_process.delay(job_id=job.id, wps_url=clean_ows_url(service_url), headers=auth)
LOGGER.debug("Celery pending task [%s] for job [%s].", result.id, job.id)
# local/provider process location
location_base = "/providers/{provider_id}".format(provider_id=provider_id) if provider_id else ""
location = "{base_url}{location_base}/processes/{process_id}/jobs/{job_id}".format(
base_url=get_wps_restapi_base_url(settings),
location_base=location_base,
process_id=process_id,
job_id=job.id)
body_data = {
"jobID": job.id,
"processID": job.process,
"providerID": provider_id, # dropped by validator if not applicable
"status": map_status(STATUS_ACCEPTED),
"location": location
}
return body_data
def get_capabilities(service, request):
"""
GetCapabilities of a wps provider.
"""
wps = WebProcessingService(url=service.url,
headers=get_cookie_headers(request.headers))
settings = get_settings(request)
return dict(
id=service.name,
title=wps.identification.title,
abstract=wps.identification.abstract,
url="{base_url}/providers/{provider_id}".format(
base_url=get_wps_restapi_base_url(settings),
provider_id=service.name),
processes="{base_url}/providers/{provider_id}/processes".format(
base_url=get_wps_restapi_base_url(settings),
provider_id=service.name),
type=PROCESS_WPS,
contact=wps.provider.contact.name)
def api_frontpage(request):
"""Frontpage of weaver."""
# import here to avoid circular import errors
from weaver.config import get_weaver_configuration
settings = get_settings(request)
weaver_url = get_weaver_url(settings)
weaver_config = get_weaver_configuration(settings)
weaver_api = asbool(settings.get("weaver.wps_restapi"))
weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None
weaver_api_def = weaver_api_url + sd.api_swagger_ui_uri if weaver_api else None
weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None
weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None
weaver_wps = asbool(settings.get("weaver.wps"))
weaver_wps_url = get_wps_url(settings) if weaver_wps else None
weaver_conform_url = weaver_url + sd.api_conformance_uri
weaver_process_url = weaver_url + sd.processes_uri
weaver_links = [
{"href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This document"},
{"href": weaver_conform_url, "rel": "conformance", "type": CONTENT_TYPE_APP_JSON,
"title": "WPS 2.0/3.0 REST-JSON Binding Extension conformance classes implemented by this service."},
]
if weaver_api_def:
weaver_links.append({"href": weaver_api_def, "rel": "service", "type": CONTENT_TYPE_APP_JSON,
"title": "API definition of this service."})
if isinstance(weaver_api_doc, str):
if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test
ext_type = weaver_api_doc.split(".")[-1]
doc_type = "application/{}".format(ext_type)
else:
doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type
weaver_links.append({"href": weaver_api_doc, "rel": "documentation", "type": doc_type,
"title": "API documentation about this service."})
if weaver_api_ref:
weaver_links.append({"href": weaver_api_ref, "rel": "reference", "type": CONTENT_TYPE_APP_JSON,
"title": "API reference specification of this service."})
weaver_links.append({"href": weaver_process_url, "rel": "processes", "type": CONTENT_TYPE_APP_JSON,
"title": "Processes offered by this service."})
return {
"message": "Weaver Information",
"configuration": weaver_config,
"parameters": [
{"name": "api", "enabled": weaver_api,
"url": weaver_api_url,
"doc": weaver_api_doc,
"api": weaver_api_def,
"ref": weaver_api_ref},
{"name": "wps", "enabled": weaver_wps,
"url": weaver_wps_url},
],
"links": weaver_links,
}
def setUpClass(cls):
# disable SSL warnings from logs
try:
import urllib3 # noqa
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
except ImportError:
pass
# logging parameter overrides
cls.logger_level = os.getenv("WEAVER_TEST_LOGGER_LEVEL", cls.logger_level)
cls.logger_enabled = asbool(os.getenv("WEAVER_TEST_LOGGER_ENABLED", cls.logger_enabled))
cls.logger_result_dir = os.getenv("WEAVER_TEST_LOGGER_RESULT_DIR", os.path.join(WEAVER_ROOT_DIR))
cls.logger_json_indent = os.getenv("WEAVER_TEST_LOGGER_JSON_INDENT", cls.logger_json_indent)
cls.logger_field_indent = os.getenv("WEAVER_TEST_LOGGER_FIELD_INDENT", cls.logger_field_indent)
cls.logger_separator_calls = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_CALLS", cls.logger_separator_calls)
cls.logger_separator_steps = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_STEPS", cls.logger_separator_steps)
cls.logger_separator_tests = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_TESTS", cls.logger_separator_tests)
cls.logger_separator_cases = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_CASES", cls.logger_separator_cases)
cls.setup_logger()
cls.log("{}Start of '{}': {}\n{}"
.format(cls.logger_separator_cases, cls.current_case_name(), now(), cls.logger_separator_cases))
# test execution configs
cls.WEAVER_TEST_REQUEST_TIMEOUT = int(os.getenv("WEAVER_TEST_REQUEST_TIMEOUT", 10))
cls.WEAVER_TEST_JOB_ACCEPTED_MAX_TIMEOUT = int(os.getenv("WEAVER_TEST_JOB_ACCEPTED_MAX_TIMEOUT", 30))
cls.WEAVER_TEST_JOB_RUNNING_MAX_TIMEOUT = int(os.getenv("WEAVER_TEST_JOB_RUNNING_MAX_TIMEOUT", 6000))
cls.WEAVER_TEST_JOB_GET_STATUS_INTERVAL = int(os.getenv("WEAVER_TEST_JOB_GET_STATUS_INTERVAL", 5))
# security configs if enabled
cls.WEAVER_TEST_PROTECTED_ENABLED = asbool(os.getenv("WEAVER_TEST_PROTECTED_ENABLED", False))
cls.WEAVER_TEST_WSO2_CLIENT_HOSTNAME = os.getenv("WEAVER_TEST_WSO2_CLIENT_HOSTNAME", "")
cls.WEAVER_TEST_WSO2_CLIENT_ID = os.getenv("WEAVER_TEST_WSO2_CLIENT_ID", "")
cls.WEAVER_TEST_WSO2_CLIENT_SECRET = os.getenv("WEAVER_TEST_WSO2_CLIENT_SECRET", "")
cls.WEAVER_TEST_WSO2_URL = os.getenv("WEAVER_TEST_WSO2_URL", "")
cls.WEAVER_TEST_MAGPIE_URL = os.getenv("WEAVER_TEST_MAGPIE_URL", "")
cls.WEAVER_TEST_ADMIN_CREDENTIALS = {"username": get_setting("ADMIN_USERNAME", cls.app),
"password": get_setting("ADMIN_PASSWORD", cls.app)}
cls.WEAVER_TEST_ALICE_CREDENTIALS = {"username": get_setting("ALICE_USERNAME", cls.app),
"password": get_setting("ALICE_PASSWORD", cls.app)}
cls.WEAVER_TEST_BOB_CREDENTIALS = {"username": get_setting("BOD_USERNAME", cls.app),
"password": get_setting("BOB_PASSWORD", cls.app)}
# server settings
cls.WEAVER_TEST_SERVER_HOSTNAME = os.getenv("WEAVER_TEST_SERVER_HOSTNAME")
cls.WEAVER_TEST_SERVER_BASE_PATH = os.getenv("WEAVER_TEST_SERVER_BASE_PATH", "/weaver")
cls.WEAVER_TEST_SERVER_API_PATH = os.getenv("WEAVER_TEST_SERVER_API_PATH", "/")
cls.WEAVER_TEST_CONFIG_INI_PATH = os.getenv("WEAVER_TEST_CONFIG_INI_PATH") # none uses default path
cls.app = WebTestApp(cls.WEAVER_TEST_SERVER_HOSTNAME)
cls.WEAVER_URL = get_weaver_url(cls.settings())
cls.WEAVER_RESTAPI_URL = get_wps_restapi_base_url(cls.settings())
# validation
cls.validate_test_server()
cls.setup_test_processes()
def retrieve_data_source_url(data_source):
# type: (Optional[Text]) -> str
"""
Finds the data source URL using the provided data source identifier.
:returns: found URL, 'default' data source if not found, or current weaver WPS Rest API base URL if `None`.
"""
if data_source is None:
# get local data source
return get_wps_restapi_base_url(get_settings())
data_sources = fetch_data_sources()
return data_sources[data_source if data_source in data_sources else get_default_data_source(data_sources)]["ades"]
def register_builtin_processes(container):
# type: (AnySettingsContainer) -> None
"""
Registers every ``builtin`` CWL package to the processes database.
CWL definitions must be located within the :mod:`weaver.processes.builtin` module.
"""
restapi_url = get_wps_restapi_base_url(container)
builtin_apps_mapping = _get_builtin_reference_mapping(os.path.abspath(os.path.dirname(__file__)))
builtin_processes = []
for process_id, process_path in builtin_apps_mapping.items():
process_info = get_process_definition({}, package=None, reference=process_path)
process_url = "/".join([restapi_url, "processes", process_id])
process_package = _get_builtin_package(process_id, process_info["package"])
process_abstract = _get_builtin_metadata(process_id, process_path, "__doc__", clean=True)
process_version = _get_builtin_metadata(process_id, process_path, "__version__")
process_title = _get_builtin_metadata(process_id, process_path, "__title__")
process_payload = {
"processDescription": {
"process": {
"id": process_id,
"type": PROCESS_BUILTIN,
"title": process_title,
"version": process_version,
"abstract": process_abstract,
}
},
"deploymentProfileName": "http://www.opengis.net/profiles/eoc/builtinApplication",
"executionUnit": [{"unit": process_package}],
}
process_payload["processDescription"]["process"].update(ows_context_href(process_url))
builtin_processes.append(Process(
id=process_id,
type=PROCESS_BUILTIN,
title=process_title,
version=process_version,
abstract=process_abstract,
payload=process_payload,
package=process_package,
inputs=process_info["inputs"],
outputs=process_info["outputs"],
processDescriptionURL=process_url,
processEndpointWPS1=get_wps_url(container),
executeEndpoint="/".join([process_url, "jobs"]),
visibility=VISIBILITY_PUBLIC,
))
# registration of missing/updated apps automatically applied with 'default_processes'
get_db(container).get_store(StoreProcesses, default_processes=builtin_processes)
def api_frontpage_body(settings):
# type: (SettingsType) -> JSON
"""
Generates the JSON body describing the Weaver API and documentation references.
"""
# import here to avoid circular import errors
from weaver.config import get_weaver_configuration
weaver_url = get_weaver_url(settings)
weaver_config = get_weaver_configuration(settings)
weaver_api = asbool(settings.get("weaver.wps_restapi"))
weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None
weaver_api_oas_ui = weaver_api_url + sd.api_openapi_ui_service.path if weaver_api else None
weaver_api_swagger = weaver_api_url + sd.api_swagger_ui_service.path if weaver_api else None
weaver_api_doc = settings.get("weaver.wps_restapi_doc",
None) if weaver_api else None
weaver_api_ref = settings.get("weaver.wps_restapi_ref",
None) if weaver_api else None
weaver_api_spec = weaver_api_url + sd.openapi_json_service.path if weaver_api else None
weaver_wps = asbool(settings.get("weaver.wps"))
weaver_wps_url = get_wps_url(settings) if weaver_wps else None
weaver_conform_url = weaver_url + sd.api_conformance_service.path
weaver_process_url = weaver_url + sd.processes_service.path
weaver_jobs_url = weaver_url + sd.jobs_service.path
weaver_links = [{
"href": weaver_url,
"rel": "self",
"type": CONTENT_TYPE_APP_JSON,
"title": "This landing page."
}, {
"href": weaver_conform_url,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/conformance",
"type": CONTENT_TYPE_APP_JSON,
"title": "Conformance classes implemented by this service."
}, {
"href": __meta__.__license_url__,
"rel": "license",
"type": CONTENT_TYPE_TEXT_PLAIN,
"title": __meta__.__license_long__
}]
if weaver_api:
weaver_links.extend([
{
"href": weaver_api_url,
"rel": "service",
"type": CONTENT_TYPE_APP_JSON,
"title": "WPS REST API endpoint of this service."
},
{
"href": weaver_api_spec,
"rel": "service-desc",
"type": CONTENT_TYPE_APP_JSON,
"title": "OpenAPI specification of this service."
},
{
"href": weaver_api_oas_ui,
"rel": "service-doc",
"type": CONTENT_TYPE_TEXT_HTML,
"title":
"Human readable OpenAPI documentation of this service."
},
{
"href": weaver_api_spec,
"rel": "OpenAPI",
"type": CONTENT_TYPE_APP_JSON,
"title": "OpenAPI specification of this service."
},
{
"href": weaver_api_swagger,
"rel": "swagger-ui",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS REST API definition of this service."
},
{
"href": weaver_process_url,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/processes",
"type": CONTENT_TYPE_APP_JSON,
"title": "Processes offered by this service."
},
{
"href": sd.OGC_API_REPO_URL,
"rel": "ogcapi-processes-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "OGC API - Processes schema definitions repository."
},
{
"href":
weaver_jobs_url,
"rel":
"http://www.opengis.net/def/rel/ogc/1.0/job-list",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Job search and listing endpoint of executions registered under this service."
},
{
"href": sd.CWL_BASE_URL,
"rel": "cwl-home",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) homepage."
},
{
"href": sd.CWL_REPO_URL,
"rel": "cwl-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) repositories."
},
{
"href": sd.CWL_SPEC_URL,
"rel": "cwl-specification",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) specification."
},
{
"href": sd.CWL_USER_GUIDE_URL,
"rel": "cwl-user-guide",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) user guide."
},
{
"href":
sd.CWL_CMD_TOOL_URL,
"rel":
"cwl-command-line-tool",
"type":
CONTENT_TYPE_TEXT_HTML,
"title":
"Common Workflow Language (CWL) CommandLineTool specification."
},
{
"href": sd.CWL_WORKFLOW_URL,
"rel": "cwl-workflow",
"type": CONTENT_TYPE_TEXT_HTML,
"title":
"Common Workflow Language (CWL) Workflow specification."
},
])
if weaver_api_ref:
# sample:
# https://app.swaggerhub.com/apis/geoprocessing/WPS/
weaver_links.append({
"href":
weaver_api_ref,
"rel":
"reference",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"API reference specification of this service."
})
if isinstance(weaver_api_doc, str):
# sample:
# https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/develop/docs/18-062.pdf
if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test
ext_type = weaver_api_doc.split(".")[-1]
doc_type = "application/{}".format(ext_type)
else:
doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type
weaver_links.append({
"href":
weaver_api_doc,
"rel":
"documentation",
"type":
doc_type,
"title":
"API reference documentation about this service."
})
else:
weaver_links.append({
"href":
__meta__.__documentation_url__,
"rel":
"documentation",
"type":
CONTENT_TYPE_TEXT_HTML,
"title":
"API reference documentation about this service."
})
if weaver_wps:
weaver_links.extend([
{
"href": weaver_wps_url,
"rel": "wps",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 1.0.0/2.0 XML endpoint of this service."
},
{
"href": "http://docs.opengeospatial.org/is/14-065/14-065.html",
"rel": "wps-specification",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS 1.0.0/2.0 definition of this service."
},
{
"href": "http://schemas.opengis.net/wps/",
"rel": "wps-schema-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS 1.0.0/2.0 XML schemas repository."
},
{
"href": "http://schemas.opengis.net/wps/1.0.0/wpsAll.xsd",
"rel": "wps-schema-1",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 1.0.0 XML validation schemas entrypoint."
},
{
"href": "http://schemas.opengis.net/wps/2.0/wps.xsd",
"rel": "wps-schema-2",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 2.0 XML validation schemas entrypoint."
},
])
return {
"message":
"Weaver Information",
"configuration":
weaver_config,
"description":
__meta__.__description__,
"parameters": [
{
"name": "api",
"enabled": weaver_api,
"url": weaver_api_url,
"api": weaver_api_oas_ui
},
{
"name": "wps",
"enabled": weaver_wps,
"url": weaver_wps_url
},
],
"links":
weaver_links,
}
def deploy_process_from_payload(payload, container):
# type: (JSON, AnyContainer) -> HTTPException
"""
Adds a :class:`weaver.datatype.Process` instance to storage using the provided JSON ``payload`` matching
:class:`weaver.wps_restapi.swagger_definitions.ProcessDescription`.
:returns: HTTPOk if the process registration was successful
:raises HTTPException: otherwise
"""
_check_deploy(payload)
# use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload
payload_copy = deepcopy(payload)
# validate identifier naming for unsupported characters
process_description = payload.get("processDescription")
process_info = process_description.get("process", {})
process_href = process_description.pop("href", None)
# retrieve CWL package definition, either via "href" (WPS-1/2), "owsContext" or "executionUnit" (package/reference)
deployment_profile_name = payload.get("deploymentProfileName", "").lower()
ows_context = process_info.pop("owsContext", None)
reference = None
package = None
if process_href:
reference = process_href # reference type handled downstream
elif isinstance(ows_context, dict):
offering = ows_context.get("offering")
if not isinstance(offering, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering'."
)
content = offering.get("content")
if not isinstance(content, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering.content'."
)
package = None
reference = content.get("href")
elif deployment_profile_name:
if not any(
deployment_profile_name.endswith(typ)
for typ in [PROCESS_APPLICATION, PROCESS_WORKFLOW]):
raise HTTPBadRequest(
"Invalid value for parameter 'deploymentProfileName'.")
execution_units = payload.get("executionUnit")
if not isinstance(execution_units, list):
raise HTTPUnprocessableEntity("Invalid parameter 'executionUnit'.")
for execution_unit in execution_units:
if not isinstance(execution_unit, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'executionUnit'.")
package = execution_unit.get("unit")
reference = execution_unit.get("href")
# stop on first package/reference found, simultaneous usage will raise during package retrieval
if package or reference:
break
else:
raise HTTPBadRequest(
"Missing one of required parameters [href, owsContext, deploymentProfileName]."
)
# obtain updated process information using WPS process offering, CWL/WPS reference or CWL package definition
process_info = _get_deploy_process_info(process_info, reference, package)
# validate process type against weaver configuration
settings = get_settings(container)
process_type = process_info["type"]
if process_type == PROCESS_WORKFLOW:
weaver_config = get_weaver_configuration(settings)
if weaver_config != WEAVER_CONFIGURATION_EMS:
raise HTTPBadRequest(
"Invalid [{0}] package deployment on [{1}].".format(
process_type, weaver_config))
restapi_url = get_wps_restapi_base_url(settings)
description_url = "/".join(
[restapi_url, "processes", process_info["identifier"]])
execute_endpoint = "/".join([description_url, "jobs"])
# ensure that required "processEndpointWPS1" in db is added,
# will be auto-fixed to localhost if not specified in body
process_info["processEndpointWPS1"] = process_description.get(
"processEndpointWPS1")
process_info["executeEndpoint"] = execute_endpoint
process_info["payload"] = payload_copy
process_info["jobControlOptions"] = process_description.get(
"jobControlOptions", [])
process_info["outputTransmission"] = process_description.get(
"outputTransmission", [])
process_info["processDescriptionURL"] = description_url
# insert the "resolved" context using details retrieved from "executionUnit"/"href" or directly with "owsContext"
if "owsContext" not in process_info and reference:
process_info["owsContext"] = {
"offering": {
"content": {
"href": str(reference)
}
}
}
elif isinstance(ows_context, dict):
process_info["owsContext"] = ows_context
try:
store = get_db(container).get_store(StoreProcesses)
saved_process = store.save_process(Process(process_info),
overwrite=False)
except ProcessRegistrationError as ex:
raise HTTPConflict(detail=str(ex))
except ValueError as ex:
# raised on invalid process name
raise HTTPBadRequest(detail=str(ex))
json_response = {
"processSummary": saved_process.process_summary(),
"deploymentDone": True
}
return HTTPOk(
json=json_response
) # FIXME: should be 201 (created), update swagger accordingly
def deploy_process_from_payload(payload, container, overwrite=False):
# type: (JSON, AnyContainer, bool) -> HTTPException
"""
Deploy the process after resolution of all references and validation of the parameters from payload definition.
Adds a :class:`weaver.datatype.Process` instance to storage using the provided JSON ``payload``
matching :class:`weaver.wps_restapi.swagger_definitions.ProcessDescription`.
:param payload: JSON payload that was specified during the process deployment request.
:param container: container to retrieve application settings.
:param overwrite: whether to allow override of an existing process definition if conflict occurs.
:returns: HTTPOk if the process registration was successful.
:raises HTTPException: for any invalid process deployment step.
"""
# use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload
payload_copy = deepcopy(payload)
payload = _check_deploy(payload)
# validate identifier naming for unsupported characters
process_description = payload.get("processDescription")
process_info = process_description.get("process", {})
process_href = process_description.pop("href", None)
# retrieve CWL package definition, either via "href" (WPS-1/2), "owsContext" or "executionUnit" (package/reference)
deployment_profile_name = payload.get("deploymentProfileName", "").lower()
ows_context = process_info.pop("owsContext", None)
reference = None
package = None
if process_href:
reference = process_href # reference type handled downstream
elif isinstance(ows_context, dict):
offering = ows_context.get("offering")
if not isinstance(offering, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering'."
)
content = offering.get("content")
if not isinstance(content, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering.content'."
)
package = None
reference = content.get("href")
elif deployment_profile_name:
if not any(
deployment_profile_name.endswith(typ)
for typ in [PROCESS_APPLICATION, PROCESS_WORKFLOW]):
raise HTTPBadRequest(
"Invalid value for parameter 'deploymentProfileName'.")
execution_units = payload.get("executionUnit")
if not isinstance(execution_units, list):
raise HTTPUnprocessableEntity("Invalid parameter 'executionUnit'.")
for execution_unit in execution_units:
if not isinstance(execution_unit, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'executionUnit'.")
package = execution_unit.get("unit")
reference = execution_unit.get("href")
# stop on first package/reference found, simultaneous usage will raise during package retrieval
if package or reference:
break
else:
raise HTTPBadRequest(
"Missing one of required parameters [href, owsContext, deploymentProfileName]."
)
if process_info.get("type", "") == PROCESS_BUILTIN:
raise HTTPBadRequest(
"Invalid process type resolved from package: [{0}]. Deployment of {0} process is not allowed."
.format(PROCESS_BUILTIN))
# update and validate process information using WPS process offering, CWL/WPS reference or CWL package definition
settings = get_settings(container)
headers = getattr(
container, "headers", {}
) # container is any request (as when called from API Deploy request)
process_info = _validate_deploy_process_info(process_info, reference,
package, settings, headers)
restapi_url = get_wps_restapi_base_url(settings)
description_url = "/".join(
[restapi_url, "processes", process_info["identifier"]])
execute_endpoint = "/".join([description_url, "jobs"])
# ensure that required "processEndpointWPS1" in db is added,
# will be auto-fixed to localhost if not specified in body
process_info["processEndpointWPS1"] = process_description.get(
"processEndpointWPS1")
process_info["executeEndpoint"] = execute_endpoint
process_info["payload"] = payload_copy
process_info["jobControlOptions"] = process_description.get(
"jobControlOptions", [])
process_info["outputTransmission"] = process_description.get(
"outputTransmission", [])
process_info["processDescriptionURL"] = description_url
# insert the "resolved" context using details retrieved from "executionUnit"/"href" or directly with "owsContext"
if "owsContext" not in process_info and reference:
process_info["owsContext"] = {
"offering": {
"content": {
"href": str(reference)
}
}
}
elif isinstance(ows_context, dict):
process_info["owsContext"] = ows_context
# bw-compat abstract/description (see: ProcessDeployment schema)
if "description" not in process_info or not process_info["description"]:
process_info["description"] = process_info.get("abstract", "")
# FIXME: handle colander invalid directly in tween (https://github.com/crim-ca/weaver/issues/112)
try:
store = get_db(container).get_store(StoreProcesses)
process = Process(process_info)
sd.ProcessSummary().deserialize(
process) # make if fail before save if invalid
store.save_process(process, overwrite=overwrite)
process_summary = process.summary()
except ProcessRegistrationError as ex:
raise HTTPConflict(detail=str(ex))
except (ValueError, colander.Invalid) as ex:
# raised on invalid process name
raise HTTPBadRequest(detail=str(ex))
return HTTPCreated(
json={
"description": sd.OkPostProcessesResponse.description,
"processSummary": process_summary,
"deploymentDone": True
})
def _job_url(self, settings):
base_job_url = get_wps_restapi_base_url(settings)
if self.service is not None:
base_job_url += sd.provider_uri.format(provider_id=self.service)
job_path = sd.process_job_uri.format(process_id=self.process, job_id=self.id)
return "{base_job_url}{job_path}".format(base_job_url=base_job_url, job_path=job_path)
