def main(global_config, **settings):
"""
Creates a Pyramid WSGI application for Weaver.
"""
setup_loggers(settings)
LOGGER.info("Initiating weaver application")
# validate and fix configuration
weaver_config = get_weaver_configuration(settings)
settings.update({"weaver.configuration": weaver_config})
# Parse extra_options and add each of them in the settings dict
LOGGER.info("Parsing extra options...")
settings.update(
parse_extra_options(settings.get("weaver.extra_options", "")))
# load requests options if found, otherwise skip
LOGGER.info("Checking for request options file...")
req_file = get_weaver_config_file(settings.get("weaver.request_options",
""),
WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG,
generate_default_from_example=False)
if req_file:
LOGGER.info("Loading request options...")
with open(req_file, "r") as f:
settings.update({"weaver.request_options": yaml.safe_load(f)})
else:
LOGGER.warning("No request options found.")
# add default caching regions if they were omitted in config file
if settings.get("weaver.celery", False):
LOGGER.info("Celery runner detected. Skipping cache options setup.")
else:
LOGGER.info("Adding default caching options...")
setup_cache(settings)
LOGGER.info("Setup celery configuration...")
local_config = Configurator(settings=settings)
if global_config.get("__file__") is not None:
local_config.include("pyramid_celery")
local_config.configure_celery(global_config["__file__"])
local_config.include("weaver")
LOGGER.info("Running database migration...")
db = get_db(settings)
db.run_migration()
if settings.get("weaver.celery", False):
LOGGER.info("Celery runner detected. Skipping process registration.")
else:
LOGGER.info("Registering builtin processes...")
register_builtin_processes(local_config)
LOGGER.info("Registering WPS-1 processes from configuration file...")
wps_processes_file = get_settings(local_config).get(
"weaver.wps_processes_file")
register_wps_processes_from_config(wps_processes_file, local_config)
return local_config.make_wsgi_app()
def api_frontpage(request):
"""Frontpage of weaver."""
# import here to avoid circular import errors
from weaver.config import get_weaver_configuration
settings = get_settings(request)
weaver_url = get_weaver_url(settings)
weaver_config = get_weaver_configuration(settings)
weaver_api = asbool(settings.get("weaver.wps_restapi"))
weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None
weaver_api_def = weaver_api_url + sd.api_swagger_ui_uri if weaver_api else None
weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None
weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None
weaver_wps = asbool(settings.get("weaver.wps"))
weaver_wps_url = get_wps_url(settings) if weaver_wps else None
weaver_conform_url = weaver_url + sd.api_conformance_uri
weaver_process_url = weaver_url + sd.processes_uri
weaver_links = [
{"href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This document"},
{"href": weaver_conform_url, "rel": "conformance", "type": CONTENT_TYPE_APP_JSON,
"title": "WPS 2.0/3.0 REST-JSON Binding Extension conformance classes implemented by this service."},
]
if weaver_api_def:
weaver_links.append({"href": weaver_api_def, "rel": "service", "type": CONTENT_TYPE_APP_JSON,
"title": "API definition of this service."})
if isinstance(weaver_api_doc, str):
if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test
ext_type = weaver_api_doc.split(".")[-1]
doc_type = "application/{}".format(ext_type)
else:
doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type
weaver_links.append({"href": weaver_api_doc, "rel": "documentation", "type": doc_type,
"title": "API documentation about this service."})
if weaver_api_ref:
weaver_links.append({"href": weaver_api_ref, "rel": "reference", "type": CONTENT_TYPE_APP_JSON,
"title": "API reference specification of this service."})
weaver_links.append({"href": weaver_process_url, "rel": "processes", "type": CONTENT_TYPE_APP_JSON,
"title": "Processes offered by this service."})
return {
"message": "Weaver Information",
"configuration": weaver_config,
"parameters": [
{"name": "api", "enabled": weaver_api,
"url": weaver_api_url,
"doc": weaver_api_doc,
"api": weaver_api_def,
"ref": weaver_api_ref},
{"name": "wps", "enabled": weaver_wps,
"url": weaver_wps_url},
],
"links": weaver_links,
}
def forbid_local_only(container):
# type: (AnySettingsContainer) -> Any
"""
Raises an HTTP exception forbidding to resume the operation if invalid configuration is detected.
"""
config = get_weaver_configuration(container)
if config not in WEAVER_CONFIGURATIONS_REMOTE:
raise HTTPForbidden(json={
"description":
"Invalid provider operation on [{}] instance. "
"Processes requires unsupported remote execution.".format(config),
})
def get_processes(request):
"""
List registered processes (GetCapabilities). Optionally list both local and provider processes.
"""
detail = asbool(request.params.get("detail", True))
try:
# get local processes and filter according to schema validity
# (previously deployed process schemas can become invalid because of modified schema definitions
processes, invalid_processes = get_processes_filtered_by_valid_schemas(
request)
if invalid_processes:
raise HTTPServiceUnavailable(
"Previously deployed processes are causing invalid schema integrity errors. "
"Manual cleanup of following processes is required: {}".format(
invalid_processes))
response_body = {
"processes":
processes if detail else [get_any_id(p) for p in processes]
}
# if 'EMS' and '?providers=True', also fetch each provider's processes
settings = get_settings(request)
if get_weaver_configuration(settings) == WEAVER_CONFIGURATION_EMS:
queries = parse_request_query(request)
if "providers" in queries and asbool(
queries["providers"][0]) is True:
prov_url = "{host}/providers".format(host=request.host_url)
providers_response = request_extra("GET",
prov_url,
settings=settings,
headers=request.headers,
cookies=request.cookies)
providers = providers_response.json()
response_body.update({"providers": providers})
for i, provider in enumerate(providers):
provider_id = get_any_id(provider)
proc_url = "{host}/providers/{prov}/processes".format(
host=request.host_url, prov=provider_id)
response = request_extra("GET",
proc_url,
settings=settings,
headers=request.headers,
cookies=request.cookies)
processes = response.json().get("processes", [])
response_body["providers"][i].update({
"processes":
processes
if detail else [get_any_id(p) for p in processes]
})
return HTTPOk(json=response_body)
except colander.Invalid as ex:
raise HTTPBadRequest("Invalid schema: [{!s}]".format(ex))
def get_processes_filtered_by_valid_schemas(request):
# type: (PyramidRequest) -> Tuple[List[JSON], List[str], Dict[str, Optional[int]], bool, int]
"""
Validates the processes summary schemas and returns them into valid/invalid lists.
:returns: List of valid process and invalid processes IDs for manual cleanup, along with filtering parameters.
"""
settings = get_settings(request)
with_providers = False
if get_weaver_configuration(settings) in WeaverFeature.REMOTE:
with_providers = asbool(request.params.get("providers", False))
revisions_param = sd.ProcessRevisionsQuery(unknown="ignore").deserialize(
request.params)
with_revisions = revisions_param.get("revisions")
paging_query = sd.ProcessPagingQuery()
paging_value = {
param.name: param.default
for param in paging_query.children
}
paging_names = set(paging_value)
paging_param = paging_query.deserialize(request.params)
if with_providers and any(value != paging_value[param]
for param, value in paging_param.items()):
raise HTTPBadRequest(
json={
"description":
"Cannot combine paging/sorting parameters with providers full listing query.",
"error": "ListingInvalidParameter",
"value": list(paging_names.intersection(request.params))
})
store = get_db(request).get_store(StoreProcesses)
processes, total_local_processes = store.list_processes(
visibility=Visibility.PUBLIC,
total=True,
**revisions_param,
**paging_param)
valid_processes = []
invalid_processes_ids = []
for process in processes: # type: Process
try:
valid_processes.append(process.summary(revision=with_revisions))
except colander.Invalid as invalid:
process_ref = process.tag if with_revisions else process.identifier
LOGGER.debug("Invalid process [%s] because:\n%s", process_ref,
invalid)
invalid_processes_ids.append(process.identifier)
return valid_processes, invalid_processes_ids, paging_param, with_providers, total_local_processes
def _validate_deploy_process_info(process_info, reference, package, settings,
headers):
# type: (JSON, Optional[str], Optional[CWL], SettingsType, Optional[AnyHeadersContainer]) -> JSON
"""
Obtain the process definition from deploy payload with exception handling.
.. seealso::
- :func:`weaver.processes.wps_package.get_process_definition`
"""
from weaver.processes.wps_package import check_package_instance_compatible, get_process_definition
try:
# data_source `None` forces workflow process to search locally for deployed step applications
info = get_process_definition(process_info,
reference,
package,
data_source=None,
headers=headers)
# validate process type and package against weaver configuration
cfg = get_weaver_configuration(settings)
if cfg not in WEAVER_CONFIGURATIONS_REMOTE:
problem = check_package_instance_compatible(info["package"])
if problem:
raise HTTPForbidden(
json={
"description":
"Invalid process deployment of type [{}] on [{}] instance. "
"Remote execution is required but not supported.".
format(info["type"], cfg),
"cause":
problem
})
return info
except PackageNotFound as ex:
# raised when a workflow sub-process is not found (not deployed locally)
raise HTTPNotFound(detail=str(ex))
except InvalidIdentifierValue as ex:
raise HTTPBadRequest(str(ex))
except (PackageRegistrationError, PackageTypeError) as ex:
msg = "Invalid package/reference definition. Loading generated error: [{!s}]".format(
ex)
LOGGER.exception(msg)
raise HTTPUnprocessableEntity(detail=msg)
def main(global_config, **settings):
"""
Creates a Pyramid WSGI application for Weaver.
"""
setup_loggers(settings)
LOGGER.info("Initiating weaver application")
# validate and fix configuration
weaver_config = get_weaver_configuration(settings)
settings.update({"weaver.configuration": weaver_config})
# Parse extra_options and add each of them in the settings dict
settings.update(
parse_extra_options(settings.get("weaver.extra_options", "")))
# load requests options if found, otherwise skip
req_file = get_weaver_config_file(settings.get("weaver.request_options",
""),
WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG,
generate_default_from_example=False)
if req_file:
with open(req_file, "r") as f:
settings.update({"weaver.request_options": yaml.safe_load(f)})
local_config = Configurator(settings=settings)
if global_config.get("__file__") is not None:
local_config.include("pyramid_celery")
local_config.configure_celery(global_config["__file__"])
local_config.include("weaver")
LOGGER.info("Registering builtin processes...")
register_builtin_processes(local_config)
LOGGER.info("Registering WPS-1 processes from configuration file...")
wps_processes_file = get_settings(local_config).get(
"weaver.wps_processes_file")
register_wps_processes_from_config(wps_processes_file, local_config)
return local_config.make_wsgi_app()
def load_pywps_config(container, config=None):
# type: (AnySettingsContainer, Optional[Union[str, Dict[str, str]]]) -> ConfigParser
"""
Loads and updates the PyWPS configuration using Weaver settings.
"""
settings = get_settings(container)
if settings.get("weaver.wps_configured"):
LOGGER.debug("Using preloaded internal Weaver WPS configuration.")
return pywps_config.CONFIG
LOGGER.info("Initial load of internal Weaver WPS configuration.")
pywps_config.load_configuration([]) # load defaults
pywps_config.CONFIG.set("logging", "db_echo", "false")
if logging.getLevelName(pywps_config.CONFIG.get("logging",
"level")) <= logging.DEBUG:
pywps_config.CONFIG.set("logging", "level", "INFO")
# update metadata
LOGGER.debug("Updating WPS metadata configuration.")
for setting_name, setting_value in settings.items():
if setting_name.startswith("weaver.wps_metadata"):
pywps_setting = setting_name.replace("weaver.wps_metadata_", "")
pywps_config.CONFIG.set("metadata:main", pywps_setting,
setting_value)
# add weaver configuration keyword if not already provided
wps_keywords = pywps_config.CONFIG.get("metadata:main",
"identification_keywords")
weaver_mode = get_weaver_configuration(settings)
if weaver_mode not in wps_keywords:
wps_keywords += ("," if wps_keywords else "") + weaver_mode
pywps_config.CONFIG.set("metadata:main", "identification_keywords",
wps_keywords)
# add additional config passed as dictionary of {'section.key': 'value'}
if isinstance(config, dict):
for key, value in config.items():
section, key = key.split(".")
pywps_config.CONFIG.set(section, key, value)
# cleanup alternative dict "PYWPS_CFG" which is not expected elsewhere
if isinstance(settings.get("PYWPS_CFG"), dict):
del settings["PYWPS_CFG"]
# set accepted languages aligned with values provided by REST API endpoints
# otherwise, execute request could fail due to languages considered not supported
languages = ", ".join(AcceptLanguage.values())
LOGGER.debug("Setting WPS languages: [%s]", languages)
pywps_config.CONFIG.set("server", "language", languages)
LOGGER.debug("Updating WPS output configuration.")
# find output directory from app config or wps config
if "weaver.wps_output_dir" not in settings:
output_dir = pywps_config.get_config_value("server", "outputpath")
settings["weaver.wps_output_dir"] = output_dir
# ensure the output dir exists if specified
output_dir = get_wps_output_dir(settings)
make_dirs(output_dir, exist_ok=True)
# find output url from app config (path/url) or wps config (url only)
# note: needs to be configured even when using S3 bucket since XML status is provided locally
if "weaver.wps_output_url" not in settings:
output_path = settings.get("weaver.wps_output_path", "").rstrip("/")
if output_path and isinstance(output_path, str):
output_url = os.path.join(get_weaver_url(settings),
output_path.strip("/"))
else:
output_url = pywps_config.get_config_value("server", "outputurl")
settings["weaver.wps_output_url"] = output_url
# apply workdir if provided, otherwise use default
if "weaver.wps_workdir" in settings:
make_dirs(settings["weaver.wps_workdir"], exist_ok=True)
pywps_config.CONFIG.set("server", "workdir",
settings["weaver.wps_workdir"])
# configure S3 bucket if requested, storage of all process outputs
# note:
# credentials and default profile are picked up automatically by 'boto3' from local AWS configs or env vars
# region can also be picked from there unless explicitly provided by weaver config
# warning:
# if we set `(server, storagetype, s3)`, ALL status (including XML) are stored to S3
# to preserve status locally, we set 'file' and override the storage instance during output rewrite in WpsPackage
# we can still make use of the server configurations here to make this overridden storage auto-find its configs
s3_bucket = settings.get("weaver.wps_output_s3_bucket")
pywps_config.CONFIG.set("server", "storagetype", "file")
# pywps_config.CONFIG.set("server", "storagetype", "s3")
if s3_bucket:
LOGGER.debug("Updating WPS S3 bucket configuration.")
import boto3
from botocore.exceptions import ClientError
s3 = boto3.client("s3")
s3_region = settings.get("weaver.wps_output_s3_region",
s3.meta.region_name)
LOGGER.info(
"Validating that S3 [Bucket=%s, Region=%s] exists or creating it.",
s3_bucket, s3_region)
try:
s3.create_bucket(
Bucket=s3_bucket,
CreateBucketConfiguration={"LocationConstraint": s3_region})
LOGGER.info("S3 bucket for WPS output created.")
except ClientError as exc:
if exc.response.get("Error",
{}).get("Code") != "BucketAlreadyExists":
LOGGER.error("Failed setup of S3 bucket for WPS output: [%s]",
exc)
raise
LOGGER.info("S3 bucket for WPS output already exists.")
pywps_config.CONFIG.set("s3", "region", s3_region)
pywps_config.CONFIG.set("s3", "bucket", s3_bucket)
pywps_config.CONFIG.set(
"s3", "public",
"false") # don't automatically push results as publicly accessible
pywps_config.CONFIG.set(
"s3", "encrypt",
"true") # encrypts data server-side, transparent from this side
# enforce back resolved values onto PyWPS config
pywps_config.CONFIG.set("server", "setworkdir", "true")
pywps_config.CONFIG.set("server", "sethomedir", "true")
pywps_config.CONFIG.set("server", "outputpath",
settings["weaver.wps_output_dir"])
pywps_config.CONFIG.set("server", "outputurl",
settings["weaver.wps_output_url"])
pywps_config.CONFIG.set("server", "url", get_wps_url(settings, load=False))
settings["weaver.wps_configured"] = True
return pywps_config.CONFIG
def api_frontpage_body(settings):
# type: (SettingsType) -> JSON
"""
Generates the JSON body describing the Weaver API and documentation references.
"""
# import here to avoid circular import errors
from weaver.config import get_weaver_configuration
weaver_url = get_weaver_url(settings)
weaver_config = get_weaver_configuration(settings)
weaver_api = asbool(settings.get("weaver.wps_restapi"))
weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None
weaver_api_oas_ui = weaver_api_url + sd.api_openapi_ui_service.path if weaver_api else None
weaver_api_swagger = weaver_api_url + sd.api_swagger_ui_service.path if weaver_api else None
weaver_api_doc = settings.get("weaver.wps_restapi_doc",
None) if weaver_api else None
weaver_api_ref = settings.get("weaver.wps_restapi_ref",
None) if weaver_api else None
weaver_api_spec = weaver_api_url + sd.openapi_json_service.path if weaver_api else None
weaver_wps = asbool(settings.get("weaver.wps"))
weaver_wps_url = get_wps_url(settings) if weaver_wps else None
weaver_conform_url = weaver_url + sd.api_conformance_service.path
weaver_process_url = weaver_url + sd.processes_service.path
weaver_jobs_url = weaver_url + sd.jobs_service.path
weaver_links = [{
"href": weaver_url,
"rel": "self",
"type": CONTENT_TYPE_APP_JSON,
"title": "This landing page."
}, {
"href": weaver_conform_url,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/conformance",
"type": CONTENT_TYPE_APP_JSON,
"title": "Conformance classes implemented by this service."
}, {
"href": __meta__.__license_url__,
"rel": "license",
"type": CONTENT_TYPE_TEXT_PLAIN,
"title": __meta__.__license_long__
}]
if weaver_api:
weaver_links.extend([
{
"href": weaver_api_url,
"rel": "service",
"type": CONTENT_TYPE_APP_JSON,
"title": "WPS REST API endpoint of this service."
},
{
"href": weaver_api_spec,
"rel": "service-desc",
"type": CONTENT_TYPE_APP_JSON,
"title": "OpenAPI specification of this service."
},
{
"href": weaver_api_oas_ui,
"rel": "service-doc",
"type": CONTENT_TYPE_TEXT_HTML,
"title":
"Human readable OpenAPI documentation of this service."
},
{
"href": weaver_api_spec,
"rel": "OpenAPI",
"type": CONTENT_TYPE_APP_JSON,
"title": "OpenAPI specification of this service."
},
{
"href": weaver_api_swagger,
"rel": "swagger-ui",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS REST API definition of this service."
},
{
"href": weaver_process_url,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/processes",
"type": CONTENT_TYPE_APP_JSON,
"title": "Processes offered by this service."
},
{
"href": sd.OGC_API_REPO_URL,
"rel": "ogcapi-processes-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "OGC API - Processes schema definitions repository."
},
{
"href":
weaver_jobs_url,
"rel":
"http://www.opengis.net/def/rel/ogc/1.0/job-list",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Job search and listing endpoint of executions registered under this service."
},
{
"href": sd.CWL_BASE_URL,
"rel": "cwl-home",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) homepage."
},
{
"href": sd.CWL_REPO_URL,
"rel": "cwl-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) repositories."
},
{
"href": sd.CWL_SPEC_URL,
"rel": "cwl-specification",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) specification."
},
{
"href": sd.CWL_USER_GUIDE_URL,
"rel": "cwl-user-guide",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) user guide."
},
{
"href":
sd.CWL_CMD_TOOL_URL,
"rel":
"cwl-command-line-tool",
"type":
CONTENT_TYPE_TEXT_HTML,
"title":
"Common Workflow Language (CWL) CommandLineTool specification."
},
{
"href": sd.CWL_WORKFLOW_URL,
"rel": "cwl-workflow",
"type": CONTENT_TYPE_TEXT_HTML,
"title":
"Common Workflow Language (CWL) Workflow specification."
},
])
if weaver_api_ref:
# sample:
# https://app.swaggerhub.com/apis/geoprocessing/WPS/
weaver_links.append({
"href":
weaver_api_ref,
"rel":
"reference",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"API reference specification of this service."
})
if isinstance(weaver_api_doc, str):
# sample:
# https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/develop/docs/18-062.pdf
if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test
ext_type = weaver_api_doc.split(".")[-1]
doc_type = "application/{}".format(ext_type)
else:
doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type
weaver_links.append({
"href":
weaver_api_doc,
"rel":
"documentation",
"type":
doc_type,
"title":
"API reference documentation about this service."
})
else:
weaver_links.append({
"href":
__meta__.__documentation_url__,
"rel":
"documentation",
"type":
CONTENT_TYPE_TEXT_HTML,
"title":
"API reference documentation about this service."
})
if weaver_wps:
weaver_links.extend([
{
"href": weaver_wps_url,
"rel": "wps",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 1.0.0/2.0 XML endpoint of this service."
},
{
"href": "http://docs.opengeospatial.org/is/14-065/14-065.html",
"rel": "wps-specification",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS 1.0.0/2.0 definition of this service."
},
{
"href": "http://schemas.opengis.net/wps/",
"rel": "wps-schema-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS 1.0.0/2.0 XML schemas repository."
},
{
"href": "http://schemas.opengis.net/wps/1.0.0/wpsAll.xsd",
"rel": "wps-schema-1",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 1.0.0 XML validation schemas entrypoint."
},
{
"href": "http://schemas.opengis.net/wps/2.0/wps.xsd",
"rel": "wps-schema-2",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 2.0 XML validation schemas entrypoint."
},
])
return {
"message":
"Weaver Information",
"configuration":
weaver_config,
"description":
__meta__.__description__,
"parameters": [
{
"name": "api",
"enabled": weaver_api,
"url": weaver_api_url,
"api": weaver_api_oas_ui
},
{
"name": "wps",
"enabled": weaver_wps,
"url": weaver_wps_url
},
],
"links":
weaver_links,
}
def request_quote(request):
"""
Request a quotation for a process.
"""
settings = get_settings(request)
weaver_config = get_weaver_configuration(settings)
if weaver_config not in [
WEAVER_CONFIGURATION_ADES, WEAVER_CONFIGURATION_EMS
]:
raise HTTPBadRequest(
"Unsupported request for configuration '{}'.".format(
weaver_config))
process_id = request.matchdict.get("process_id")
process_store = get_db(request).get_store("processes")
try:
process = process_store.fetch_by_id(process_id)
except ProcessNotFound:
raise HTTPNotFound(
"Could not find process with specified 'process_id'.")
store = get_db(request).get_store(StoreQuotes)
process_url = get_process_location(process_id,
data_source=get_weaver_url(settings))
process_type = process.type
process_params = dict()
for param in ["inputs", "outputs", "mode", "response"]:
if param in request.json:
process_params[param] = request.json.pop(param)
process_quote_info = process_quote_estimator(process)
process_quote_info.update({
"process": process_id,
"processParameters": process_params,
"location": process_url,
"user": str(request.authenticated_userid)
})
# loop workflow sub-process steps to get individual quotes
if process_type == PROCESS_WORKFLOW and weaver_config == WEAVER_CONFIGURATION_EMS:
workflow_quotes = list()
for step in get_package_workflow_steps(process_url):
# retrieve quote from provider ADES
# TODO: data source mapping
process_step_url = get_process_location(step["reference"])
process_quote_url = "{}/quotations".format(process_step_url)
subreq = request.copy()
subreq.path_info = process_quote_url
resp_json = request.invoke_subrequest(subreq).json()
quote_json = resp_json["quote"]
quote = store.save_quote(Quote(**quote_json))
workflow_quotes.append(quote.id)
process_quote_info.update({"steps": workflow_quotes})
quote = store.save_quote(Quote(**process_quote_info))
return HTTPCreated(json={"quote": quote.json()})
# single application quotes (ADES or EMS)
elif process_type == PROCESS_APPLICATION:
quote = store.save_quote(Quote(**process_quote_info))
quote_json = quote.json()
quote_json.pop("steps", None)
return HTTPCreated(json={"quote": quote_json})
# error if not handled up to this point
raise HTTPBadRequest(
"Unsupported quoting process type '{0}' on '{1}'.".format(
process_type, weaver_config))
def deploy_process_from_payload(payload, container):
# type: (JSON, AnyContainer) -> HTTPException
"""
Adds a :class:`weaver.datatype.Process` instance to storage using the provided JSON ``payload`` matching
:class:`weaver.wps_restapi.swagger_definitions.ProcessDescription`.
:returns: HTTPOk if the process registration was successful
:raises HTTPException: otherwise
"""
_check_deploy(payload)
# use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload
payload_copy = deepcopy(payload)
# validate identifier naming for unsupported characters
process_description = payload.get("processDescription")
process_info = process_description.get("process", {})
process_href = process_description.pop("href", None)
# retrieve CWL package definition, either via "href" (WPS-1/2), "owsContext" or "executionUnit" (package/reference)
deployment_profile_name = payload.get("deploymentProfileName", "").lower()
ows_context = process_info.pop("owsContext", None)
reference = None
package = None
if process_href:
reference = process_href # reference type handled downstream
elif isinstance(ows_context, dict):
offering = ows_context.get("offering")
if not isinstance(offering, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering'."
)
content = offering.get("content")
if not isinstance(content, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'processDescription.process.owsContext.offering.content'."
)
package = None
reference = content.get("href")
elif deployment_profile_name:
if not any(
deployment_profile_name.endswith(typ)
for typ in [PROCESS_APPLICATION, PROCESS_WORKFLOW]):
raise HTTPBadRequest(
"Invalid value for parameter 'deploymentProfileName'.")
execution_units = payload.get("executionUnit")
if not isinstance(execution_units, list):
raise HTTPUnprocessableEntity("Invalid parameter 'executionUnit'.")
for execution_unit in execution_units:
if not isinstance(execution_unit, dict):
raise HTTPUnprocessableEntity(
"Invalid parameter 'executionUnit'.")
package = execution_unit.get("unit")
reference = execution_unit.get("href")
# stop on first package/reference found, simultaneous usage will raise during package retrieval
if package or reference:
break
else:
raise HTTPBadRequest(
"Missing one of required parameters [href, owsContext, deploymentProfileName]."
)
# obtain updated process information using WPS process offering, CWL/WPS reference or CWL package definition
process_info = _get_deploy_process_info(process_info, reference, package)
# validate process type against weaver configuration
settings = get_settings(container)
process_type = process_info["type"]
if process_type == PROCESS_WORKFLOW:
weaver_config = get_weaver_configuration(settings)
if weaver_config != WEAVER_CONFIGURATION_EMS:
raise HTTPBadRequest(
"Invalid [{0}] package deployment on [{1}].".format(
process_type, weaver_config))
restapi_url = get_wps_restapi_base_url(settings)
description_url = "/".join(
[restapi_url, "processes", process_info["identifier"]])
execute_endpoint = "/".join([description_url, "jobs"])
# ensure that required "processEndpointWPS1" in db is added,
# will be auto-fixed to localhost if not specified in body
process_info["processEndpointWPS1"] = process_description.get(
"processEndpointWPS1")
process_info["executeEndpoint"] = execute_endpoint
process_info["payload"] = payload_copy
process_info["jobControlOptions"] = process_description.get(
"jobControlOptions", [])
process_info["outputTransmission"] = process_description.get(
"outputTransmission", [])
process_info["processDescriptionURL"] = description_url
# insert the "resolved" context using details retrieved from "executionUnit"/"href" or directly with "owsContext"
if "owsContext" not in process_info and reference:
process_info["owsContext"] = {
"offering": {
"content": {
"href": str(reference)
}
}
}
elif isinstance(ows_context, dict):
process_info["owsContext"] = ows_context
try:
store = get_db(container).get_store(StoreProcesses)
saved_process = store.save_process(Process(process_info),
overwrite=False)
except ProcessRegistrationError as ex:
raise HTTPConflict(detail=str(ex))
except ValueError as ex:
# raised on invalid process name
raise HTTPBadRequest(detail=str(ex))
json_response = {
"processSummary": saved_process.process_summary(),
"deploymentDone": True
}
return HTTPOk(
json=json_response
) # FIXME: should be 201 (created), update swagger accordingly
def request_quote(request):
# type: (PyramidRequest) -> AnyViewResponse
"""
Request a quotation for a process.
"""
settings = get_settings(request)
weaver_config = get_weaver_configuration(settings)
if weaver_config not in WeaverFeature.QUOTING:
raise HTTPBadRequest(f"Unsupported quoting request for configuration '{weaver_config}'.")
process_id = request.matchdict.get("process_id")
process_store = get_db(request).get_store(StoreProcesses)
try:
process = process_store.fetch_by_id(process_id) # type: Process
except ProcessNotFound:
raise ProcessNotFound(json={
"title": "NoSuchProcess",
"type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process",
"detail": "Process with specified reference identifier does not exist.",
"status": ProcessNotFound.code,
"cause": str(process_id)
})
if (
(process.type not in [ProcessType.APPLICATION, ProcessType.WORKFLOW]) or
(process.type == ProcessType.WORKFLOW and weaver_config not in WeaverFeature.REMOTE)
):
raise HTTPBadRequest(json={
"title": "UnsupportedOperation",
"detail": f"Unsupported quoting process type '{process.type}' on '{weaver_config}' instance.",
"status": HTTPBadRequest.code,
"instance": process.href(settings)
})
try:
process_params = sd.QuoteProcessParametersSchema().deserialize(request.json)
except colander.Invalid as exc:
raise OWSMissingParameterValue(json={
"title": "MissingParameterValue",
"cause": f"Invalid schema: [{exc.msg!s}]",
"error": exc.__class__.__name__,
"value": exc.value
})
quote_store = get_db(request).get_store(StoreQuotes)
quote_user = request.authenticated_userid
quote_info = {
"process": process_id,
"processParameters": process_params,
"user": quote_user
}
quote = Quote(**quote_info)
quote = quote_store.save_quote(quote)
max_wait = as_int(settings.get("weaver.quote_sync_max_wait"), default=20)
mode, wait, applied = parse_prefer_header_execute_mode(request.headers, process.jobControlOptions, max_wait)
result = process_quote_estimator.delay(quote.id)
LOGGER.debug("Celery pending task [%s] for quote [%s].", result.id, quote.id)
if mode == ExecuteMode.SYNC and wait:
LOGGER.debug("Celery task requested as sync if it completes before (wait=%ss)", wait)
try:
result.wait(timeout=wait)
except CeleryTaskTimeoutError:
pass
if result.ready():
quote = quote_store.fetch_by_id(quote.id)
data = quote.json()
data.update({"description": sd.CreatedQuoteResponse.description})
data.update({"links": quote.links(settings)})
data = sd.CreatedQuoteResponse().deserialize(data)
return HTTPCreated(json=data)
else:
LOGGER.debug("Celery task requested as sync took too long to complete (wait=%ss). Continue in async.", wait)
# sync not respected, therefore must drop it
# since both could be provided as alternative preferences, drop only async with limited subset
prefer = get_header("Preference-Applied", applied, pop=True)
_, _, async_applied = parse_prefer_header_execute_mode({"Prefer": prefer}, [ExecuteMode.ASYNC])
applied = async_applied
data = quote.partial()
data.update({"description": sd.AcceptedQuoteResponse.description})
headers = {"Location": quote.href(settings)}
headers.update(applied)
return HTTPAccepted(headers=headers, json=data)
