def main(global_config, **settings): """ Creates a Pyramid WSGI application for Weaver. """ setup_loggers(settings) LOGGER.info("Initiating weaver application") # validate and fix configuration weaver_config = get_weaver_configuration(settings) settings.update({"weaver.configuration": weaver_config}) # Parse extra_options and add each of them in the settings dict LOGGER.info("Parsing extra options...") settings.update( parse_extra_options(settings.get("weaver.extra_options", ""))) # load requests options if found, otherwise skip LOGGER.info("Checking for request options file...") req_file = get_weaver_config_file(settings.get("weaver.request_options", ""), WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG, generate_default_from_example=False) if req_file: LOGGER.info("Loading request options...") with open(req_file, "r") as f: settings.update({"weaver.request_options": yaml.safe_load(f)}) else: LOGGER.warning("No request options found.") # add default caching regions if they were omitted in config file if settings.get("weaver.celery", False): LOGGER.info("Celery runner detected. Skipping cache options setup.") else: LOGGER.info("Adding default caching options...") setup_cache(settings) LOGGER.info("Setup celery configuration...") local_config = Configurator(settings=settings) if global_config.get("__file__") is not None: local_config.include("pyramid_celery") local_config.configure_celery(global_config["__file__"]) local_config.include("weaver") LOGGER.info("Running database migration...") db = get_db(settings) db.run_migration() if settings.get("weaver.celery", False): LOGGER.info("Celery runner detected. Skipping process registration.") else: LOGGER.info("Registering builtin processes...") register_builtin_processes(local_config) LOGGER.info("Registering WPS-1 processes from configuration file...") wps_processes_file = get_settings(local_config).get( "weaver.wps_processes_file") register_wps_processes_from_config(wps_processes_file, local_config) return local_config.make_wsgi_app()
def api_frontpage(request): """Frontpage of weaver.""" # import here to avoid circular import errors from weaver.config import get_weaver_configuration settings = get_settings(request) weaver_url = get_weaver_url(settings) weaver_config = get_weaver_configuration(settings) weaver_api = asbool(settings.get("weaver.wps_restapi")) weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None weaver_api_def = weaver_api_url + sd.api_swagger_ui_uri if weaver_api else None weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None weaver_wps = asbool(settings.get("weaver.wps")) weaver_wps_url = get_wps_url(settings) if weaver_wps else None weaver_conform_url = weaver_url + sd.api_conformance_uri weaver_process_url = weaver_url + sd.processes_uri weaver_links = [ {"href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This document"}, {"href": weaver_conform_url, "rel": "conformance", "type": CONTENT_TYPE_APP_JSON, "title": "WPS 2.0/3.0 REST-JSON Binding Extension conformance classes implemented by this service."}, ] if weaver_api_def: weaver_links.append({"href": weaver_api_def, "rel": "service", "type": CONTENT_TYPE_APP_JSON, "title": "API definition of this service."}) if isinstance(weaver_api_doc, str): if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test ext_type = weaver_api_doc.split(".")[-1] doc_type = "application/{}".format(ext_type) else: doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type weaver_links.append({"href": weaver_api_doc, "rel": "documentation", "type": doc_type, "title": "API documentation about this service."}) if weaver_api_ref: weaver_links.append({"href": weaver_api_ref, "rel": "reference", "type": CONTENT_TYPE_APP_JSON, "title": "API reference specification of this service."}) weaver_links.append({"href": weaver_process_url, "rel": "processes", "type": CONTENT_TYPE_APP_JSON, "title": "Processes offered by this service."}) return { "message": "Weaver Information", "configuration": weaver_config, "parameters": [ {"name": "api", "enabled": weaver_api, "url": weaver_api_url, "doc": weaver_api_doc, "api": weaver_api_def, "ref": weaver_api_ref}, {"name": "wps", "enabled": weaver_wps, "url": weaver_wps_url}, ], "links": weaver_links, }
def forbid_local_only(container): # type: (AnySettingsContainer) -> Any """ Raises an HTTP exception forbidding to resume the operation if invalid configuration is detected. """ config = get_weaver_configuration(container) if config not in WEAVER_CONFIGURATIONS_REMOTE: raise HTTPForbidden(json={ "description": "Invalid provider operation on [{}] instance. " "Processes requires unsupported remote execution.".format(config), })
def get_processes(request): """ List registered processes (GetCapabilities). Optionally list both local and provider processes. """ detail = asbool(request.params.get("detail", True)) try: # get local processes and filter according to schema validity # (previously deployed process schemas can become invalid because of modified schema definitions processes, invalid_processes = get_processes_filtered_by_valid_schemas( request) if invalid_processes: raise HTTPServiceUnavailable( "Previously deployed processes are causing invalid schema integrity errors. " "Manual cleanup of following processes is required: {}".format( invalid_processes)) response_body = { "processes": processes if detail else [get_any_id(p) for p in processes] } # if 'EMS' and '?providers=True', also fetch each provider's processes settings = get_settings(request) if get_weaver_configuration(settings) == WEAVER_CONFIGURATION_EMS: queries = parse_request_query(request) if "providers" in queries and asbool( queries["providers"][0]) is True: prov_url = "{host}/providers".format(host=request.host_url) providers_response = request_extra("GET", prov_url, settings=settings, headers=request.headers, cookies=request.cookies) providers = providers_response.json() response_body.update({"providers": providers}) for i, provider in enumerate(providers): provider_id = get_any_id(provider) proc_url = "{host}/providers/{prov}/processes".format( host=request.host_url, prov=provider_id) response = request_extra("GET", proc_url, settings=settings, headers=request.headers, cookies=request.cookies) processes = response.json().get("processes", []) response_body["providers"][i].update({ "processes": processes if detail else [get_any_id(p) for p in processes] }) return HTTPOk(json=response_body) except colander.Invalid as ex: raise HTTPBadRequest("Invalid schema: [{!s}]".format(ex))
def get_processes_filtered_by_valid_schemas(request): # type: (PyramidRequest) -> Tuple[List[JSON], List[str], Dict[str, Optional[int]], bool, int] """ Validates the processes summary schemas and returns them into valid/invalid lists. :returns: List of valid process and invalid processes IDs for manual cleanup, along with filtering parameters. """ settings = get_settings(request) with_providers = False if get_weaver_configuration(settings) in WeaverFeature.REMOTE: with_providers = asbool(request.params.get("providers", False)) revisions_param = sd.ProcessRevisionsQuery(unknown="ignore").deserialize( request.params) with_revisions = revisions_param.get("revisions") paging_query = sd.ProcessPagingQuery() paging_value = { param.name: param.default for param in paging_query.children } paging_names = set(paging_value) paging_param = paging_query.deserialize(request.params) if with_providers and any(value != paging_value[param] for param, value in paging_param.items()): raise HTTPBadRequest( json={ "description": "Cannot combine paging/sorting parameters with providers full listing query.", "error": "ListingInvalidParameter", "value": list(paging_names.intersection(request.params)) }) store = get_db(request).get_store(StoreProcesses) processes, total_local_processes = store.list_processes( visibility=Visibility.PUBLIC, total=True, **revisions_param, **paging_param) valid_processes = [] invalid_processes_ids = [] for process in processes: # type: Process try: valid_processes.append(process.summary(revision=with_revisions)) except colander.Invalid as invalid: process_ref = process.tag if with_revisions else process.identifier LOGGER.debug("Invalid process [%s] because:\n%s", process_ref, invalid) invalid_processes_ids.append(process.identifier) return valid_processes, invalid_processes_ids, paging_param, with_providers, total_local_processes
def _validate_deploy_process_info(process_info, reference, package, settings, headers): # type: (JSON, Optional[str], Optional[CWL], SettingsType, Optional[AnyHeadersContainer]) -> JSON """ Obtain the process definition from deploy payload with exception handling. .. seealso:: - :func:`weaver.processes.wps_package.get_process_definition` """ from weaver.processes.wps_package import check_package_instance_compatible, get_process_definition try: # data_source `None` forces workflow process to search locally for deployed step applications info = get_process_definition(process_info, reference, package, data_source=None, headers=headers) # validate process type and package against weaver configuration cfg = get_weaver_configuration(settings) if cfg not in WEAVER_CONFIGURATIONS_REMOTE: problem = check_package_instance_compatible(info["package"]) if problem: raise HTTPForbidden( json={ "description": "Invalid process deployment of type [{}] on [{}] instance. " "Remote execution is required but not supported.". format(info["type"], cfg), "cause": problem }) return info except PackageNotFound as ex: # raised when a workflow sub-process is not found (not deployed locally) raise HTTPNotFound(detail=str(ex)) except InvalidIdentifierValue as ex: raise HTTPBadRequest(str(ex)) except (PackageRegistrationError, PackageTypeError) as ex: msg = "Invalid package/reference definition. Loading generated error: [{!s}]".format( ex) LOGGER.exception(msg) raise HTTPUnprocessableEntity(detail=msg)
def main(global_config, **settings): """ Creates a Pyramid WSGI application for Weaver. """ setup_loggers(settings) LOGGER.info("Initiating weaver application") # validate and fix configuration weaver_config = get_weaver_configuration(settings) settings.update({"weaver.configuration": weaver_config}) # Parse extra_options and add each of them in the settings dict settings.update( parse_extra_options(settings.get("weaver.extra_options", ""))) # load requests options if found, otherwise skip req_file = get_weaver_config_file(settings.get("weaver.request_options", ""), WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG, generate_default_from_example=False) if req_file: with open(req_file, "r") as f: settings.update({"weaver.request_options": yaml.safe_load(f)}) local_config = Configurator(settings=settings) if global_config.get("__file__") is not None: local_config.include("pyramid_celery") local_config.configure_celery(global_config["__file__"]) local_config.include("weaver") LOGGER.info("Registering builtin processes...") register_builtin_processes(local_config) LOGGER.info("Registering WPS-1 processes from configuration file...") wps_processes_file = get_settings(local_config).get( "weaver.wps_processes_file") register_wps_processes_from_config(wps_processes_file, local_config) return local_config.make_wsgi_app()
def load_pywps_config(container, config=None): # type: (AnySettingsContainer, Optional[Union[str, Dict[str, str]]]) -> ConfigParser """ Loads and updates the PyWPS configuration using Weaver settings. """ settings = get_settings(container) if settings.get("weaver.wps_configured"): LOGGER.debug("Using preloaded internal Weaver WPS configuration.") return pywps_config.CONFIG LOGGER.info("Initial load of internal Weaver WPS configuration.") pywps_config.load_configuration([]) # load defaults pywps_config.CONFIG.set("logging", "db_echo", "false") if logging.getLevelName(pywps_config.CONFIG.get("logging", "level")) <= logging.DEBUG: pywps_config.CONFIG.set("logging", "level", "INFO") # update metadata LOGGER.debug("Updating WPS metadata configuration.") for setting_name, setting_value in settings.items(): if setting_name.startswith("weaver.wps_metadata"): pywps_setting = setting_name.replace("weaver.wps_metadata_", "") pywps_config.CONFIG.set("metadata:main", pywps_setting, setting_value) # add weaver configuration keyword if not already provided wps_keywords = pywps_config.CONFIG.get("metadata:main", "identification_keywords") weaver_mode = get_weaver_configuration(settings) if weaver_mode not in wps_keywords: wps_keywords += ("," if wps_keywords else "") + weaver_mode pywps_config.CONFIG.set("metadata:main", "identification_keywords", wps_keywords) # add additional config passed as dictionary of {'section.key': 'value'} if isinstance(config, dict): for key, value in config.items(): section, key = key.split(".") pywps_config.CONFIG.set(section, key, value) # cleanup alternative dict "PYWPS_CFG" which is not expected elsewhere if isinstance(settings.get("PYWPS_CFG"), dict): del settings["PYWPS_CFG"] # set accepted languages aligned with values provided by REST API endpoints # otherwise, execute request could fail due to languages considered not supported languages = ", ".join(AcceptLanguage.values()) LOGGER.debug("Setting WPS languages: [%s]", languages) pywps_config.CONFIG.set("server", "language", languages) LOGGER.debug("Updating WPS output configuration.") # find output directory from app config or wps config if "weaver.wps_output_dir" not in settings: output_dir = pywps_config.get_config_value("server", "outputpath") settings["weaver.wps_output_dir"] = output_dir # ensure the output dir exists if specified output_dir = get_wps_output_dir(settings) make_dirs(output_dir, exist_ok=True) # find output url from app config (path/url) or wps config (url only) # note: needs to be configured even when using S3 bucket since XML status is provided locally if "weaver.wps_output_url" not in settings: output_path = settings.get("weaver.wps_output_path", "").rstrip("/") if output_path and isinstance(output_path, str): output_url = os.path.join(get_weaver_url(settings), output_path.strip("/")) else: output_url = pywps_config.get_config_value("server", "outputurl") settings["weaver.wps_output_url"] = output_url # apply workdir if provided, otherwise use default if "weaver.wps_workdir" in settings: make_dirs(settings["weaver.wps_workdir"], exist_ok=True) pywps_config.CONFIG.set("server", "workdir", settings["weaver.wps_workdir"]) # configure S3 bucket if requested, storage of all process outputs # note: # credentials and default profile are picked up automatically by 'boto3' from local AWS configs or env vars # region can also be picked from there unless explicitly provided by weaver config # warning: # if we set `(server, storagetype, s3)`, ALL status (including XML) are stored to S3 # to preserve status locally, we set 'file' and override the storage instance during output rewrite in WpsPackage # we can still make use of the server configurations here to make this overridden storage auto-find its configs s3_bucket = settings.get("weaver.wps_output_s3_bucket") pywps_config.CONFIG.set("server", "storagetype", "file") # pywps_config.CONFIG.set("server", "storagetype", "s3") if s3_bucket: LOGGER.debug("Updating WPS S3 bucket configuration.") import boto3 from botocore.exceptions import ClientError s3 = boto3.client("s3") s3_region = settings.get("weaver.wps_output_s3_region", s3.meta.region_name) LOGGER.info( "Validating that S3 [Bucket=%s, Region=%s] exists or creating it.", s3_bucket, s3_region) try: s3.create_bucket( Bucket=s3_bucket, CreateBucketConfiguration={"LocationConstraint": s3_region}) LOGGER.info("S3 bucket for WPS output created.") except ClientError as exc: if exc.response.get("Error", {}).get("Code") != "BucketAlreadyExists": LOGGER.error("Failed setup of S3 bucket for WPS output: [%s]", exc) raise LOGGER.info("S3 bucket for WPS output already exists.") pywps_config.CONFIG.set("s3", "region", s3_region) pywps_config.CONFIG.set("s3", "bucket", s3_bucket) pywps_config.CONFIG.set( "s3", "public", "false") # don't automatically push results as publicly accessible pywps_config.CONFIG.set( "s3", "encrypt", "true") # encrypts data server-side, transparent from this side # enforce back resolved values onto PyWPS config pywps_config.CONFIG.set("server", "setworkdir", "true") pywps_config.CONFIG.set("server", "sethomedir", "true") pywps_config.CONFIG.set("server", "outputpath", settings["weaver.wps_output_dir"]) pywps_config.CONFIG.set("server", "outputurl", settings["weaver.wps_output_url"]) pywps_config.CONFIG.set("server", "url", get_wps_url(settings, load=False)) settings["weaver.wps_configured"] = True return pywps_config.CONFIG
def api_frontpage_body(settings): # type: (SettingsType) -> JSON """ Generates the JSON body describing the Weaver API and documentation references. """ # import here to avoid circular import errors from weaver.config import get_weaver_configuration weaver_url = get_weaver_url(settings) weaver_config = get_weaver_configuration(settings) weaver_api = asbool(settings.get("weaver.wps_restapi")) weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None weaver_api_oas_ui = weaver_api_url + sd.api_openapi_ui_service.path if weaver_api else None weaver_api_swagger = weaver_api_url + sd.api_swagger_ui_service.path if weaver_api else None weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None weaver_api_spec = weaver_api_url + sd.openapi_json_service.path if weaver_api else None weaver_wps = asbool(settings.get("weaver.wps")) weaver_wps_url = get_wps_url(settings) if weaver_wps else None weaver_conform_url = weaver_url + sd.api_conformance_service.path weaver_process_url = weaver_url + sd.processes_service.path weaver_jobs_url = weaver_url + sd.jobs_service.path weaver_links = [{ "href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This landing page." }, { "href": weaver_conform_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/conformance", "type": CONTENT_TYPE_APP_JSON, "title": "Conformance classes implemented by this service." }, { "href": __meta__.__license_url__, "rel": "license", "type": CONTENT_TYPE_TEXT_PLAIN, "title": __meta__.__license_long__ }] if weaver_api: weaver_links.extend([ { "href": weaver_api_url, "rel": "service", "type": CONTENT_TYPE_APP_JSON, "title": "WPS REST API endpoint of this service." }, { "href": weaver_api_spec, "rel": "service-desc", "type": CONTENT_TYPE_APP_JSON, "title": "OpenAPI specification of this service." }, { "href": weaver_api_oas_ui, "rel": "service-doc", "type": CONTENT_TYPE_TEXT_HTML, "title": "Human readable OpenAPI documentation of this service." }, { "href": weaver_api_spec, "rel": "OpenAPI", "type": CONTENT_TYPE_APP_JSON, "title": "OpenAPI specification of this service." }, { "href": weaver_api_swagger, "rel": "swagger-ui", "type": CONTENT_TYPE_TEXT_HTML, "title": "WPS REST API definition of this service." }, { "href": weaver_process_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/processes", "type": CONTENT_TYPE_APP_JSON, "title": "Processes offered by this service." }, { "href": sd.OGC_API_REPO_URL, "rel": "ogcapi-processes-repository", "type": CONTENT_TYPE_TEXT_HTML, "title": "OGC API - Processes schema definitions repository." }, { "href": weaver_jobs_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/job-list", "type": CONTENT_TYPE_APP_JSON, "title": "Job search and listing endpoint of executions registered under this service." }, { "href": sd.CWL_BASE_URL, "rel": "cwl-home", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) homepage." }, { "href": sd.CWL_REPO_URL, "rel": "cwl-repository", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) repositories." }, { "href": sd.CWL_SPEC_URL, "rel": "cwl-specification", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) specification." }, { "href": sd.CWL_USER_GUIDE_URL, "rel": "cwl-user-guide", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) user guide." }, { "href": sd.CWL_CMD_TOOL_URL, "rel": "cwl-command-line-tool", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) CommandLineTool specification." }, { "href": sd.CWL_WORKFLOW_URL, "rel": "cwl-workflow", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) Workflow specification." }, ]) if weaver_api_ref: # sample: # https://app.swaggerhub.com/apis/geoprocessing/WPS/ weaver_links.append({ "href": weaver_api_ref, "rel": "reference", "type": CONTENT_TYPE_APP_JSON, "title": "API reference specification of this service." }) if isinstance(weaver_api_doc, str): # sample: # https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/develop/docs/18-062.pdf if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test ext_type = weaver_api_doc.split(".")[-1] doc_type = "application/{}".format(ext_type) else: doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type weaver_links.append({ "href": weaver_api_doc, "rel": "documentation", "type": doc_type, "title": "API reference documentation about this service." }) else: weaver_links.append({ "href": __meta__.__documentation_url__, "rel": "documentation", "type": CONTENT_TYPE_TEXT_HTML, "title": "API reference documentation about this service." }) if weaver_wps: weaver_links.extend([ { "href": weaver_wps_url, "rel": "wps", "type": CONTENT_TYPE_TEXT_XML, "title": "WPS 1.0.0/2.0 XML endpoint of this service." }, { "href": "http://docs.opengeospatial.org/is/14-065/14-065.html", "rel": "wps-specification", "type": CONTENT_TYPE_TEXT_HTML, "title": "WPS 1.0.0/2.0 definition of this service." }, { "href": "http://schemas.opengis.net/wps/", "rel": "wps-schema-repository", "type": CONTENT_TYPE_TEXT_HTML, "title": "WPS 1.0.0/2.0 XML schemas repository." }, { "href": "http://schemas.opengis.net/wps/1.0.0/wpsAll.xsd", "rel": "wps-schema-1", "type": CONTENT_TYPE_TEXT_XML, "title": "WPS 1.0.0 XML validation schemas entrypoint." }, { "href": "http://schemas.opengis.net/wps/2.0/wps.xsd", "rel": "wps-schema-2", "type": CONTENT_TYPE_TEXT_XML, "title": "WPS 2.0 XML validation schemas entrypoint." }, ]) return { "message": "Weaver Information", "configuration": weaver_config, "description": __meta__.__description__, "parameters": [ { "name": "api", "enabled": weaver_api, "url": weaver_api_url, "api": weaver_api_oas_ui }, { "name": "wps", "enabled": weaver_wps, "url": weaver_wps_url }, ], "links": weaver_links, }
def request_quote(request): """ Request a quotation for a process. """ settings = get_settings(request) weaver_config = get_weaver_configuration(settings) if weaver_config not in [ WEAVER_CONFIGURATION_ADES, WEAVER_CONFIGURATION_EMS ]: raise HTTPBadRequest( "Unsupported request for configuration '{}'.".format( weaver_config)) process_id = request.matchdict.get("process_id") process_store = get_db(request).get_store("processes") try: process = process_store.fetch_by_id(process_id) except ProcessNotFound: raise HTTPNotFound( "Could not find process with specified 'process_id'.") store = get_db(request).get_store(StoreQuotes) process_url = get_process_location(process_id, data_source=get_weaver_url(settings)) process_type = process.type process_params = dict() for param in ["inputs", "outputs", "mode", "response"]: if param in request.json: process_params[param] = request.json.pop(param) process_quote_info = process_quote_estimator(process) process_quote_info.update({ "process": process_id, "processParameters": process_params, "location": process_url, "user": str(request.authenticated_userid) }) # loop workflow sub-process steps to get individual quotes if process_type == PROCESS_WORKFLOW and weaver_config == WEAVER_CONFIGURATION_EMS: workflow_quotes = list() for step in get_package_workflow_steps(process_url): # retrieve quote from provider ADES # TODO: data source mapping process_step_url = get_process_location(step["reference"]) process_quote_url = "{}/quotations".format(process_step_url) subreq = request.copy() subreq.path_info = process_quote_url resp_json = request.invoke_subrequest(subreq).json() quote_json = resp_json["quote"] quote = store.save_quote(Quote(**quote_json)) workflow_quotes.append(quote.id) process_quote_info.update({"steps": workflow_quotes}) quote = store.save_quote(Quote(**process_quote_info)) return HTTPCreated(json={"quote": quote.json()}) # single application quotes (ADES or EMS) elif process_type == PROCESS_APPLICATION: quote = store.save_quote(Quote(**process_quote_info)) quote_json = quote.json() quote_json.pop("steps", None) return HTTPCreated(json={"quote": quote_json}) # error if not handled up to this point raise HTTPBadRequest( "Unsupported quoting process type '{0}' on '{1}'.".format( process_type, weaver_config))
def deploy_process_from_payload(payload, container): # type: (JSON, AnyContainer) -> HTTPException """ Adds a :class:`weaver.datatype.Process` instance to storage using the provided JSON ``payload`` matching :class:`weaver.wps_restapi.swagger_definitions.ProcessDescription`. :returns: HTTPOk if the process registration was successful :raises HTTPException: otherwise """ _check_deploy(payload) # use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload payload_copy = deepcopy(payload) # validate identifier naming for unsupported characters process_description = payload.get("processDescription") process_info = process_description.get("process", {}) process_href = process_description.pop("href", None) # retrieve CWL package definition, either via "href" (WPS-1/2), "owsContext" or "executionUnit" (package/reference) deployment_profile_name = payload.get("deploymentProfileName", "").lower() ows_context = process_info.pop("owsContext", None) reference = None package = None if process_href: reference = process_href # reference type handled downstream elif isinstance(ows_context, dict): offering = ows_context.get("offering") if not isinstance(offering, dict): raise HTTPUnprocessableEntity( "Invalid parameter 'processDescription.process.owsContext.offering'." ) content = offering.get("content") if not isinstance(content, dict): raise HTTPUnprocessableEntity( "Invalid parameter 'processDescription.process.owsContext.offering.content'." ) package = None reference = content.get("href") elif deployment_profile_name: if not any( deployment_profile_name.endswith(typ) for typ in [PROCESS_APPLICATION, PROCESS_WORKFLOW]): raise HTTPBadRequest( "Invalid value for parameter 'deploymentProfileName'.") execution_units = payload.get("executionUnit") if not isinstance(execution_units, list): raise HTTPUnprocessableEntity("Invalid parameter 'executionUnit'.") for execution_unit in execution_units: if not isinstance(execution_unit, dict): raise HTTPUnprocessableEntity( "Invalid parameter 'executionUnit'.") package = execution_unit.get("unit") reference = execution_unit.get("href") # stop on first package/reference found, simultaneous usage will raise during package retrieval if package or reference: break else: raise HTTPBadRequest( "Missing one of required parameters [href, owsContext, deploymentProfileName]." ) # obtain updated process information using WPS process offering, CWL/WPS reference or CWL package definition process_info = _get_deploy_process_info(process_info, reference, package) # validate process type against weaver configuration settings = get_settings(container) process_type = process_info["type"] if process_type == PROCESS_WORKFLOW: weaver_config = get_weaver_configuration(settings) if weaver_config != WEAVER_CONFIGURATION_EMS: raise HTTPBadRequest( "Invalid [{0}] package deployment on [{1}].".format( process_type, weaver_config)) restapi_url = get_wps_restapi_base_url(settings) description_url = "/".join( [restapi_url, "processes", process_info["identifier"]]) execute_endpoint = "/".join([description_url, "jobs"]) # ensure that required "processEndpointWPS1" in db is added, # will be auto-fixed to localhost if not specified in body process_info["processEndpointWPS1"] = process_description.get( "processEndpointWPS1") process_info["executeEndpoint"] = execute_endpoint process_info["payload"] = payload_copy process_info["jobControlOptions"] = process_description.get( "jobControlOptions", []) process_info["outputTransmission"] = process_description.get( "outputTransmission", []) process_info["processDescriptionURL"] = description_url # insert the "resolved" context using details retrieved from "executionUnit"/"href" or directly with "owsContext" if "owsContext" not in process_info and reference: process_info["owsContext"] = { "offering": { "content": { "href": str(reference) } } } elif isinstance(ows_context, dict): process_info["owsContext"] = ows_context try: store = get_db(container).get_store(StoreProcesses) saved_process = store.save_process(Process(process_info), overwrite=False) except ProcessRegistrationError as ex: raise HTTPConflict(detail=str(ex)) except ValueError as ex: # raised on invalid process name raise HTTPBadRequest(detail=str(ex)) json_response = { "processSummary": saved_process.process_summary(), "deploymentDone": True } return HTTPOk( json=json_response ) # FIXME: should be 201 (created), update swagger accordingly
def request_quote(request): # type: (PyramidRequest) -> AnyViewResponse """ Request a quotation for a process. """ settings = get_settings(request) weaver_config = get_weaver_configuration(settings) if weaver_config not in WeaverFeature.QUOTING: raise HTTPBadRequest(f"Unsupported quoting request for configuration '{weaver_config}'.") process_id = request.matchdict.get("process_id") process_store = get_db(request).get_store(StoreProcesses) try: process = process_store.fetch_by_id(process_id) # type: Process except ProcessNotFound: raise ProcessNotFound(json={ "title": "NoSuchProcess", "type": "http://www.opengis.net/def/exceptions/ogcapi-processes-1/1.0/no-such-process", "detail": "Process with specified reference identifier does not exist.", "status": ProcessNotFound.code, "cause": str(process_id) }) if ( (process.type not in [ProcessType.APPLICATION, ProcessType.WORKFLOW]) or (process.type == ProcessType.WORKFLOW and weaver_config not in WeaverFeature.REMOTE) ): raise HTTPBadRequest(json={ "title": "UnsupportedOperation", "detail": f"Unsupported quoting process type '{process.type}' on '{weaver_config}' instance.", "status": HTTPBadRequest.code, "instance": process.href(settings) }) try: process_params = sd.QuoteProcessParametersSchema().deserialize(request.json) except colander.Invalid as exc: raise OWSMissingParameterValue(json={ "title": "MissingParameterValue", "cause": f"Invalid schema: [{exc.msg!s}]", "error": exc.__class__.__name__, "value": exc.value }) quote_store = get_db(request).get_store(StoreQuotes) quote_user = request.authenticated_userid quote_info = { "process": process_id, "processParameters": process_params, "user": quote_user } quote = Quote(**quote_info) quote = quote_store.save_quote(quote) max_wait = as_int(settings.get("weaver.quote_sync_max_wait"), default=20) mode, wait, applied = parse_prefer_header_execute_mode(request.headers, process.jobControlOptions, max_wait) result = process_quote_estimator.delay(quote.id) LOGGER.debug("Celery pending task [%s] for quote [%s].", result.id, quote.id) if mode == ExecuteMode.SYNC and wait: LOGGER.debug("Celery task requested as sync if it completes before (wait=%ss)", wait) try: result.wait(timeout=wait) except CeleryTaskTimeoutError: pass if result.ready(): quote = quote_store.fetch_by_id(quote.id) data = quote.json() data.update({"description": sd.CreatedQuoteResponse.description}) data.update({"links": quote.links(settings)}) data = sd.CreatedQuoteResponse().deserialize(data) return HTTPCreated(json=data) else: LOGGER.debug("Celery task requested as sync took too long to complete (wait=%ss). Continue in async.", wait) # sync not respected, therefore must drop it # since both could be provided as alternative preferences, drop only async with limited subset prefer = get_header("Preference-Applied", applied, pop=True) _, _, async_applied = parse_prefer_header_execute_mode({"Prefer": prefer}, [ExecuteMode.ASYNC]) applied = async_applied data = quote.partial() data.update({"description": sd.AcceptedQuoteResponse.description}) headers = {"Location": quote.href(settings)} headers.update(applied) return HTTPAccepted(headers=headers, json=data)