def from_ows(service, process, container): # type: (Service, ProcessWPS, AnySettingsContainer) -> Process """ Converts a :mod:`owslib.wps` Process to local storage :class:`weaver.datatype.Process`. """ wps_url = get_wps_restapi_base_url(container) if wps_url == service.url: provider_url = wps_url # local weaver process, using WPS-XML endpoint else: provider_url = "{}/providers/{}".format(wps_url, service.get("name")) describe_process_url = "{}/processes/{}".format(provider_url, process.identifier) execute_process_url = "{describe_url}/jobs".format(describe_url=describe_process_url) # FIXME: should use common function inputs = [ows2json_io_FIXME(ows_input) for ows_input in get_field(process, "dataInputs", default=[])] outputs = [ows2json_io(ows_output) for ows_output in get_field(process, "processOutputs", default=[])] return Process( id=process.identifier, title=get_field(process, "title", default=process.identifier, search_variations=True), abstract=get_field(process, "abstract", default=None, search_variations=True), inputs=inputs, outputs=outputs, url=describe_process_url, processEndpointWPS1=service.get("url"), processDescriptionURL=describe_process_url, executeEndpoint=execute_process_url, package=None, )
def test_execute_rest_xml_post_resp_json(self): """ Test :term:`XML` content using :term:`WPS` format submitted to REST endpoint gets redirected automatically. """ base = get_wps_restapi_base_url(self.settings) url = f"{base}/processes/{self.process_id}/execution" self.wps_execute("2.0.0", ContentType.APP_JSON, url=url)
def get_providers(request): """ Lists registered providers. """ store = get_db(request).get_store(StoreServices) providers = [] for service in store.list_services(): try: if service.type.lower() != "wps": continue wps = WebProcessingService(url=service.url, headers=get_cookie_headers( request.headers)) providers.append( dict(id=service.name, title=getattr(wps.identification, "title", ""), abstract=getattr(wps.identification, "abstract", ""), url="{base_url}/providers/{provider_id}".format( base_url=get_wps_restapi_base_url( get_settings(request)), provider_id=service.name), public=service.public)) except Exception as exc: warnings.warn( "Exception occurred while fetching wps {0} : {1!r}".format( service.url, exc), NonBreakingExceptionWarning) return HTTPOk(json={"providers": providers})
def submit_job_handler(payload, # type: JSON settings, # type: SettingsType service_url, # type: str provider_id=None, # type: Optional[str] process_id=None, # type: str is_workflow=False, # type: bool is_local=True, # type: bool visibility=None, # type: Optional[str] language=None, # type: Optional[str] auth=None, # type: Optional[HeaderCookiesType] tags=None, # type: Optional[List[str]] user=None, # type: Optional[int] context=None, # type: Optional[str] ): # type: (...) -> JSON """ Submits the job to the Celery worker with provided parameters. Assumes that parameters have been pre-fetched and validated, except for the input payload. """ try: json_body = sd.Execute().deserialize(payload) except colander.Invalid as ex: raise HTTPBadRequest("Invalid schema: [{}]".format(str(ex))) # TODO: remove when all parameter variations are supported # FIXME: # - support 'sync' and 'Prefer' header variants (https://github.com/crim-ca/weaver/issues/247) # - support 'response: raw' (https://github.com/crim-ca/weaver/issues/376) # - allow omitting 'outputs' (https://github.com/crim-ca/weaver/issues/375) _validate_job_parameters(json_body) is_execute_async = json_body["mode"] != EXECUTE_MODE_SYNC # convert auto to async notification_email = json_body.get("notification_email") encrypted_email = encrypt_email(notification_email, settings) if notification_email else None store = get_db(settings).get_store(StoreJobs) job = store.save_job(task_id=STATUS_ACCEPTED, process=process_id, service=provider_id, inputs=json_body.get("inputs"), is_local=is_local, is_workflow=is_workflow, access=visibility, user_id=user, execute_async=is_execute_async, custom_tags=tags, notification_email=encrypted_email, accept_language=language, context=context) job.save_log(logger=LOGGER, message="Job task submitted for execution.", status=STATUS_ACCEPTED, progress=0) job = store.update_job(job) result = execute_process.delay(job_id=job.id, wps_url=clean_ows_url(service_url), headers=auth) LOGGER.debug("Celery pending task [%s] for job [%s].", result.id, job.id) # local/provider process location location_base = "/providers/{provider_id}".format(provider_id=provider_id) if provider_id else "" location = "{base_url}{location_base}/processes/{process_id}/jobs/{job_id}".format( base_url=get_wps_restapi_base_url(settings), location_base=location_base, process_id=process_id, job_id=job.id) body_data = { "jobID": job.id, "processID": job.process, "providerID": provider_id, # dropped by validator if not applicable "status": map_status(STATUS_ACCEPTED), "location": location } return body_data
def get_capabilities(service, request): """ GetCapabilities of a wps provider. """ wps = WebProcessingService(url=service.url, headers=get_cookie_headers(request.headers)) settings = get_settings(request) return dict( id=service.name, title=wps.identification.title, abstract=wps.identification.abstract, url="{base_url}/providers/{provider_id}".format( base_url=get_wps_restapi_base_url(settings), provider_id=service.name), processes="{base_url}/providers/{provider_id}/processes".format( base_url=get_wps_restapi_base_url(settings), provider_id=service.name), type=PROCESS_WPS, contact=wps.provider.contact.name)
def api_frontpage(request): """Frontpage of weaver.""" # import here to avoid circular import errors from weaver.config import get_weaver_configuration settings = get_settings(request) weaver_url = get_weaver_url(settings) weaver_config = get_weaver_configuration(settings) weaver_api = asbool(settings.get("weaver.wps_restapi")) weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None weaver_api_def = weaver_api_url + sd.api_swagger_ui_uri if weaver_api else None weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None weaver_wps = asbool(settings.get("weaver.wps")) weaver_wps_url = get_wps_url(settings) if weaver_wps else None weaver_conform_url = weaver_url + sd.api_conformance_uri weaver_process_url = weaver_url + sd.processes_uri weaver_links = [ {"href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This document"}, {"href": weaver_conform_url, "rel": "conformance", "type": CONTENT_TYPE_APP_JSON, "title": "WPS 2.0/3.0 REST-JSON Binding Extension conformance classes implemented by this service."}, ] if weaver_api_def: weaver_links.append({"href": weaver_api_def, "rel": "service", "type": CONTENT_TYPE_APP_JSON, "title": "API definition of this service."}) if isinstance(weaver_api_doc, str): if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test ext_type = weaver_api_doc.split(".")[-1] doc_type = "application/{}".format(ext_type) else: doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type weaver_links.append({"href": weaver_api_doc, "rel": "documentation", "type": doc_type, "title": "API documentation about this service."}) if weaver_api_ref: weaver_links.append({"href": weaver_api_ref, "rel": "reference", "type": CONTENT_TYPE_APP_JSON, "title": "API reference specification of this service."}) weaver_links.append({"href": weaver_process_url, "rel": "processes", "type": CONTENT_TYPE_APP_JSON, "title": "Processes offered by this service."}) return { "message": "Weaver Information", "configuration": weaver_config, "parameters": [ {"name": "api", "enabled": weaver_api, "url": weaver_api_url, "doc": weaver_api_doc, "api": weaver_api_def, "ref": weaver_api_ref}, {"name": "wps", "enabled": weaver_wps, "url": weaver_wps_url}, ], "links": weaver_links, }
def setUpClass(cls): # disable SSL warnings from logs try: import urllib3 # noqa urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) except ImportError: pass # logging parameter overrides cls.logger_level = os.getenv("WEAVER_TEST_LOGGER_LEVEL", cls.logger_level) cls.logger_enabled = asbool(os.getenv("WEAVER_TEST_LOGGER_ENABLED", cls.logger_enabled)) cls.logger_result_dir = os.getenv("WEAVER_TEST_LOGGER_RESULT_DIR", os.path.join(WEAVER_ROOT_DIR)) cls.logger_json_indent = os.getenv("WEAVER_TEST_LOGGER_JSON_INDENT", cls.logger_json_indent) cls.logger_field_indent = os.getenv("WEAVER_TEST_LOGGER_FIELD_INDENT", cls.logger_field_indent) cls.logger_separator_calls = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_CALLS", cls.logger_separator_calls) cls.logger_separator_steps = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_STEPS", cls.logger_separator_steps) cls.logger_separator_tests = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_TESTS", cls.logger_separator_tests) cls.logger_separator_cases = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_CASES", cls.logger_separator_cases) cls.setup_logger() cls.log("{}Start of '{}': {}\n{}" .format(cls.logger_separator_cases, cls.current_case_name(), now(), cls.logger_separator_cases)) # test execution configs cls.WEAVER_TEST_REQUEST_TIMEOUT = int(os.getenv("WEAVER_TEST_REQUEST_TIMEOUT", 10)) cls.WEAVER_TEST_JOB_ACCEPTED_MAX_TIMEOUT = int(os.getenv("WEAVER_TEST_JOB_ACCEPTED_MAX_TIMEOUT", 30)) cls.WEAVER_TEST_JOB_RUNNING_MAX_TIMEOUT = int(os.getenv("WEAVER_TEST_JOB_RUNNING_MAX_TIMEOUT", 6000)) cls.WEAVER_TEST_JOB_GET_STATUS_INTERVAL = int(os.getenv("WEAVER_TEST_JOB_GET_STATUS_INTERVAL", 5)) # security configs if enabled cls.WEAVER_TEST_PROTECTED_ENABLED = asbool(os.getenv("WEAVER_TEST_PROTECTED_ENABLED", False)) cls.WEAVER_TEST_WSO2_CLIENT_HOSTNAME = os.getenv("WEAVER_TEST_WSO2_CLIENT_HOSTNAME", "") cls.WEAVER_TEST_WSO2_CLIENT_ID = os.getenv("WEAVER_TEST_WSO2_CLIENT_ID", "") cls.WEAVER_TEST_WSO2_CLIENT_SECRET = os.getenv("WEAVER_TEST_WSO2_CLIENT_SECRET", "") cls.WEAVER_TEST_WSO2_URL = os.getenv("WEAVER_TEST_WSO2_URL", "") cls.WEAVER_TEST_MAGPIE_URL = os.getenv("WEAVER_TEST_MAGPIE_URL", "") cls.WEAVER_TEST_ADMIN_CREDENTIALS = {"username": get_setting("ADMIN_USERNAME", cls.app), "password": get_setting("ADMIN_PASSWORD", cls.app)} cls.WEAVER_TEST_ALICE_CREDENTIALS = {"username": get_setting("ALICE_USERNAME", cls.app), "password": get_setting("ALICE_PASSWORD", cls.app)} cls.WEAVER_TEST_BOB_CREDENTIALS = {"username": get_setting("BOD_USERNAME", cls.app), "password": get_setting("BOB_PASSWORD", cls.app)} # server settings cls.WEAVER_TEST_SERVER_HOSTNAME = os.getenv("WEAVER_TEST_SERVER_HOSTNAME") cls.WEAVER_TEST_SERVER_BASE_PATH = os.getenv("WEAVER_TEST_SERVER_BASE_PATH", "/weaver") cls.WEAVER_TEST_SERVER_API_PATH = os.getenv("WEAVER_TEST_SERVER_API_PATH", "/") cls.WEAVER_TEST_CONFIG_INI_PATH = os.getenv("WEAVER_TEST_CONFIG_INI_PATH") # none uses default path cls.app = WebTestApp(cls.WEAVER_TEST_SERVER_HOSTNAME) cls.WEAVER_URL = get_weaver_url(cls.settings()) cls.WEAVER_RESTAPI_URL = get_wps_restapi_base_url(cls.settings()) # validation cls.validate_test_server() cls.setup_test_processes()
def retrieve_data_source_url(data_source): # type: (Optional[Text]) -> str """ Finds the data source URL using the provided data source identifier. :returns: found URL, 'default' data source if not found, or current weaver WPS Rest API base URL if `None`. """ if data_source is None: # get local data source return get_wps_restapi_base_url(get_settings()) data_sources = fetch_data_sources() return data_sources[data_source if data_source in data_sources else get_default_data_source(data_sources)]["ades"]
def register_builtin_processes(container): # type: (AnySettingsContainer) -> None """ Registers every ``builtin`` CWL package to the processes database. CWL definitions must be located within the :mod:`weaver.processes.builtin` module. """ restapi_url = get_wps_restapi_base_url(container) builtin_apps_mapping = _get_builtin_reference_mapping(os.path.abspath(os.path.dirname(__file__))) builtin_processes = [] for process_id, process_path in builtin_apps_mapping.items(): process_info = get_process_definition({}, package=None, reference=process_path) process_url = "/".join([restapi_url, "processes", process_id]) process_package = _get_builtin_package(process_id, process_info["package"]) process_abstract = _get_builtin_metadata(process_id, process_path, "__doc__", clean=True) process_version = _get_builtin_metadata(process_id, process_path, "__version__") process_title = _get_builtin_metadata(process_id, process_path, "__title__") process_payload = { "processDescription": { "process": { "id": process_id, "type": PROCESS_BUILTIN, "title": process_title, "version": process_version, "abstract": process_abstract, } }, "deploymentProfileName": "http://www.opengis.net/profiles/eoc/builtinApplication", "executionUnit": [{"unit": process_package}], } process_payload["processDescription"]["process"].update(ows_context_href(process_url)) builtin_processes.append(Process( id=process_id, type=PROCESS_BUILTIN, title=process_title, version=process_version, abstract=process_abstract, payload=process_payload, package=process_package, inputs=process_info["inputs"], outputs=process_info["outputs"], processDescriptionURL=process_url, processEndpointWPS1=get_wps_url(container), executeEndpoint="/".join([process_url, "jobs"]), visibility=VISIBILITY_PUBLIC, )) # registration of missing/updated apps automatically applied with 'default_processes' get_db(container).get_store(StoreProcesses, default_processes=builtin_processes)
def api_frontpage_body(settings): # type: (SettingsType) -> JSON """ Generates the JSON body describing the Weaver API and documentation references. """ # import here to avoid circular import errors from weaver.config import get_weaver_configuration weaver_url = get_weaver_url(settings) weaver_config = get_weaver_configuration(settings) weaver_api = asbool(settings.get("weaver.wps_restapi")) weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None weaver_api_oas_ui = weaver_api_url + sd.api_openapi_ui_service.path if weaver_api else None weaver_api_swagger = weaver_api_url + sd.api_swagger_ui_service.path if weaver_api else None weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None weaver_api_spec = weaver_api_url + sd.openapi_json_service.path if weaver_api else None weaver_wps = asbool(settings.get("weaver.wps")) weaver_wps_url = get_wps_url(settings) if weaver_wps else None weaver_conform_url = weaver_url + sd.api_conformance_service.path weaver_process_url = weaver_url + sd.processes_service.path weaver_jobs_url = weaver_url + sd.jobs_service.path weaver_links = [{ "href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This landing page." }, { "href": weaver_conform_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/conformance", "type": CONTENT_TYPE_APP_JSON, "title": "Conformance classes implemented by this service." }, { "href": __meta__.__license_url__, "rel": "license", "type": CONTENT_TYPE_TEXT_PLAIN, "title": __meta__.__license_long__ }] if weaver_api: weaver_links.extend([ { "href": weaver_api_url, "rel": "service", "type": CONTENT_TYPE_APP_JSON, "title": "WPS REST API endpoint of this service." }, { "href": weaver_api_spec, "rel": "service-desc", "type": CONTENT_TYPE_APP_JSON, "title": "OpenAPI specification of this service." }, { "href": weaver_api_oas_ui, "rel": "service-doc", "type": CONTENT_TYPE_TEXT_HTML, "title": "Human readable OpenAPI documentation of this service." }, { "href": weaver_api_spec, "rel": "OpenAPI", "type": CONTENT_TYPE_APP_JSON, "title": "OpenAPI specification of this service." }, { "href": weaver_api_swagger, "rel": "swagger-ui", "type": CONTENT_TYPE_TEXT_HTML, "title": "WPS REST API definition of this service." }, { "href": weaver_process_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/processes", "type": CONTENT_TYPE_APP_JSON, "title": "Processes offered by this service." }, { "href": sd.OGC_API_REPO_URL, "rel": "ogcapi-processes-repository", "type": CONTENT_TYPE_TEXT_HTML, "title": "OGC API - Processes schema definitions repository." }, { "href": weaver_jobs_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/job-list", "type": CONTENT_TYPE_APP_JSON, "title": "Job search and listing endpoint of executions registered under this service." }, { "href": sd.CWL_BASE_URL, "rel": "cwl-home", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) homepage." }, { "href": sd.CWL_REPO_URL, "rel": "cwl-repository", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) repositories." }, { "href": sd.CWL_SPEC_URL, "rel": "cwl-specification", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) specification." }, { "href": sd.CWL_USER_GUIDE_URL, "rel": "cwl-user-guide", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) user guide." }, { "href": sd.CWL_CMD_TOOL_URL, "rel": "cwl-command-line-tool", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) CommandLineTool specification." }, { "href": sd.CWL_WORKFLOW_URL, "rel": "cwl-workflow", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) Workflow specification." }, ]) if weaver_api_ref: # sample: # https://app.swaggerhub.com/apis/geoprocessing/WPS/ weaver_links.append({ "href": weaver_api_ref, "rel": "reference", "type": CONTENT_TYPE_APP_JSON, "title": "API reference specification of this service." }) if isinstance(weaver_api_doc, str): # sample: # https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/develop/docs/18-062.pdf if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test ext_type = weaver_api_doc.split(".")[-1] doc_type = "application/{}".format(ext_type) else: doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type weaver_links.append({ "href": weaver_api_doc, "rel": "documentation", "type": doc_type, "title": "API reference documentation about this service." }) else: weaver_links.append({ "href": __meta__.__documentation_url__, "rel": "documentation", "type": CONTENT_TYPE_TEXT_HTML, "title": "API reference documentation about this service." }) if weaver_wps: weaver_links.extend([ { "href": weaver_wps_url, "rel": "wps", "type": CONTENT_TYPE_TEXT_XML, "title": "WPS 1.0.0/2.0 XML endpoint of this service." }, { "href": "http://docs.opengeospatial.org/is/14-065/14-065.html", "rel": "wps-specification", "type": CONTENT_TYPE_TEXT_HTML, "title": "WPS 1.0.0/2.0 definition of this service." }, { "href": "http://schemas.opengis.net/wps/", "rel": "wps-schema-repository", "type": CONTENT_TYPE_TEXT_HTML, "title": "WPS 1.0.0/2.0 XML schemas repository." }, { "href": "http://schemas.opengis.net/wps/1.0.0/wpsAll.xsd", "rel": "wps-schema-1", "type": CONTENT_TYPE_TEXT_XML, "title": "WPS 1.0.0 XML validation schemas entrypoint." }, { "href": "http://schemas.opengis.net/wps/2.0/wps.xsd", "rel": "wps-schema-2", "type": CONTENT_TYPE_TEXT_XML, "title": "WPS 2.0 XML validation schemas entrypoint." }, ]) return { "message": "Weaver Information", "configuration": weaver_config, "description": __meta__.__description__, "parameters": [ { "name": "api", "enabled": weaver_api, "url": weaver_api_url, "api": weaver_api_oas_ui }, { "name": "wps", "enabled": weaver_wps, "url": weaver_wps_url }, ], "links": weaver_links, }
def deploy_process_from_payload(payload, container): # type: (JSON, AnyContainer) -> HTTPException """ Adds a :class:`weaver.datatype.Process` instance to storage using the provided JSON ``payload`` matching :class:`weaver.wps_restapi.swagger_definitions.ProcessDescription`. :returns: HTTPOk if the process registration was successful :raises HTTPException: otherwise """ _check_deploy(payload) # use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload payload_copy = deepcopy(payload) # validate identifier naming for unsupported characters process_description = payload.get("processDescription") process_info = process_description.get("process", {}) process_href = process_description.pop("href", None) # retrieve CWL package definition, either via "href" (WPS-1/2), "owsContext" or "executionUnit" (package/reference) deployment_profile_name = payload.get("deploymentProfileName", "").lower() ows_context = process_info.pop("owsContext", None) reference = None package = None if process_href: reference = process_href # reference type handled downstream elif isinstance(ows_context, dict): offering = ows_context.get("offering") if not isinstance(offering, dict): raise HTTPUnprocessableEntity( "Invalid parameter 'processDescription.process.owsContext.offering'." ) content = offering.get("content") if not isinstance(content, dict): raise HTTPUnprocessableEntity( "Invalid parameter 'processDescription.process.owsContext.offering.content'." ) package = None reference = content.get("href") elif deployment_profile_name: if not any( deployment_profile_name.endswith(typ) for typ in [PROCESS_APPLICATION, PROCESS_WORKFLOW]): raise HTTPBadRequest( "Invalid value for parameter 'deploymentProfileName'.") execution_units = payload.get("executionUnit") if not isinstance(execution_units, list): raise HTTPUnprocessableEntity("Invalid parameter 'executionUnit'.") for execution_unit in execution_units: if not isinstance(execution_unit, dict): raise HTTPUnprocessableEntity( "Invalid parameter 'executionUnit'.") package = execution_unit.get("unit") reference = execution_unit.get("href") # stop on first package/reference found, simultaneous usage will raise during package retrieval if package or reference: break else: raise HTTPBadRequest( "Missing one of required parameters [href, owsContext, deploymentProfileName]." ) # obtain updated process information using WPS process offering, CWL/WPS reference or CWL package definition process_info = _get_deploy_process_info(process_info, reference, package) # validate process type against weaver configuration settings = get_settings(container) process_type = process_info["type"] if process_type == PROCESS_WORKFLOW: weaver_config = get_weaver_configuration(settings) if weaver_config != WEAVER_CONFIGURATION_EMS: raise HTTPBadRequest( "Invalid [{0}] package deployment on [{1}].".format( process_type, weaver_config)) restapi_url = get_wps_restapi_base_url(settings) description_url = "/".join( [restapi_url, "processes", process_info["identifier"]]) execute_endpoint = "/".join([description_url, "jobs"]) # ensure that required "processEndpointWPS1" in db is added, # will be auto-fixed to localhost if not specified in body process_info["processEndpointWPS1"] = process_description.get( "processEndpointWPS1") process_info["executeEndpoint"] = execute_endpoint process_info["payload"] = payload_copy process_info["jobControlOptions"] = process_description.get( "jobControlOptions", []) process_info["outputTransmission"] = process_description.get( "outputTransmission", []) process_info["processDescriptionURL"] = description_url # insert the "resolved" context using details retrieved from "executionUnit"/"href" or directly with "owsContext" if "owsContext" not in process_info and reference: process_info["owsContext"] = { "offering": { "content": { "href": str(reference) } } } elif isinstance(ows_context, dict): process_info["owsContext"] = ows_context try: store = get_db(container).get_store(StoreProcesses) saved_process = store.save_process(Process(process_info), overwrite=False) except ProcessRegistrationError as ex: raise HTTPConflict(detail=str(ex)) except ValueError as ex: # raised on invalid process name raise HTTPBadRequest(detail=str(ex)) json_response = { "processSummary": saved_process.process_summary(), "deploymentDone": True } return HTTPOk( json=json_response ) # FIXME: should be 201 (created), update swagger accordingly
def deploy_process_from_payload(payload, container, overwrite=False): # type: (JSON, AnyContainer, bool) -> HTTPException """ Deploy the process after resolution of all references and validation of the parameters from payload definition. Adds a :class:`weaver.datatype.Process` instance to storage using the provided JSON ``payload`` matching :class:`weaver.wps_restapi.swagger_definitions.ProcessDescription`. :param payload: JSON payload that was specified during the process deployment request. :param container: container to retrieve application settings. :param overwrite: whether to allow override of an existing process definition if conflict occurs. :returns: HTTPOk if the process registration was successful. :raises HTTPException: for any invalid process deployment step. """ # use deepcopy of to remove any circular dependencies before writing to mongodb or any updates to the payload payload_copy = deepcopy(payload) payload = _check_deploy(payload) # validate identifier naming for unsupported characters process_description = payload.get("processDescription") process_info = process_description.get("process", {}) process_href = process_description.pop("href", None) # retrieve CWL package definition, either via "href" (WPS-1/2), "owsContext" or "executionUnit" (package/reference) deployment_profile_name = payload.get("deploymentProfileName", "").lower() ows_context = process_info.pop("owsContext", None) reference = None package = None if process_href: reference = process_href # reference type handled downstream elif isinstance(ows_context, dict): offering = ows_context.get("offering") if not isinstance(offering, dict): raise HTTPUnprocessableEntity( "Invalid parameter 'processDescription.process.owsContext.offering'." ) content = offering.get("content") if not isinstance(content, dict): raise HTTPUnprocessableEntity( "Invalid parameter 'processDescription.process.owsContext.offering.content'." ) package = None reference = content.get("href") elif deployment_profile_name: if not any( deployment_profile_name.endswith(typ) for typ in [PROCESS_APPLICATION, PROCESS_WORKFLOW]): raise HTTPBadRequest( "Invalid value for parameter 'deploymentProfileName'.") execution_units = payload.get("executionUnit") if not isinstance(execution_units, list): raise HTTPUnprocessableEntity("Invalid parameter 'executionUnit'.") for execution_unit in execution_units: if not isinstance(execution_unit, dict): raise HTTPUnprocessableEntity( "Invalid parameter 'executionUnit'.") package = execution_unit.get("unit") reference = execution_unit.get("href") # stop on first package/reference found, simultaneous usage will raise during package retrieval if package or reference: break else: raise HTTPBadRequest( "Missing one of required parameters [href, owsContext, deploymentProfileName]." ) if process_info.get("type", "") == PROCESS_BUILTIN: raise HTTPBadRequest( "Invalid process type resolved from package: [{0}]. Deployment of {0} process is not allowed." .format(PROCESS_BUILTIN)) # update and validate process information using WPS process offering, CWL/WPS reference or CWL package definition settings = get_settings(container) headers = getattr( container, "headers", {} ) # container is any request (as when called from API Deploy request) process_info = _validate_deploy_process_info(process_info, reference, package, settings, headers) restapi_url = get_wps_restapi_base_url(settings) description_url = "/".join( [restapi_url, "processes", process_info["identifier"]]) execute_endpoint = "/".join([description_url, "jobs"]) # ensure that required "processEndpointWPS1" in db is added, # will be auto-fixed to localhost if not specified in body process_info["processEndpointWPS1"] = process_description.get( "processEndpointWPS1") process_info["executeEndpoint"] = execute_endpoint process_info["payload"] = payload_copy process_info["jobControlOptions"] = process_description.get( "jobControlOptions", []) process_info["outputTransmission"] = process_description.get( "outputTransmission", []) process_info["processDescriptionURL"] = description_url # insert the "resolved" context using details retrieved from "executionUnit"/"href" or directly with "owsContext" if "owsContext" not in process_info and reference: process_info["owsContext"] = { "offering": { "content": { "href": str(reference) } } } elif isinstance(ows_context, dict): process_info["owsContext"] = ows_context # bw-compat abstract/description (see: ProcessDeployment schema) if "description" not in process_info or not process_info["description"]: process_info["description"] = process_info.get("abstract", "") # FIXME: handle colander invalid directly in tween (https://github.com/crim-ca/weaver/issues/112) try: store = get_db(container).get_store(StoreProcesses) process = Process(process_info) sd.ProcessSummary().deserialize( process) # make if fail before save if invalid store.save_process(process, overwrite=overwrite) process_summary = process.summary() except ProcessRegistrationError as ex: raise HTTPConflict(detail=str(ex)) except (ValueError, colander.Invalid) as ex: # raised on invalid process name raise HTTPBadRequest(detail=str(ex)) return HTTPCreated( json={ "description": sd.OkPostProcessesResponse.description, "processSummary": process_summary, "deploymentDone": True })
def _job_url(self, settings): base_job_url = get_wps_restapi_base_url(settings) if self.service is not None: base_job_url += sd.provider_uri.format(provider_id=self.service) job_path = sd.process_job_uri.format(process_id=self.process, job_id=self.id) return "{base_job_url}{job_path}".format(base_job_url=base_job_url, job_path=job_path)