def mocked_app_request(method, url=None, session=None, **req_kwargs): """ Mock requests under the web test application under specific conditions. Request corresponding to :func:`requests.request` that instead gets executed by :class:`webTest.TestApp`, unless permitted to call real external requests. """ # if URL starts with '/' directly, it is the shorthand path for this test app, always mock # otherwise, filter according to full URL hostname url_test_app = get_weaver_url(app.app.registry) if only_local and not url.startswith("/") and not url.startswith( url_test_app): with session or RealSession() as request_session: return real_request(request_session, method, url, **req_kwargs) url, func, req_kwargs = _parse_for_app_req(method, url, **req_kwargs) redirects = req_kwargs.pop("allow_redirects", True) if url.startswith("mock://"): path = get_url_without_query(url.replace("mock://", "")) _resp = mocked_file_response(path, url) else: _resp = func(url, expect_errors=True, **req_kwargs) if redirects: # must handle redirects manually with TestApp while 300 <= _resp.status_code < 400: _resp = _resp.follow() _patch_response_methods(_resp, url) return _resp
def _describe_process_redirect(self, wps_request, *_, **__): # type: (WPSRequest, *Any, **Any) -> Optional[Union[WPSResponse, HTTPValid]] """ Redirects to WPS-REST endpoint if requested ``Content-Type`` is JSON. """ req = wps_request.http_request req = extend_instance(req, PyramidRequest) # apply query 'params' method accept_type = guess_target_format(req, default=None) if accept_type == ContentType.APP_JSON: url = get_weaver_url(self.settings) proc = wps_request.identifiers if not proc: raise HTTPBadRequest( sd.BadRequestGetProcessInfoResponse.description) if len(proc) > 1: raise HTTPBadRequest( "Unsupported multi-process ID for description. Only provide one." ) path = sd.process_service.path.format(process_id=proc[0]) resp = HTTPSeeOther(location=f"{url}{path}") # redirect setattr( resp, "_update_status", lambda *_, **__: None) # patch to avoid pywps server raising return resp return None
def get_wps_url(container, load=True): # type: (AnySettingsContainer, bool) -> str """ Retrieves the full WPS URL (hostname + WPS path). Searches directly in settings, then `weaver.wps_cfg` file, or finally, uses the default values if not found. """ return get_settings(container).get("weaver.wps_url") or get_weaver_url( container) + get_wps_path(container, load)
def get_wps_restapi_base_url(container): # type: (AnySettingsContainer) -> str settings = get_settings(container) weaver_rest_url = settings.get("weaver.wps_restapi_url") if not weaver_rest_url: weaver_url = get_weaver_url(settings) restapi_path = wps_restapi_base_path(settings) weaver_rest_url = weaver_url + restapi_path return weaver_rest_url.rstrip("/").strip()
def api_swagger_json(request): # noqa: F811 # type: (Request) -> dict """weaver REST API schema generation in JSON format.""" # obtain 'server' host and api-base-path, which doesn't correspond necessarily to the app's host and path # ex: 'server' adds '/weaver' with proxy redirect before API routes weaver_server_url = get_weaver_url(request) LOGGER.debug("Request app URL: [%s]", request.url) LOGGER.debug("Weaver config URL: [%s]", weaver_server_url) # http_scheme=request.scheme, http_host=request.host return get_swagger_json(base_url=weaver_server_url, use_docstring_summary=True)
def api_frontpage(request): """Frontpage of weaver.""" # import here to avoid circular import errors from weaver.config import get_weaver_configuration settings = get_settings(request) weaver_url = get_weaver_url(settings) weaver_config = get_weaver_configuration(settings) weaver_api = asbool(settings.get("weaver.wps_restapi")) weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None weaver_api_def = weaver_api_url + sd.api_swagger_ui_uri if weaver_api else None weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None weaver_wps = asbool(settings.get("weaver.wps")) weaver_wps_url = get_wps_url(settings) if weaver_wps else None weaver_conform_url = weaver_url + sd.api_conformance_uri weaver_process_url = weaver_url + sd.processes_uri weaver_links = [ {"href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This document"}, {"href": weaver_conform_url, "rel": "conformance", "type": CONTENT_TYPE_APP_JSON, "title": "WPS 2.0/3.0 REST-JSON Binding Extension conformance classes implemented by this service."}, ] if weaver_api_def: weaver_links.append({"href": weaver_api_def, "rel": "service", "type": CONTENT_TYPE_APP_JSON, "title": "API definition of this service."}) if isinstance(weaver_api_doc, str): if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test ext_type = weaver_api_doc.split(".")[-1] doc_type = "application/{}".format(ext_type) else: doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type weaver_links.append({"href": weaver_api_doc, "rel": "documentation", "type": doc_type, "title": "API documentation about this service."}) if weaver_api_ref: weaver_links.append({"href": weaver_api_ref, "rel": "reference", "type": CONTENT_TYPE_APP_JSON, "title": "API reference specification of this service."}) weaver_links.append({"href": weaver_process_url, "rel": "processes", "type": CONTENT_TYPE_APP_JSON, "title": "Processes offered by this service."}) return { "message": "Weaver Information", "configuration": weaver_config, "parameters": [ {"name": "api", "enabled": weaver_api, "url": weaver_api_url, "doc": weaver_api_doc, "api": weaver_api_def, "ref": weaver_api_ref}, {"name": "wps", "enabled": weaver_wps, "url": weaver_wps_url}, ], "links": weaver_links, }
def setUpClass(cls): # disable SSL warnings from logs try: import urllib3 # noqa urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) except ImportError: pass # logging parameter overrides cls.logger_level = os.getenv("WEAVER_TEST_LOGGER_LEVEL", cls.logger_level) cls.logger_enabled = asbool(os.getenv("WEAVER_TEST_LOGGER_ENABLED", cls.logger_enabled)) cls.logger_result_dir = os.getenv("WEAVER_TEST_LOGGER_RESULT_DIR", os.path.join(WEAVER_ROOT_DIR)) cls.logger_json_indent = os.getenv("WEAVER_TEST_LOGGER_JSON_INDENT", cls.logger_json_indent) cls.logger_field_indent = os.getenv("WEAVER_TEST_LOGGER_FIELD_INDENT", cls.logger_field_indent) cls.logger_separator_calls = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_CALLS", cls.logger_separator_calls) cls.logger_separator_steps = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_STEPS", cls.logger_separator_steps) cls.logger_separator_tests = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_TESTS", cls.logger_separator_tests) cls.logger_separator_cases = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_CASES", cls.logger_separator_cases) cls.setup_logger() cls.log("{}Start of '{}': {}\n{}" .format(cls.logger_separator_cases, cls.current_case_name(), now(), cls.logger_separator_cases)) # test execution configs cls.WEAVER_TEST_REQUEST_TIMEOUT = int(os.getenv("WEAVER_TEST_REQUEST_TIMEOUT", 10)) cls.WEAVER_TEST_JOB_ACCEPTED_MAX_TIMEOUT = int(os.getenv("WEAVER_TEST_JOB_ACCEPTED_MAX_TIMEOUT", 30)) cls.WEAVER_TEST_JOB_RUNNING_MAX_TIMEOUT = int(os.getenv("WEAVER_TEST_JOB_RUNNING_MAX_TIMEOUT", 6000)) cls.WEAVER_TEST_JOB_GET_STATUS_INTERVAL = int(os.getenv("WEAVER_TEST_JOB_GET_STATUS_INTERVAL", 5)) # security configs if enabled cls.WEAVER_TEST_PROTECTED_ENABLED = asbool(os.getenv("WEAVER_TEST_PROTECTED_ENABLED", False)) cls.WEAVER_TEST_WSO2_CLIENT_HOSTNAME = os.getenv("WEAVER_TEST_WSO2_CLIENT_HOSTNAME", "") cls.WEAVER_TEST_WSO2_CLIENT_ID = os.getenv("WEAVER_TEST_WSO2_CLIENT_ID", "") cls.WEAVER_TEST_WSO2_CLIENT_SECRET = os.getenv("WEAVER_TEST_WSO2_CLIENT_SECRET", "") cls.WEAVER_TEST_WSO2_URL = os.getenv("WEAVER_TEST_WSO2_URL", "") cls.WEAVER_TEST_MAGPIE_URL = os.getenv("WEAVER_TEST_MAGPIE_URL", "") cls.WEAVER_TEST_ADMIN_CREDENTIALS = {"username": get_setting("ADMIN_USERNAME", cls.app), "password": get_setting("ADMIN_PASSWORD", cls.app)} cls.WEAVER_TEST_ALICE_CREDENTIALS = {"username": get_setting("ALICE_USERNAME", cls.app), "password": get_setting("ALICE_PASSWORD", cls.app)} cls.WEAVER_TEST_BOB_CREDENTIALS = {"username": get_setting("BOD_USERNAME", cls.app), "password": get_setting("BOB_PASSWORD", cls.app)} # server settings cls.WEAVER_TEST_SERVER_HOSTNAME = os.getenv("WEAVER_TEST_SERVER_HOSTNAME") cls.WEAVER_TEST_SERVER_BASE_PATH = os.getenv("WEAVER_TEST_SERVER_BASE_PATH", "/weaver") cls.WEAVER_TEST_SERVER_API_PATH = os.getenv("WEAVER_TEST_SERVER_API_PATH", "/") cls.WEAVER_TEST_CONFIG_INI_PATH = os.getenv("WEAVER_TEST_CONFIG_INI_PATH") # none uses default path cls.app = WebTestApp(cls.WEAVER_TEST_SERVER_HOSTNAME) cls.WEAVER_URL = get_weaver_url(cls.settings()) cls.WEAVER_RESTAPI_URL = get_wps_restapi_base_url(cls.settings()) # validation cls.validate_test_server() cls.setup_test_processes()
def redoc_ui_cached(request): settings = get_settings(request) weaver_server_url = get_weaver_url(settings) spec = openapi_json_cached(base_url=weaver_server_url, settings=settings, use_docstring_summary=True, use_refs=False) data_mako = {"openapi_spec": json.dumps(spec, ensure_ascii=False)} resp = render_to_response("templates/redoc_ui.mako", data_mako, request=request) return resp
def get_vault_url(file, container=None): # type: (Union[VaultFile, uuid.UUID, str], Optional[AnySettingsContainer]) -> str """ Obtain the vault link corresponding to the file. """ if isinstance(file, uuid.UUID) or is_uuid(file): file_id = str(file) else: file_id = file.id settings = get_settings(container) base_url = get_weaver_url(settings) vault_url = base_url + sd.vault_file_service.path.format(file_id=file_id) return vault_url
def get_wps_output_url(container): # type: (AnySettingsContainer) -> str """ Retrieves the WPS output URL that maps to WPS output directory path. Searches directly in settings, then `weaver.wps_cfg` file, or finally, uses the default values if not found. """ wps_output_default = get_weaver_url(container) + "/wpsoutputs" wps_output_config = _get_settings_or_wps_config(container, "weaver.wps_output_url", "server", "outputurl", wps_output_default, "WPS output url") return wps_output_config or wps_output_default
def __init__(self, *args, **kwargs): db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs) StoreProcesses.__init__(self) MongodbStore.__init__(self, *db_args, **db_kwargs) registry = kwargs.get("registry") default_processes = kwargs.get("default_processes") self.settings = kwargs.get("settings", {}) if not registry else registry.settings self.default_host = get_weaver_url(self.settings) self.default_wps_endpoint = get_wps_url(self.settings) # enforce default process re-registration to receive any applicable update if default_processes: self._register_defaults(default_processes)
def openapi_json(request): # noqa: F811 # type: (Request) -> dict """ Weaver OpenAPI schema definitions. """ # obtain 'server' host and api-base-path, which doesn't correspond necessarily to the app's host and path # ex: 'server' adds '/weaver' with proxy redirect before API routes settings = get_settings(request) weaver_server_url = get_weaver_url(settings) LOGGER.debug("Request app URL: [%s]", request.url) LOGGER.debug("Weaver config URL: [%s]", weaver_server_url) return openapi_json_cached(base_url=weaver_server_url, use_docstring_summary=True, settings=settings)
def _get_capabilities_redirect(self, wps_request, *_, **__): # type: (WPSRequest, Any, Any) -> Optional[Union[WPSResponse, HTTPValid]] """ Redirects to WPS-REST endpoint if requested ``Content-Type`` is JSON. """ req = wps_request.http_request accept_type = get_header("Accept", req.headers) if accept_type == CONTENT_TYPE_APP_JSON: url = get_weaver_url(self.settings) resp = HTTPSeeOther(location="{}{}".format( url, sd.processes_uri)) # redirect setattr( resp, "_update_status", lambda *_, **__: None) # patch to avoid pywps server raising return resp return None
def __init__(self, *args, **kwargs): db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs) StoreProcesses.__init__(self) MongodbStore.__init__(self, *db_args, **db_kwargs) registry = kwargs.get("registry") settings = kwargs.get("settings", {}) if not registry else registry.settings default_processes = kwargs.get("default_processes") self.default_host = get_weaver_url(settings) self.default_wps_endpoint = get_wps_url(settings) # enforce default process re-registration to receive any applicable update if default_processes: registered_processes = [process.identifier for process in self.list_processes()] for process in default_processes: process_name = self._get_process_id(process) if process_name in registered_processes: self.delete_process(process_name) self._add_process(process)
def mocked_app_request(method, url=None, **req_kwargs): """ Request corresponding to :func:`requests.request` that instead gets executed by :class:`webTest.TestApp`, unless permitted to call real external requests. """ # if URL starts with '/' directly, it is the shorthand path for this test app, always mock # otherwise, filter according to full URL hostname url_test_app = get_weaver_url(app.app.registry) if only_local and not url.startswith("/") and not url.startswith( url_test_app): with RealSession() as session: return real_request(session, method, url, **req_kwargs) url, func, req_kwargs = _parse_for_app_req(method, url, **req_kwargs) if not url.startswith("mock://"): resp = func(url, expect_errors=True, **req_kwargs) setattr(resp, "content", resp.body) else: path = get_url_without_query(url.replace("mock://", "")) resp = mocked_file_response(path, url) return resp
def _describe_process_redirect(self, wps_request, *_, **__): # type: (WPSRequest, Any, Any) -> Optional[Union[WPSResponse, HTTPValid]] """ Redirects to WPS-REST endpoint if requested ``Content-Type`` is JSON. """ req = wps_request.http_request accept_type = get_header("Accept", req.headers) if accept_type == CONTENT_TYPE_APP_JSON: url = get_weaver_url(self.settings) proc = wps_request.identifiers if not proc: raise HTTPBadRequest( sd.BadRequestGetProcessInfoResponse.description) if len(proc) > 1: raise HTTPBadRequest( "Unsupported multi-process ID for description. Only provide one." ) path = sd.process_uri.format(process_id=proc[0]) resp = HTTPSeeOther(location="{}{}".format(url, path)) # redirect setattr( resp, "_update_status", lambda *_, **__: None) # patch to avoid pywps server raising return resp return None
def get_job_list_links(job_total, filters, request): # type: (int, Dict[str, AnyValueType], Request) -> List[JSON] """ Obtains a list of all relevant links for the corresponding job listing defined by query parameter filters. :raises IndexError: if the paging values are out of bounds compared to available total :term:`Job` matching search. """ base_url = get_weaver_url(request) # reapply queries that must be given to obtain the same result in case of subsequent requests (sort, limits, etc.) kvp_params = { param: value for param, value in request.params.items() if param != "page" } # patch datetime that have some extra character manipulation (reapply '+' auto-converted to ' ' by params parser) if "datetime" in kvp_params: kvp_params["datetime"] = kvp_params["datetime"].replace(" ", "+") alt_kvp = deepcopy(kvp_params) # request job uses general endpoint, obtain the full path if any service/process was given as alternate location if request.path.startswith(sd.jobs_service.path): job_path = base_url + sd.jobs_service.path alt_path = None parent_url = None # cannot generate full path apply for 'service' by itself if filters["process"] and filters["service"]: alt_path = base_url + sd.provider_jobs_service.path.format( provider_id=filters["service"], process_id=filters["process"]) parent_url = alt_path.rsplit("/", 1)[0] elif filters["process"]: alt_path = base_url + sd.process_jobs_service.path.format( process_id=filters["process"]) parent_url = alt_path.rsplit("/", 1)[0] for param in ["service", "provider", "process"]: alt_kvp.pop(param, None) # path is whichever specific service/process endpoint, jobs are pre-filtered by them # transform sub-endpoints into matching query parameters and use generic path as alternate location else: job_path = base_url + request.path alt_path = base_url + sd.jobs_service.path alt_kvp["process"] = filters["process"] if filters["service"]: alt_kvp["provider"] = filters["service"] parent_url = job_path.rsplit("/", 1)[0] cur_page = filters["page"] per_page = filters["limit"] max_page = max(math.ceil(job_total / per_page) - 1, 0) if cur_page < 0 or cur_page > max_page: raise IndexError( f"Page index {cur_page} is out of range from [0,{max_page}].") alt_links = [] if alt_path: alt_links = [{ "href": get_path_kvp(alt_path, page=cur_page, **alt_kvp), "rel": "alternate", "type": CONTENT_TYPE_APP_JSON, "title": "Alternate endpoint with equivalent set of filtered jobs." }] links = alt_links + [ { "href": job_path, "rel": "collection", "type": CONTENT_TYPE_APP_JSON, "title": "Complete job listing (no filtering queries applied)." }, { "href": base_url + sd.jobs_service.path, "rel": "search", "type": CONTENT_TYPE_APP_JSON, "title": "Generic query endpoint to search for jobs." }, { "href": job_path + "?detail=false", "rel": "preview", "type": CONTENT_TYPE_APP_JSON, "title": "Job listing summary (UUID and count only)." }, { "href": job_path, "rel": "http://www.opengis.net/def/rel/ogc/1.0/job-list", "type": CONTENT_TYPE_APP_JSON, "title": "List of registered jobs." }, { "href": get_path_kvp(job_path, page=cur_page, **kvp_params), "rel": "current", "type": CONTENT_TYPE_APP_JSON, "title": "Current page of job query listing." }, { "href": get_path_kvp(job_path, page=0, **kvp_params), "rel": "first", "type": CONTENT_TYPE_APP_JSON, "title": "First page of job query listing." }, { "href": get_path_kvp(job_path, page=max_page, **kvp_params), "rel": "last", "type": CONTENT_TYPE_APP_JSON, "title": "Last page of job query listing." }, ] if cur_page > 0: links.append({ "href": get_path_kvp(job_path, page=cur_page - 1, **kvp_params), "rel": "prev", "type": CONTENT_TYPE_APP_JSON, "title": "Previous page of job query listing." }) if cur_page < max_page: links.append({ "href": get_path_kvp(job_path, page=cur_page + 1, **kvp_params), "rel": "next", "type": CONTENT_TYPE_APP_JSON, "title": "Next page of job query listing." }) if parent_url: links.append({ "href": parent_url, "rel": "up", "type": CONTENT_TYPE_APP_JSON, "title": "Parent collection for which listed jobs apply." }) return links
def get_process_list_links(request, paging, total, provider=None): # type: (Request, Dict[str, int], Optional[int], Optional[Service]) -> List[JSON] """ Obtains a list of all relevant links for the corresponding :term:`Process` listing defined by query parameters. :raises IndexError: if the paging values are out of bounds compared to available total :term:`Process`. """ # reapply queries that must be given to obtain the same result in case of subsequent requests (sort, limits, etc.) kvp_params = { param: value for param, value in request.params.items() if param != "page" } base_url = get_weaver_url(request) links = [] if provider: proc_path = sd.provider_processes_service.path.format( provider_id=provider.id) links.extend(provider.links(request, self_link="provider")) else: proc_path = sd.processes_service.path proc_url = base_url + proc_path links.extend([ { "href": proc_url, "rel": "collection", "type": CONTENT_TYPE_APP_JSON, "title": "Process listing (no filtering queries applied)." }, { "href": proc_url, "rel": "search", "type": CONTENT_TYPE_APP_JSON, "title": "Generic query endpoint to list processes." }, { "href": proc_url + "?detail=false", "rel": "preview", "type": CONTENT_TYPE_APP_JSON, "title": "Process listing summary (identifiers and count only)." }, { "href": proc_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/processes", "type": CONTENT_TYPE_APP_JSON, "title": "List of registered local processes." }, { "href": get_path_kvp(proc_url, **request.params), "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "Current process listing." }, ]) if provider: prov_url = proc_url.rsplit("/", 1)[0] links.append({ "href": prov_url, "rel": "up", "type": CONTENT_TYPE_APP_JSON, "title": "Provider description." }) else: links.append({ "href": base_url, "rel": "up", "type": CONTENT_TYPE_APP_JSON, "title": "API entrypoint." }) cur_page = paging.get("page", None) per_page = paging.get("limit", None) if all(isinstance(num, int) for num in [cur_page, per_page, total]): max_page = max(math.ceil(total / per_page) - 1, 0) if cur_page < 0 or cur_page > max_page: raise IndexError( f"Page index {cur_page} is out of range from [0,{max_page}].") links.extend([ { "href": get_path_kvp(proc_url, page=cur_page, **kvp_params), "rel": "current", "type": CONTENT_TYPE_APP_JSON, "title": "Current page of processes query listing." }, { "href": get_path_kvp(proc_url, page=0, **kvp_params), "rel": "first", "type": CONTENT_TYPE_APP_JSON, "title": "First page of processes query listing." }, { "href": get_path_kvp(proc_url, page=max_page, **kvp_params), "rel": "last", "type": CONTENT_TYPE_APP_JSON, "title": "Last page of processes query listing." }, ]) if cur_page > 0: links.append({ "href": get_path_kvp(proc_url, page=cur_page - 1, **kvp_params), "rel": "prev", "type": CONTENT_TYPE_APP_JSON, "title": "Previous page of processes query listing." }) if cur_page < max_page: links.append({ "href": get_path_kvp(proc_url, page=cur_page + 1, **kvp_params), "rel": "next", "type": CONTENT_TYPE_APP_JSON, "title": "Next page of processes query listing." }) return links
def request_quote(request): """ Request a quotation for a process. """ settings = get_settings(request) weaver_config = get_weaver_configuration(settings) if weaver_config not in [ WEAVER_CONFIGURATION_ADES, WEAVER_CONFIGURATION_EMS ]: raise HTTPBadRequest( "Unsupported request for configuration '{}'.".format( weaver_config)) process_id = request.matchdict.get("process_id") process_store = get_db(request).get_store("processes") try: process = process_store.fetch_by_id(process_id) except ProcessNotFound: raise HTTPNotFound( "Could not find process with specified 'process_id'.") store = get_db(request).get_store(StoreQuotes) process_url = get_process_location(process_id, data_source=get_weaver_url(settings)) process_type = process.type process_params = dict() for param in ["inputs", "outputs", "mode", "response"]: if param in request.json: process_params[param] = request.json.pop(param) process_quote_info = process_quote_estimator(process) process_quote_info.update({ "process": process_id, "processParameters": process_params, "location": process_url, "user": str(request.authenticated_userid) }) # loop workflow sub-process steps to get individual quotes if process_type == PROCESS_WORKFLOW and weaver_config == WEAVER_CONFIGURATION_EMS: workflow_quotes = list() for step in get_package_workflow_steps(process_url): # retrieve quote from provider ADES # TODO: data source mapping process_step_url = get_process_location(step["reference"]) process_quote_url = "{}/quotations".format(process_step_url) subreq = request.copy() subreq.path_info = process_quote_url resp_json = request.invoke_subrequest(subreq).json() quote_json = resp_json["quote"] quote = store.save_quote(Quote(**quote_json)) workflow_quotes.append(quote.id) process_quote_info.update({"steps": workflow_quotes}) quote = store.save_quote(Quote(**process_quote_info)) return HTTPCreated(json={"quote": quote.json()}) # single application quotes (ADES or EMS) elif process_type == PROCESS_APPLICATION: quote = store.save_quote(Quote(**process_quote_info)) quote_json = quote.json() quote_json.pop("steps", None) return HTTPCreated(json={"quote": quote_json}) # error if not handled up to this point raise HTTPBadRequest( "Unsupported quoting process type '{0}' on '{1}'.".format( process_type, weaver_config))
def load_pywps_config(container, config=None): # type: (AnySettingsContainer, Optional[Union[str, Dict[str, str]]]) -> ConfigParser """ Loads and updates the PyWPS configuration using Weaver settings. """ settings = get_settings(container) if settings.get("weaver.wps_configured"): LOGGER.debug("Using preloaded internal Weaver WPS configuration.") return pywps_config.CONFIG LOGGER.info("Initial load of internal Weaver WPS configuration.") pywps_config.load_configuration([]) # load defaults pywps_config.CONFIG.set("logging", "db_echo", "false") if logging.getLevelName(pywps_config.CONFIG.get("logging", "level")) <= logging.DEBUG: pywps_config.CONFIG.set("logging", "level", "INFO") # update metadata LOGGER.debug("Updating WPS metadata configuration.") for setting_name, setting_value in settings.items(): if setting_name.startswith("weaver.wps_metadata"): pywps_setting = setting_name.replace("weaver.wps_metadata_", "") pywps_config.CONFIG.set("metadata:main", pywps_setting, setting_value) # add weaver configuration keyword if not already provided wps_keywords = pywps_config.CONFIG.get("metadata:main", "identification_keywords") weaver_mode = get_weaver_configuration(settings) if weaver_mode not in wps_keywords: wps_keywords += ("," if wps_keywords else "") + weaver_mode pywps_config.CONFIG.set("metadata:main", "identification_keywords", wps_keywords) # add additional config passed as dictionary of {'section.key': 'value'} if isinstance(config, dict): for key, value in config.items(): section, key = key.split(".") pywps_config.CONFIG.set(section, key, value) # cleanup alternative dict "PYWPS_CFG" which is not expected elsewhere if isinstance(settings.get("PYWPS_CFG"), dict): del settings["PYWPS_CFG"] # set accepted languages aligned with values provided by REST API endpoints # otherwise, execute request could fail due to languages considered not supported languages = ", ".join(AcceptLanguage.values()) LOGGER.debug("Setting WPS languages: [%s]", languages) pywps_config.CONFIG.set("server", "language", languages) LOGGER.debug("Updating WPS output configuration.") # find output directory from app config or wps config if "weaver.wps_output_dir" not in settings: output_dir = pywps_config.get_config_value("server", "outputpath") settings["weaver.wps_output_dir"] = output_dir # ensure the output dir exists if specified output_dir = get_wps_output_dir(settings) make_dirs(output_dir, exist_ok=True) # find output url from app config (path/url) or wps config (url only) # note: needs to be configured even when using S3 bucket since XML status is provided locally if "weaver.wps_output_url" not in settings: output_path = settings.get("weaver.wps_output_path", "").rstrip("/") if output_path and isinstance(output_path, str): output_url = os.path.join(get_weaver_url(settings), output_path.strip("/")) else: output_url = pywps_config.get_config_value("server", "outputurl") settings["weaver.wps_output_url"] = output_url # apply workdir if provided, otherwise use default if "weaver.wps_workdir" in settings: make_dirs(settings["weaver.wps_workdir"], exist_ok=True) pywps_config.CONFIG.set("server", "workdir", settings["weaver.wps_workdir"]) # configure S3 bucket if requested, storage of all process outputs # note: # credentials and default profile are picked up automatically by 'boto3' from local AWS configs or env vars # region can also be picked from there unless explicitly provided by weaver config # warning: # if we set `(server, storagetype, s3)`, ALL status (including XML) are stored to S3 # to preserve status locally, we set 'file' and override the storage instance during output rewrite in WpsPackage # we can still make use of the server configurations here to make this overridden storage auto-find its configs s3_bucket = settings.get("weaver.wps_output_s3_bucket") pywps_config.CONFIG.set("server", "storagetype", "file") # pywps_config.CONFIG.set("server", "storagetype", "s3") if s3_bucket: LOGGER.debug("Updating WPS S3 bucket configuration.") import boto3 from botocore.exceptions import ClientError s3 = boto3.client("s3") s3_region = settings.get("weaver.wps_output_s3_region", s3.meta.region_name) LOGGER.info( "Validating that S3 [Bucket=%s, Region=%s] exists or creating it.", s3_bucket, s3_region) try: s3.create_bucket( Bucket=s3_bucket, CreateBucketConfiguration={"LocationConstraint": s3_region}) LOGGER.info("S3 bucket for WPS output created.") except ClientError as exc: if exc.response.get("Error", {}).get("Code") != "BucketAlreadyExists": LOGGER.error("Failed setup of S3 bucket for WPS output: [%s]", exc) raise LOGGER.info("S3 bucket for WPS output already exists.") pywps_config.CONFIG.set("s3", "region", s3_region) pywps_config.CONFIG.set("s3", "bucket", s3_bucket) pywps_config.CONFIG.set( "s3", "public", "false") # don't automatically push results as publicly accessible pywps_config.CONFIG.set( "s3", "encrypt", "true") # encrypts data server-side, transparent from this side # enforce back resolved values onto PyWPS config pywps_config.CONFIG.set("server", "setworkdir", "true") pywps_config.CONFIG.set("server", "sethomedir", "true") pywps_config.CONFIG.set("server", "outputpath", settings["weaver.wps_output_dir"]) pywps_config.CONFIG.set("server", "outputurl", settings["weaver.wps_output_url"]) pywps_config.CONFIG.set("server", "url", get_wps_url(settings, load=False)) settings["weaver.wps_configured"] = True return pywps_config.CONFIG
def api_frontpage_body(settings): # type: (SettingsType) -> JSON """ Generates the JSON body describing the Weaver API and documentation references. """ # import here to avoid circular import errors from weaver.config import get_weaver_configuration weaver_url = get_weaver_url(settings) weaver_config = get_weaver_configuration(settings) weaver_api = asbool(settings.get("weaver.wps_restapi")) weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None weaver_api_oas_ui = weaver_api_url + sd.api_openapi_ui_service.path if weaver_api else None weaver_api_swagger = weaver_api_url + sd.api_swagger_ui_service.path if weaver_api else None weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None weaver_api_spec = weaver_api_url + sd.openapi_json_service.path if weaver_api else None weaver_wps = asbool(settings.get("weaver.wps")) weaver_wps_url = get_wps_url(settings) if weaver_wps else None weaver_conform_url = weaver_url + sd.api_conformance_service.path weaver_process_url = weaver_url + sd.processes_service.path weaver_jobs_url = weaver_url + sd.jobs_service.path weaver_links = [{ "href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This landing page." }, { "href": weaver_conform_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/conformance", "type": CONTENT_TYPE_APP_JSON, "title": "Conformance classes implemented by this service." }, { "href": __meta__.__license_url__, "rel": "license", "type": CONTENT_TYPE_TEXT_PLAIN, "title": __meta__.__license_long__ }] if weaver_api: weaver_links.extend([ { "href": weaver_api_url, "rel": "service", "type": CONTENT_TYPE_APP_JSON, "title": "WPS REST API endpoint of this service." }, { "href": weaver_api_spec, "rel": "service-desc", "type": CONTENT_TYPE_APP_JSON, "title": "OpenAPI specification of this service." }, { "href": weaver_api_oas_ui, "rel": "service-doc", "type": CONTENT_TYPE_TEXT_HTML, "title": "Human readable OpenAPI documentation of this service." }, { "href": weaver_api_spec, "rel": "OpenAPI", "type": CONTENT_TYPE_APP_JSON, "title": "OpenAPI specification of this service." }, { "href": weaver_api_swagger, "rel": "swagger-ui", "type": CONTENT_TYPE_TEXT_HTML, "title": "WPS REST API definition of this service." }, { "href": weaver_process_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/processes", "type": CONTENT_TYPE_APP_JSON, "title": "Processes offered by this service." }, { "href": sd.OGC_API_REPO_URL, "rel": "ogcapi-processes-repository", "type": CONTENT_TYPE_TEXT_HTML, "title": "OGC API - Processes schema definitions repository." }, { "href": weaver_jobs_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/job-list", "type": CONTENT_TYPE_APP_JSON, "title": "Job search and listing endpoint of executions registered under this service." }, { "href": sd.CWL_BASE_URL, "rel": "cwl-home", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) homepage." }, { "href": sd.CWL_REPO_URL, "rel": "cwl-repository", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) repositories." }, { "href": sd.CWL_SPEC_URL, "rel": "cwl-specification", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) specification." }, { "href": sd.CWL_USER_GUIDE_URL, "rel": "cwl-user-guide", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) user guide." }, { "href": sd.CWL_CMD_TOOL_URL, "rel": "cwl-command-line-tool", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) CommandLineTool specification." }, { "href": sd.CWL_WORKFLOW_URL, "rel": "cwl-workflow", "type": CONTENT_TYPE_TEXT_HTML, "title": "Common Workflow Language (CWL) Workflow specification." }, ]) if weaver_api_ref: # sample: # https://app.swaggerhub.com/apis/geoprocessing/WPS/ weaver_links.append({ "href": weaver_api_ref, "rel": "reference", "type": CONTENT_TYPE_APP_JSON, "title": "API reference specification of this service." }) if isinstance(weaver_api_doc, str): # sample: # https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/develop/docs/18-062.pdf if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test ext_type = weaver_api_doc.split(".")[-1] doc_type = "application/{}".format(ext_type) else: doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type weaver_links.append({ "href": weaver_api_doc, "rel": "documentation", "type": doc_type, "title": "API reference documentation about this service." }) else: weaver_links.append({ "href": __meta__.__documentation_url__, "rel": "documentation", "type": CONTENT_TYPE_TEXT_HTML, "title": "API reference documentation about this service." }) if weaver_wps: weaver_links.extend([ { "href": weaver_wps_url, "rel": "wps", "type": CONTENT_TYPE_TEXT_XML, "title": "WPS 1.0.0/2.0 XML endpoint of this service." }, { "href": "http://docs.opengeospatial.org/is/14-065/14-065.html", "rel": "wps-specification", "type": CONTENT_TYPE_TEXT_HTML, "title": "WPS 1.0.0/2.0 definition of this service." }, { "href": "http://schemas.opengis.net/wps/", "rel": "wps-schema-repository", "type": CONTENT_TYPE_TEXT_HTML, "title": "WPS 1.0.0/2.0 XML schemas repository." }, { "href": "http://schemas.opengis.net/wps/1.0.0/wpsAll.xsd", "rel": "wps-schema-1", "type": CONTENT_TYPE_TEXT_XML, "title": "WPS 1.0.0 XML validation schemas entrypoint." }, { "href": "http://schemas.opengis.net/wps/2.0/wps.xsd", "rel": "wps-schema-2", "type": CONTENT_TYPE_TEXT_XML, "title": "WPS 2.0 XML validation schemas entrypoint." }, ]) return { "message": "Weaver Information", "configuration": weaver_config, "description": __meta__.__description__, "parameters": [ { "name": "api", "enabled": weaver_api, "url": weaver_api_url, "api": weaver_api_oas_ui }, { "name": "wps", "enabled": weaver_wps, "url": weaver_wps_url }, ], "links": weaver_links, }