def mocked_app_request(method, url=None, session=None, **req_kwargs):
"""
Mock requests under the web test application under specific conditions.
Request corresponding to :func:`requests.request` that instead gets executed by :class:`webTest.TestApp`,
unless permitted to call real external requests.
"""
# if URL starts with '/' directly, it is the shorthand path for this test app, always mock
# otherwise, filter according to full URL hostname
url_test_app = get_weaver_url(app.app.registry)
if only_local and not url.startswith("/") and not url.startswith(
url_test_app):
with session or RealSession() as request_session:
return real_request(request_session, method, url, **req_kwargs)
url, func, req_kwargs = _parse_for_app_req(method, url, **req_kwargs)
redirects = req_kwargs.pop("allow_redirects", True)
if url.startswith("mock://"):
path = get_url_without_query(url.replace("mock://", ""))
_resp = mocked_file_response(path, url)
else:
_resp = func(url, expect_errors=True, **req_kwargs)
if redirects:
# must handle redirects manually with TestApp
while 300 <= _resp.status_code < 400:
_resp = _resp.follow()
_patch_response_methods(_resp, url)
return _resp
def _describe_process_redirect(self, wps_request, *_, **__):
# type: (WPSRequest, *Any, **Any) -> Optional[Union[WPSResponse, HTTPValid]]
"""
Redirects to WPS-REST endpoint if requested ``Content-Type`` is JSON.
"""
req = wps_request.http_request
req = extend_instance(req,
PyramidRequest) # apply query 'params' method
accept_type = guess_target_format(req, default=None)
if accept_type == ContentType.APP_JSON:
url = get_weaver_url(self.settings)
proc = wps_request.identifiers
if not proc:
raise HTTPBadRequest(
sd.BadRequestGetProcessInfoResponse.description)
if len(proc) > 1:
raise HTTPBadRequest(
"Unsupported multi-process ID for description. Only provide one."
)
path = sd.process_service.path.format(process_id=proc[0])
resp = HTTPSeeOther(location=f"{url}{path}") # redirect
setattr(
resp, "_update_status",
lambda *_, **__: None) # patch to avoid pywps server raising
return resp
return None
def get_wps_url(container, load=True):
# type: (AnySettingsContainer, bool) -> str
"""
Retrieves the full WPS URL (hostname + WPS path).
Searches directly in settings, then `weaver.wps_cfg` file, or finally, uses the default values if not found.
"""
return get_settings(container).get("weaver.wps_url") or get_weaver_url(
container) + get_wps_path(container, load)
def get_wps_restapi_base_url(container):
# type: (AnySettingsContainer) -> str
settings = get_settings(container)
weaver_rest_url = settings.get("weaver.wps_restapi_url")
if not weaver_rest_url:
weaver_url = get_weaver_url(settings)
restapi_path = wps_restapi_base_path(settings)
weaver_rest_url = weaver_url + restapi_path
return weaver_rest_url.rstrip("/").strip()
def api_swagger_json(request): # noqa: F811
# type: (Request) -> dict
"""weaver REST API schema generation in JSON format."""
# obtain 'server' host and api-base-path, which doesn't correspond necessarily to the app's host and path
# ex: 'server' adds '/weaver' with proxy redirect before API routes
weaver_server_url = get_weaver_url(request)
LOGGER.debug("Request app URL: [%s]", request.url)
LOGGER.debug("Weaver config URL: [%s]", weaver_server_url)
# http_scheme=request.scheme, http_host=request.host
return get_swagger_json(base_url=weaver_server_url, use_docstring_summary=True)
def api_frontpage(request):
"""Frontpage of weaver."""
# import here to avoid circular import errors
from weaver.config import get_weaver_configuration
settings = get_settings(request)
weaver_url = get_weaver_url(settings)
weaver_config = get_weaver_configuration(settings)
weaver_api = asbool(settings.get("weaver.wps_restapi"))
weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None
weaver_api_def = weaver_api_url + sd.api_swagger_ui_uri if weaver_api else None
weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None
weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None
weaver_wps = asbool(settings.get("weaver.wps"))
weaver_wps_url = get_wps_url(settings) if weaver_wps else None
weaver_conform_url = weaver_url + sd.api_conformance_uri
weaver_process_url = weaver_url + sd.processes_uri
weaver_links = [
{"href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This document"},
{"href": weaver_conform_url, "rel": "conformance", "type": CONTENT_TYPE_APP_JSON,
"title": "WPS 2.0/3.0 REST-JSON Binding Extension conformance classes implemented by this service."},
]
if weaver_api_def:
weaver_links.append({"href": weaver_api_def, "rel": "service", "type": CONTENT_TYPE_APP_JSON,
"title": "API definition of this service."})
if isinstance(weaver_api_doc, str):
if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test
ext_type = weaver_api_doc.split(".")[-1]
doc_type = "application/{}".format(ext_type)
else:
doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type
weaver_links.append({"href": weaver_api_doc, "rel": "documentation", "type": doc_type,
"title": "API documentation about this service."})
if weaver_api_ref:
weaver_links.append({"href": weaver_api_ref, "rel": "reference", "type": CONTENT_TYPE_APP_JSON,
"title": "API reference specification of this service."})
weaver_links.append({"href": weaver_process_url, "rel": "processes", "type": CONTENT_TYPE_APP_JSON,
"title": "Processes offered by this service."})
return {
"message": "Weaver Information",
"configuration": weaver_config,
"parameters": [
{"name": "api", "enabled": weaver_api,
"url": weaver_api_url,
"doc": weaver_api_doc,
"api": weaver_api_def,
"ref": weaver_api_ref},
{"name": "wps", "enabled": weaver_wps,
"url": weaver_wps_url},
],
"links": weaver_links,
}
def setUpClass(cls):
# disable SSL warnings from logs
try:
import urllib3 # noqa
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
except ImportError:
pass
# logging parameter overrides
cls.logger_level = os.getenv("WEAVER_TEST_LOGGER_LEVEL", cls.logger_level)
cls.logger_enabled = asbool(os.getenv("WEAVER_TEST_LOGGER_ENABLED", cls.logger_enabled))
cls.logger_result_dir = os.getenv("WEAVER_TEST_LOGGER_RESULT_DIR", os.path.join(WEAVER_ROOT_DIR))
cls.logger_json_indent = os.getenv("WEAVER_TEST_LOGGER_JSON_INDENT", cls.logger_json_indent)
cls.logger_field_indent = os.getenv("WEAVER_TEST_LOGGER_FIELD_INDENT", cls.logger_field_indent)
cls.logger_separator_calls = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_CALLS", cls.logger_separator_calls)
cls.logger_separator_steps = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_STEPS", cls.logger_separator_steps)
cls.logger_separator_tests = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_TESTS", cls.logger_separator_tests)
cls.logger_separator_cases = os.getenv("WEAVER_TEST_LOGGER_SEPARATOR_CASES", cls.logger_separator_cases)
cls.setup_logger()
cls.log("{}Start of '{}': {}\n{}"
.format(cls.logger_separator_cases, cls.current_case_name(), now(), cls.logger_separator_cases))
# test execution configs
cls.WEAVER_TEST_REQUEST_TIMEOUT = int(os.getenv("WEAVER_TEST_REQUEST_TIMEOUT", 10))
cls.WEAVER_TEST_JOB_ACCEPTED_MAX_TIMEOUT = int(os.getenv("WEAVER_TEST_JOB_ACCEPTED_MAX_TIMEOUT", 30))
cls.WEAVER_TEST_JOB_RUNNING_MAX_TIMEOUT = int(os.getenv("WEAVER_TEST_JOB_RUNNING_MAX_TIMEOUT", 6000))
cls.WEAVER_TEST_JOB_GET_STATUS_INTERVAL = int(os.getenv("WEAVER_TEST_JOB_GET_STATUS_INTERVAL", 5))
# security configs if enabled
cls.WEAVER_TEST_PROTECTED_ENABLED = asbool(os.getenv("WEAVER_TEST_PROTECTED_ENABLED", False))
cls.WEAVER_TEST_WSO2_CLIENT_HOSTNAME = os.getenv("WEAVER_TEST_WSO2_CLIENT_HOSTNAME", "")
cls.WEAVER_TEST_WSO2_CLIENT_ID = os.getenv("WEAVER_TEST_WSO2_CLIENT_ID", "")
cls.WEAVER_TEST_WSO2_CLIENT_SECRET = os.getenv("WEAVER_TEST_WSO2_CLIENT_SECRET", "")
cls.WEAVER_TEST_WSO2_URL = os.getenv("WEAVER_TEST_WSO2_URL", "")
cls.WEAVER_TEST_MAGPIE_URL = os.getenv("WEAVER_TEST_MAGPIE_URL", "")
cls.WEAVER_TEST_ADMIN_CREDENTIALS = {"username": get_setting("ADMIN_USERNAME", cls.app),
"password": get_setting("ADMIN_PASSWORD", cls.app)}
cls.WEAVER_TEST_ALICE_CREDENTIALS = {"username": get_setting("ALICE_USERNAME", cls.app),
"password": get_setting("ALICE_PASSWORD", cls.app)}
cls.WEAVER_TEST_BOB_CREDENTIALS = {"username": get_setting("BOD_USERNAME", cls.app),
"password": get_setting("BOB_PASSWORD", cls.app)}
# server settings
cls.WEAVER_TEST_SERVER_HOSTNAME = os.getenv("WEAVER_TEST_SERVER_HOSTNAME")
cls.WEAVER_TEST_SERVER_BASE_PATH = os.getenv("WEAVER_TEST_SERVER_BASE_PATH", "/weaver")
cls.WEAVER_TEST_SERVER_API_PATH = os.getenv("WEAVER_TEST_SERVER_API_PATH", "/")
cls.WEAVER_TEST_CONFIG_INI_PATH = os.getenv("WEAVER_TEST_CONFIG_INI_PATH") # none uses default path
cls.app = WebTestApp(cls.WEAVER_TEST_SERVER_HOSTNAME)
cls.WEAVER_URL = get_weaver_url(cls.settings())
cls.WEAVER_RESTAPI_URL = get_wps_restapi_base_url(cls.settings())
# validation
cls.validate_test_server()
cls.setup_test_processes()
def redoc_ui_cached(request):
settings = get_settings(request)
weaver_server_url = get_weaver_url(settings)
spec = openapi_json_cached(base_url=weaver_server_url,
settings=settings,
use_docstring_summary=True,
use_refs=False)
data_mako = {"openapi_spec": json.dumps(spec, ensure_ascii=False)}
resp = render_to_response("templates/redoc_ui.mako",
data_mako,
request=request)
return resp
def get_vault_url(file, container=None):
# type: (Union[VaultFile, uuid.UUID, str], Optional[AnySettingsContainer]) -> str
"""
Obtain the vault link corresponding to the file.
"""
if isinstance(file, uuid.UUID) or is_uuid(file):
file_id = str(file)
else:
file_id = file.id
settings = get_settings(container)
base_url = get_weaver_url(settings)
vault_url = base_url + sd.vault_file_service.path.format(file_id=file_id)
return vault_url
def get_wps_output_url(container):
# type: (AnySettingsContainer) -> str
"""
Retrieves the WPS output URL that maps to WPS output directory path.
Searches directly in settings, then `weaver.wps_cfg` file, or finally, uses the default values if not found.
"""
wps_output_default = get_weaver_url(container) + "/wpsoutputs"
wps_output_config = _get_settings_or_wps_config(container,
"weaver.wps_output_url",
"server", "outputurl",
wps_output_default,
"WPS output url")
return wps_output_config or wps_output_default
def __init__(self, *args, **kwargs):
db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs)
StoreProcesses.__init__(self)
MongodbStore.__init__(self, *db_args, **db_kwargs)
registry = kwargs.get("registry")
default_processes = kwargs.get("default_processes")
self.settings = kwargs.get("settings",
{}) if not registry else registry.settings
self.default_host = get_weaver_url(self.settings)
self.default_wps_endpoint = get_wps_url(self.settings)
# enforce default process re-registration to receive any applicable update
if default_processes:
self._register_defaults(default_processes)
def openapi_json(request): # noqa: F811
# type: (Request) -> dict
"""
Weaver OpenAPI schema definitions.
"""
# obtain 'server' host and api-base-path, which doesn't correspond necessarily to the app's host and path
# ex: 'server' adds '/weaver' with proxy redirect before API routes
settings = get_settings(request)
weaver_server_url = get_weaver_url(settings)
LOGGER.debug("Request app URL: [%s]", request.url)
LOGGER.debug("Weaver config URL: [%s]", weaver_server_url)
return openapi_json_cached(base_url=weaver_server_url,
use_docstring_summary=True,
settings=settings)
def _get_capabilities_redirect(self, wps_request, *_, **__):
# type: (WPSRequest, Any, Any) -> Optional[Union[WPSResponse, HTTPValid]]
"""
Redirects to WPS-REST endpoint if requested ``Content-Type`` is JSON.
"""
req = wps_request.http_request
accept_type = get_header("Accept", req.headers)
if accept_type == CONTENT_TYPE_APP_JSON:
url = get_weaver_url(self.settings)
resp = HTTPSeeOther(location="{}{}".format(
url, sd.processes_uri)) # redirect
setattr(
resp, "_update_status",
lambda *_, **__: None) # patch to avoid pywps server raising
return resp
return None
def __init__(self, *args, **kwargs):
db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs)
StoreProcesses.__init__(self)
MongodbStore.__init__(self, *db_args, **db_kwargs)
registry = kwargs.get("registry")
settings = kwargs.get("settings", {}) if not registry else registry.settings
default_processes = kwargs.get("default_processes")
self.default_host = get_weaver_url(settings)
self.default_wps_endpoint = get_wps_url(settings)
# enforce default process re-registration to receive any applicable update
if default_processes:
registered_processes = [process.identifier for process in self.list_processes()]
for process in default_processes:
process_name = self._get_process_id(process)
if process_name in registered_processes:
self.delete_process(process_name)
self._add_process(process)
def mocked_app_request(method, url=None, **req_kwargs):
"""
Request corresponding to :func:`requests.request` that instead gets executed by :class:`webTest.TestApp`,
unless permitted to call real external requests.
"""
# if URL starts with '/' directly, it is the shorthand path for this test app, always mock
# otherwise, filter according to full URL hostname
url_test_app = get_weaver_url(app.app.registry)
if only_local and not url.startswith("/") and not url.startswith(
url_test_app):
with RealSession() as session:
return real_request(session, method, url, **req_kwargs)
url, func, req_kwargs = _parse_for_app_req(method, url, **req_kwargs)
if not url.startswith("mock://"):
resp = func(url, expect_errors=True, **req_kwargs)
setattr(resp, "content", resp.body)
else:
path = get_url_without_query(url.replace("mock://", ""))
resp = mocked_file_response(path, url)
return resp
def _describe_process_redirect(self, wps_request, *_, **__):
# type: (WPSRequest, Any, Any) -> Optional[Union[WPSResponse, HTTPValid]]
"""
Redirects to WPS-REST endpoint if requested ``Content-Type`` is JSON.
"""
req = wps_request.http_request
accept_type = get_header("Accept", req.headers)
if accept_type == CONTENT_TYPE_APP_JSON:
url = get_weaver_url(self.settings)
proc = wps_request.identifiers
if not proc:
raise HTTPBadRequest(
sd.BadRequestGetProcessInfoResponse.description)
if len(proc) > 1:
raise HTTPBadRequest(
"Unsupported multi-process ID for description. Only provide one."
)
path = sd.process_uri.format(process_id=proc[0])
resp = HTTPSeeOther(location="{}{}".format(url, path)) # redirect
setattr(
resp, "_update_status",
lambda *_, **__: None) # patch to avoid pywps server raising
return resp
return None
def get_job_list_links(job_total, filters, request):
# type: (int, Dict[str, AnyValueType], Request) -> List[JSON]
"""
Obtains a list of all relevant links for the corresponding job listing defined by query parameter filters.
:raises IndexError: if the paging values are out of bounds compared to available total :term:`Job` matching search.
"""
base_url = get_weaver_url(request)
# reapply queries that must be given to obtain the same result in case of subsequent requests (sort, limits, etc.)
kvp_params = {
param: value
for param, value in request.params.items() if param != "page"
}
# patch datetime that have some extra character manipulation (reapply '+' auto-converted to ' ' by params parser)
if "datetime" in kvp_params:
kvp_params["datetime"] = kvp_params["datetime"].replace(" ", "+")
alt_kvp = deepcopy(kvp_params)
# request job uses general endpoint, obtain the full path if any service/process was given as alternate location
if request.path.startswith(sd.jobs_service.path):
job_path = base_url + sd.jobs_service.path
alt_path = None
parent_url = None
# cannot generate full path apply for 'service' by itself
if filters["process"] and filters["service"]:
alt_path = base_url + sd.provider_jobs_service.path.format(
provider_id=filters["service"], process_id=filters["process"])
parent_url = alt_path.rsplit("/", 1)[0]
elif filters["process"]:
alt_path = base_url + sd.process_jobs_service.path.format(
process_id=filters["process"])
parent_url = alt_path.rsplit("/", 1)[0]
for param in ["service", "provider", "process"]:
alt_kvp.pop(param, None)
# path is whichever specific service/process endpoint, jobs are pre-filtered by them
# transform sub-endpoints into matching query parameters and use generic path as alternate location
else:
job_path = base_url + request.path
alt_path = base_url + sd.jobs_service.path
alt_kvp["process"] = filters["process"]
if filters["service"]:
alt_kvp["provider"] = filters["service"]
parent_url = job_path.rsplit("/", 1)[0]
cur_page = filters["page"]
per_page = filters["limit"]
max_page = max(math.ceil(job_total / per_page) - 1, 0)
if cur_page < 0 or cur_page > max_page:
raise IndexError(
f"Page index {cur_page} is out of range from [0,{max_page}].")
alt_links = []
if alt_path:
alt_links = [{
"href":
get_path_kvp(alt_path, page=cur_page, **alt_kvp),
"rel":
"alternate",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Alternate endpoint with equivalent set of filtered jobs."
}]
links = alt_links + [
{
"href": job_path,
"rel": "collection",
"type": CONTENT_TYPE_APP_JSON,
"title": "Complete job listing (no filtering queries applied)."
},
{
"href": base_url + sd.jobs_service.path,
"rel": "search",
"type": CONTENT_TYPE_APP_JSON,
"title": "Generic query endpoint to search for jobs."
},
{
"href": job_path + "?detail=false",
"rel": "preview",
"type": CONTENT_TYPE_APP_JSON,
"title": "Job listing summary (UUID and count only)."
},
{
"href": job_path,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/job-list",
"type": CONTENT_TYPE_APP_JSON,
"title": "List of registered jobs."
},
{
"href": get_path_kvp(job_path, page=cur_page, **kvp_params),
"rel": "current",
"type": CONTENT_TYPE_APP_JSON,
"title": "Current page of job query listing."
},
{
"href": get_path_kvp(job_path, page=0, **kvp_params),
"rel": "first",
"type": CONTENT_TYPE_APP_JSON,
"title": "First page of job query listing."
},
{
"href": get_path_kvp(job_path, page=max_page, **kvp_params),
"rel": "last",
"type": CONTENT_TYPE_APP_JSON,
"title": "Last page of job query listing."
},
]
if cur_page > 0:
links.append({
"href":
get_path_kvp(job_path, page=cur_page - 1, **kvp_params),
"rel":
"prev",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Previous page of job query listing."
})
if cur_page < max_page:
links.append({
"href":
get_path_kvp(job_path, page=cur_page + 1, **kvp_params),
"rel":
"next",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Next page of job query listing."
})
if parent_url:
links.append({
"href": parent_url,
"rel": "up",
"type": CONTENT_TYPE_APP_JSON,
"title": "Parent collection for which listed jobs apply."
})
return links
def get_process_list_links(request, paging, total, provider=None):
# type: (Request, Dict[str, int], Optional[int], Optional[Service]) -> List[JSON]
"""
Obtains a list of all relevant links for the corresponding :term:`Process` listing defined by query parameters.
:raises IndexError: if the paging values are out of bounds compared to available total :term:`Process`.
"""
# reapply queries that must be given to obtain the same result in case of subsequent requests (sort, limits, etc.)
kvp_params = {
param: value
for param, value in request.params.items() if param != "page"
}
base_url = get_weaver_url(request)
links = []
if provider:
proc_path = sd.provider_processes_service.path.format(
provider_id=provider.id)
links.extend(provider.links(request, self_link="provider"))
else:
proc_path = sd.processes_service.path
proc_url = base_url + proc_path
links.extend([
{
"href": proc_url,
"rel": "collection",
"type": CONTENT_TYPE_APP_JSON,
"title": "Process listing (no filtering queries applied)."
},
{
"href": proc_url,
"rel": "search",
"type": CONTENT_TYPE_APP_JSON,
"title": "Generic query endpoint to list processes."
},
{
"href": proc_url + "?detail=false",
"rel": "preview",
"type": CONTENT_TYPE_APP_JSON,
"title": "Process listing summary (identifiers and count only)."
},
{
"href": proc_url,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/processes",
"type": CONTENT_TYPE_APP_JSON,
"title": "List of registered local processes."
},
{
"href": get_path_kvp(proc_url, **request.params),
"rel": "self",
"type": CONTENT_TYPE_APP_JSON,
"title": "Current process listing."
},
])
if provider:
prov_url = proc_url.rsplit("/", 1)[0]
links.append({
"href": prov_url,
"rel": "up",
"type": CONTENT_TYPE_APP_JSON,
"title": "Provider description."
})
else:
links.append({
"href": base_url,
"rel": "up",
"type": CONTENT_TYPE_APP_JSON,
"title": "API entrypoint."
})
cur_page = paging.get("page", None)
per_page = paging.get("limit", None)
if all(isinstance(num, int) for num in [cur_page, per_page, total]):
max_page = max(math.ceil(total / per_page) - 1, 0)
if cur_page < 0 or cur_page > max_page:
raise IndexError(
f"Page index {cur_page} is out of range from [0,{max_page}].")
links.extend([
{
"href": get_path_kvp(proc_url, page=cur_page, **kvp_params),
"rel": "current",
"type": CONTENT_TYPE_APP_JSON,
"title": "Current page of processes query listing."
},
{
"href": get_path_kvp(proc_url, page=0, **kvp_params),
"rel": "first",
"type": CONTENT_TYPE_APP_JSON,
"title": "First page of processes query listing."
},
{
"href": get_path_kvp(proc_url, page=max_page, **kvp_params),
"rel": "last",
"type": CONTENT_TYPE_APP_JSON,
"title": "Last page of processes query listing."
},
])
if cur_page > 0:
links.append({
"href":
get_path_kvp(proc_url, page=cur_page - 1, **kvp_params),
"rel":
"prev",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Previous page of processes query listing."
})
if cur_page < max_page:
links.append({
"href":
get_path_kvp(proc_url, page=cur_page + 1, **kvp_params),
"rel":
"next",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Next page of processes query listing."
})
return links
def request_quote(request):
"""
Request a quotation for a process.
"""
settings = get_settings(request)
weaver_config = get_weaver_configuration(settings)
if weaver_config not in [
WEAVER_CONFIGURATION_ADES, WEAVER_CONFIGURATION_EMS
]:
raise HTTPBadRequest(
"Unsupported request for configuration '{}'.".format(
weaver_config))
process_id = request.matchdict.get("process_id")
process_store = get_db(request).get_store("processes")
try:
process = process_store.fetch_by_id(process_id)
except ProcessNotFound:
raise HTTPNotFound(
"Could not find process with specified 'process_id'.")
store = get_db(request).get_store(StoreQuotes)
process_url = get_process_location(process_id,
data_source=get_weaver_url(settings))
process_type = process.type
process_params = dict()
for param in ["inputs", "outputs", "mode", "response"]:
if param in request.json:
process_params[param] = request.json.pop(param)
process_quote_info = process_quote_estimator(process)
process_quote_info.update({
"process": process_id,
"processParameters": process_params,
"location": process_url,
"user": str(request.authenticated_userid)
})
# loop workflow sub-process steps to get individual quotes
if process_type == PROCESS_WORKFLOW and weaver_config == WEAVER_CONFIGURATION_EMS:
workflow_quotes = list()
for step in get_package_workflow_steps(process_url):
# retrieve quote from provider ADES
# TODO: data source mapping
process_step_url = get_process_location(step["reference"])
process_quote_url = "{}/quotations".format(process_step_url)
subreq = request.copy()
subreq.path_info = process_quote_url
resp_json = request.invoke_subrequest(subreq).json()
quote_json = resp_json["quote"]
quote = store.save_quote(Quote(**quote_json))
workflow_quotes.append(quote.id)
process_quote_info.update({"steps": workflow_quotes})
quote = store.save_quote(Quote(**process_quote_info))
return HTTPCreated(json={"quote": quote.json()})
# single application quotes (ADES or EMS)
elif process_type == PROCESS_APPLICATION:
quote = store.save_quote(Quote(**process_quote_info))
quote_json = quote.json()
quote_json.pop("steps", None)
return HTTPCreated(json={"quote": quote_json})
# error if not handled up to this point
raise HTTPBadRequest(
"Unsupported quoting process type '{0}' on '{1}'.".format(
process_type, weaver_config))
def load_pywps_config(container, config=None):
# type: (AnySettingsContainer, Optional[Union[str, Dict[str, str]]]) -> ConfigParser
"""
Loads and updates the PyWPS configuration using Weaver settings.
"""
settings = get_settings(container)
if settings.get("weaver.wps_configured"):
LOGGER.debug("Using preloaded internal Weaver WPS configuration.")
return pywps_config.CONFIG
LOGGER.info("Initial load of internal Weaver WPS configuration.")
pywps_config.load_configuration([]) # load defaults
pywps_config.CONFIG.set("logging", "db_echo", "false")
if logging.getLevelName(pywps_config.CONFIG.get("logging",
"level")) <= logging.DEBUG:
pywps_config.CONFIG.set("logging", "level", "INFO")
# update metadata
LOGGER.debug("Updating WPS metadata configuration.")
for setting_name, setting_value in settings.items():
if setting_name.startswith("weaver.wps_metadata"):
pywps_setting = setting_name.replace("weaver.wps_metadata_", "")
pywps_config.CONFIG.set("metadata:main", pywps_setting,
setting_value)
# add weaver configuration keyword if not already provided
wps_keywords = pywps_config.CONFIG.get("metadata:main",
"identification_keywords")
weaver_mode = get_weaver_configuration(settings)
if weaver_mode not in wps_keywords:
wps_keywords += ("," if wps_keywords else "") + weaver_mode
pywps_config.CONFIG.set("metadata:main", "identification_keywords",
wps_keywords)
# add additional config passed as dictionary of {'section.key': 'value'}
if isinstance(config, dict):
for key, value in config.items():
section, key = key.split(".")
pywps_config.CONFIG.set(section, key, value)
# cleanup alternative dict "PYWPS_CFG" which is not expected elsewhere
if isinstance(settings.get("PYWPS_CFG"), dict):
del settings["PYWPS_CFG"]
# set accepted languages aligned with values provided by REST API endpoints
# otherwise, execute request could fail due to languages considered not supported
languages = ", ".join(AcceptLanguage.values())
LOGGER.debug("Setting WPS languages: [%s]", languages)
pywps_config.CONFIG.set("server", "language", languages)
LOGGER.debug("Updating WPS output configuration.")
# find output directory from app config or wps config
if "weaver.wps_output_dir" not in settings:
output_dir = pywps_config.get_config_value("server", "outputpath")
settings["weaver.wps_output_dir"] = output_dir
# ensure the output dir exists if specified
output_dir = get_wps_output_dir(settings)
make_dirs(output_dir, exist_ok=True)
# find output url from app config (path/url) or wps config (url only)
# note: needs to be configured even when using S3 bucket since XML status is provided locally
if "weaver.wps_output_url" not in settings:
output_path = settings.get("weaver.wps_output_path", "").rstrip("/")
if output_path and isinstance(output_path, str):
output_url = os.path.join(get_weaver_url(settings),
output_path.strip("/"))
else:
output_url = pywps_config.get_config_value("server", "outputurl")
settings["weaver.wps_output_url"] = output_url
# apply workdir if provided, otherwise use default
if "weaver.wps_workdir" in settings:
make_dirs(settings["weaver.wps_workdir"], exist_ok=True)
pywps_config.CONFIG.set("server", "workdir",
settings["weaver.wps_workdir"])
# configure S3 bucket if requested, storage of all process outputs
# note:
# credentials and default profile are picked up automatically by 'boto3' from local AWS configs or env vars
# region can also be picked from there unless explicitly provided by weaver config
# warning:
# if we set `(server, storagetype, s3)`, ALL status (including XML) are stored to S3
# to preserve status locally, we set 'file' and override the storage instance during output rewrite in WpsPackage
# we can still make use of the server configurations here to make this overridden storage auto-find its configs
s3_bucket = settings.get("weaver.wps_output_s3_bucket")
pywps_config.CONFIG.set("server", "storagetype", "file")
# pywps_config.CONFIG.set("server", "storagetype", "s3")
if s3_bucket:
LOGGER.debug("Updating WPS S3 bucket configuration.")
import boto3
from botocore.exceptions import ClientError
s3 = boto3.client("s3")
s3_region = settings.get("weaver.wps_output_s3_region",
s3.meta.region_name)
LOGGER.info(
"Validating that S3 [Bucket=%s, Region=%s] exists or creating it.",
s3_bucket, s3_region)
try:
s3.create_bucket(
Bucket=s3_bucket,
CreateBucketConfiguration={"LocationConstraint": s3_region})
LOGGER.info("S3 bucket for WPS output created.")
except ClientError as exc:
if exc.response.get("Error",
{}).get("Code") != "BucketAlreadyExists":
LOGGER.error("Failed setup of S3 bucket for WPS output: [%s]",
exc)
raise
LOGGER.info("S3 bucket for WPS output already exists.")
pywps_config.CONFIG.set("s3", "region", s3_region)
pywps_config.CONFIG.set("s3", "bucket", s3_bucket)
pywps_config.CONFIG.set(
"s3", "public",
"false") # don't automatically push results as publicly accessible
pywps_config.CONFIG.set(
"s3", "encrypt",
"true") # encrypts data server-side, transparent from this side
# enforce back resolved values onto PyWPS config
pywps_config.CONFIG.set("server", "setworkdir", "true")
pywps_config.CONFIG.set("server", "sethomedir", "true")
pywps_config.CONFIG.set("server", "outputpath",
settings["weaver.wps_output_dir"])
pywps_config.CONFIG.set("server", "outputurl",
settings["weaver.wps_output_url"])
pywps_config.CONFIG.set("server", "url", get_wps_url(settings, load=False))
settings["weaver.wps_configured"] = True
return pywps_config.CONFIG
def api_frontpage_body(settings):
# type: (SettingsType) -> JSON
"""
Generates the JSON body describing the Weaver API and documentation references.
"""
# import here to avoid circular import errors
from weaver.config import get_weaver_configuration
weaver_url = get_weaver_url(settings)
weaver_config = get_weaver_configuration(settings)
weaver_api = asbool(settings.get("weaver.wps_restapi"))
weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None
weaver_api_oas_ui = weaver_api_url + sd.api_openapi_ui_service.path if weaver_api else None
weaver_api_swagger = weaver_api_url + sd.api_swagger_ui_service.path if weaver_api else None
weaver_api_doc = settings.get("weaver.wps_restapi_doc",
None) if weaver_api else None
weaver_api_ref = settings.get("weaver.wps_restapi_ref",
None) if weaver_api else None
weaver_api_spec = weaver_api_url + sd.openapi_json_service.path if weaver_api else None
weaver_wps = asbool(settings.get("weaver.wps"))
weaver_wps_url = get_wps_url(settings) if weaver_wps else None
weaver_conform_url = weaver_url + sd.api_conformance_service.path
weaver_process_url = weaver_url + sd.processes_service.path
weaver_jobs_url = weaver_url + sd.jobs_service.path
weaver_links = [{
"href": weaver_url,
"rel": "self",
"type": CONTENT_TYPE_APP_JSON,
"title": "This landing page."
}, {
"href": weaver_conform_url,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/conformance",
"type": CONTENT_TYPE_APP_JSON,
"title": "Conformance classes implemented by this service."
}, {
"href": __meta__.__license_url__,
"rel": "license",
"type": CONTENT_TYPE_TEXT_PLAIN,
"title": __meta__.__license_long__
}]
if weaver_api:
weaver_links.extend([
{
"href": weaver_api_url,
"rel": "service",
"type": CONTENT_TYPE_APP_JSON,
"title": "WPS REST API endpoint of this service."
},
{
"href": weaver_api_spec,
"rel": "service-desc",
"type": CONTENT_TYPE_APP_JSON,
"title": "OpenAPI specification of this service."
},
{
"href": weaver_api_oas_ui,
"rel": "service-doc",
"type": CONTENT_TYPE_TEXT_HTML,
"title":
"Human readable OpenAPI documentation of this service."
},
{
"href": weaver_api_spec,
"rel": "OpenAPI",
"type": CONTENT_TYPE_APP_JSON,
"title": "OpenAPI specification of this service."
},
{
"href": weaver_api_swagger,
"rel": "swagger-ui",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS REST API definition of this service."
},
{
"href": weaver_process_url,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/processes",
"type": CONTENT_TYPE_APP_JSON,
"title": "Processes offered by this service."
},
{
"href": sd.OGC_API_REPO_URL,
"rel": "ogcapi-processes-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "OGC API - Processes schema definitions repository."
},
{
"href":
weaver_jobs_url,
"rel":
"http://www.opengis.net/def/rel/ogc/1.0/job-list",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Job search and listing endpoint of executions registered under this service."
},
{
"href": sd.CWL_BASE_URL,
"rel": "cwl-home",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) homepage."
},
{
"href": sd.CWL_REPO_URL,
"rel": "cwl-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) repositories."
},
{
"href": sd.CWL_SPEC_URL,
"rel": "cwl-specification",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) specification."
},
{
"href": sd.CWL_USER_GUIDE_URL,
"rel": "cwl-user-guide",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) user guide."
},
{
"href":
sd.CWL_CMD_TOOL_URL,
"rel":
"cwl-command-line-tool",
"type":
CONTENT_TYPE_TEXT_HTML,
"title":
"Common Workflow Language (CWL) CommandLineTool specification."
},
{
"href": sd.CWL_WORKFLOW_URL,
"rel": "cwl-workflow",
"type": CONTENT_TYPE_TEXT_HTML,
"title":
"Common Workflow Language (CWL) Workflow specification."
},
])
if weaver_api_ref:
# sample:
# https://app.swaggerhub.com/apis/geoprocessing/WPS/
weaver_links.append({
"href":
weaver_api_ref,
"rel":
"reference",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"API reference specification of this service."
})
if isinstance(weaver_api_doc, str):
# sample:
# https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/develop/docs/18-062.pdf
if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test
ext_type = weaver_api_doc.split(".")[-1]
doc_type = "application/{}".format(ext_type)
else:
doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type
weaver_links.append({
"href":
weaver_api_doc,
"rel":
"documentation",
"type":
doc_type,
"title":
"API reference documentation about this service."
})
else:
weaver_links.append({
"href":
__meta__.__documentation_url__,
"rel":
"documentation",
"type":
CONTENT_TYPE_TEXT_HTML,
"title":
"API reference documentation about this service."
})
if weaver_wps:
weaver_links.extend([
{
"href": weaver_wps_url,
"rel": "wps",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 1.0.0/2.0 XML endpoint of this service."
},
{
"href": "http://docs.opengeospatial.org/is/14-065/14-065.html",
"rel": "wps-specification",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS 1.0.0/2.0 definition of this service."
},
{
"href": "http://schemas.opengis.net/wps/",
"rel": "wps-schema-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS 1.0.0/2.0 XML schemas repository."
},
{
"href": "http://schemas.opengis.net/wps/1.0.0/wpsAll.xsd",
"rel": "wps-schema-1",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 1.0.0 XML validation schemas entrypoint."
},
{
"href": "http://schemas.opengis.net/wps/2.0/wps.xsd",
"rel": "wps-schema-2",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 2.0 XML validation schemas entrypoint."
},
])
return {
"message":
"Weaver Information",
"configuration":
weaver_config,
"description":
__meta__.__description__,
"parameters": [
{
"name": "api",
"enabled": weaver_api,
"url": weaver_api_url,
"api": weaver_api_oas_ui
},
{
"name": "wps",
"enabled": weaver_wps,
"url": weaver_wps_url
},
],
"links":
weaver_links,
}
