def api_frontpage(request):
"""Frontpage of weaver."""
# import here to avoid circular import errors
from weaver.config import get_weaver_configuration
settings = get_settings(request)
weaver_url = get_weaver_url(settings)
weaver_config = get_weaver_configuration(settings)
weaver_api = asbool(settings.get("weaver.wps_restapi"))
weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None
weaver_api_def = weaver_api_url + sd.api_swagger_ui_uri if weaver_api else None
weaver_api_doc = settings.get("weaver.wps_restapi_doc", None) if weaver_api else None
weaver_api_ref = settings.get("weaver.wps_restapi_ref", None) if weaver_api else None
weaver_wps = asbool(settings.get("weaver.wps"))
weaver_wps_url = get_wps_url(settings) if weaver_wps else None
weaver_conform_url = weaver_url + sd.api_conformance_uri
weaver_process_url = weaver_url + sd.processes_uri
weaver_links = [
{"href": weaver_url, "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "This document"},
{"href": weaver_conform_url, "rel": "conformance", "type": CONTENT_TYPE_APP_JSON,
"title": "WPS 2.0/3.0 REST-JSON Binding Extension conformance classes implemented by this service."},
]
if weaver_api_def:
weaver_links.append({"href": weaver_api_def, "rel": "service", "type": CONTENT_TYPE_APP_JSON,
"title": "API definition of this service."})
if isinstance(weaver_api_doc, str):
if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test
ext_type = weaver_api_doc.split(".")[-1]
doc_type = "application/{}".format(ext_type)
else:
doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type
weaver_links.append({"href": weaver_api_doc, "rel": "documentation", "type": doc_type,
"title": "API documentation about this service."})
if weaver_api_ref:
weaver_links.append({"href": weaver_api_ref, "rel": "reference", "type": CONTENT_TYPE_APP_JSON,
"title": "API reference specification of this service."})
weaver_links.append({"href": weaver_process_url, "rel": "processes", "type": CONTENT_TYPE_APP_JSON,
"title": "Processes offered by this service."})
return {
"message": "Weaver Information",
"configuration": weaver_config,
"parameters": [
{"name": "api", "enabled": weaver_api,
"url": weaver_api_url,
"doc": weaver_api_doc,
"api": weaver_api_def,
"ref": weaver_api_ref},
{"name": "wps", "enabled": weaver_wps,
"url": weaver_wps_url},
],
"links": weaver_links,
}
def __init__(self, *args, **kwargs):
db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs)
StoreProcesses.__init__(self)
MongodbStore.__init__(self, *db_args, **db_kwargs)
registry = kwargs.get("registry")
default_processes = kwargs.get("default_processes")
self.settings = kwargs.get("settings",
{}) if not registry else registry.settings
self.default_host = get_weaver_url(self.settings)
self.default_wps_endpoint = get_wps_url(self.settings)
# enforce default process re-registration to receive any applicable update
if default_processes:
self._register_defaults(default_processes)
def register_builtin_processes(container):
# type: (AnySettingsContainer) -> None
"""
Registers every ``builtin`` CWL package to the processes database.
CWL definitions must be located within the :mod:`weaver.processes.builtin` module.
"""
restapi_url = get_wps_restapi_base_url(container)
builtin_apps_mapping = _get_builtin_reference_mapping(os.path.abspath(os.path.dirname(__file__)))
builtin_processes = []
for process_id, process_path in builtin_apps_mapping.items():
process_info = get_process_definition({}, package=None, reference=process_path)
process_url = "/".join([restapi_url, "processes", process_id])
process_package = _get_builtin_package(process_id, process_info["package"])
process_abstract = _get_builtin_metadata(process_id, process_path, "__doc__", clean=True)
process_version = _get_builtin_metadata(process_id, process_path, "__version__")
process_title = _get_builtin_metadata(process_id, process_path, "__title__")
process_payload = {
"processDescription": {
"process": {
"id": process_id,
"type": PROCESS_BUILTIN,
"title": process_title,
"version": process_version,
"abstract": process_abstract,
}
},
"deploymentProfileName": "http://www.opengis.net/profiles/eoc/builtinApplication",
"executionUnit": [{"unit": process_package}],
}
process_payload["processDescription"]["process"].update(ows_context_href(process_url))
builtin_processes.append(Process(
id=process_id,
type=PROCESS_BUILTIN,
title=process_title,
version=process_version,
abstract=process_abstract,
payload=process_payload,
package=process_package,
inputs=process_info["inputs"],
outputs=process_info["outputs"],
processDescriptionURL=process_url,
processEndpointWPS1=get_wps_url(container),
executeEndpoint="/".join([process_url, "jobs"]),
visibility=VISIBILITY_PUBLIC,
))
# registration of missing/updated apps automatically applied with 'default_processes'
get_db(container).get_store(StoreProcesses, default_processes=builtin_processes)
def __init__(self, *args, **kwargs):
db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs)
StoreProcesses.__init__(self)
MongodbStore.__init__(self, *db_args, **db_kwargs)
registry = kwargs.get("registry")
settings = kwargs.get("settings", {}) if not registry else registry.settings
default_processes = kwargs.get("default_processes")
self.default_host = get_weaver_url(settings)
self.default_wps_endpoint = get_wps_url(settings)
# enforce default process re-registration to receive any applicable update
if default_processes:
registered_processes = [process.identifier for process in self.list_processes()]
for process in default_processes:
process_name = self._get_process_id(process)
if process_name in registered_processes:
self.delete_process(process_name)
self._add_process(process)
def test_deploy_process_default_endpoint_wps1(self):
"""Validates that the default (localhost) endpoint to execute WPS requests are saved during deployment."""
process_name = self.fully_qualified_test_process_name()
process_data = self.get_process_deploy_template(process_name)
package_mock = mocked_process_package()
with contextlib.ExitStack() as stack:
for pkg in package_mock:
stack.enter_context(pkg)
uri = "/processes"
resp = self.app.post_json(uri,
params=process_data,
headers=self.json_headers,
expect_errors=True)
# TODO: status should be 201 when properly modified to match API conformance
assert resp.status_code == 200
weaver_wps_path = get_wps_url(self.config.registry.settings)
process_wps_endpoint = self.process_store.fetch_by_id(
process_name).processEndpointWPS1
assert isinstance(process_wps_endpoint,
str) and len(process_wps_endpoint)
assert process_wps_endpoint == weaver_wps_path
def wps_execute(self, version, accept):
wps_url = get_wps_url(self.settings)
if version == "1.0.0":
test_content = "Test file in Docker - WPS KVP"
wps_method = "GET"
elif version == "2.0.0":
test_content = "Test file in Docker - WPS XML"
wps_method = "POST"
else:
raise ValueError("Invalid WPS version: {}".format(version))
test_content += " {} request - Accept {}".format(wps_method, accept.split("/")[-1].upper())
with contextlib.ExitStack() as stack_exec:
# setup
dir_name = tempfile.gettempdir()
tmp_path = tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt")
tmp_file = stack_exec.enter_context(tmp_path) # noqa
tmp_file.write(test_content)
tmp_file.seek(0)
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
# execute
if version == "1.0.0":
wps_inputs = ["file={}@mimeType={}".format(tmp_file.name, CONTENT_TYPE_TEXT_PLAIN)]
wps_params = {
"service": "WPS",
"request": "Execute",
"version": version,
"identifier": self.process_id,
"DataInputs": wps_inputs,
}
wps_headers = {"Accept": accept}
wps_data = None
else:
wps_inputs = [("file", ComplexDataInput(tmp_file.name, mimeType=CONTENT_TYPE_TEXT_PLAIN))]
wps_outputs = [(self.out_key, True)] # as reference
wps_exec = WPSExecution(version=version, url=wps_url)
wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs)
wps_data = lxml.etree.tostring(wps_req)
wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML}
wps_params = None
resp = mocked_sub_requests(self.app, wps_method, wps_url,
params=wps_params, data=wps_data, headers=wps_headers, only_local=True)
assert resp.status_code in [200, 201], \
"Failed with: [{}]\nTest: [{}]\nReason:\n{}".format(resp.status_code, test_content, resp.text)
# parse response status
if accept == CONTENT_TYPE_APP_XML:
assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content
xml = lxml.etree.fromstring(str2bytes(resp.text))
status_url = xml.get("statusLocation")
job_id = status_url.split("/")[-1]
elif accept == CONTENT_TYPE_APP_JSON:
assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content
status_url = resp.json["location"]
job_id = resp.json["jobID"]
assert status_url
assert job_id
# job monitoring
result = self.monitor_job(status_url)
self.validate_outputs(job_id, result, test_content)
def api_frontpage_body(settings):
# type: (SettingsType) -> JSON
"""
Generates the JSON body describing the Weaver API and documentation references.
"""
# import here to avoid circular import errors
from weaver.config import get_weaver_configuration
weaver_url = get_weaver_url(settings)
weaver_config = get_weaver_configuration(settings)
weaver_api = asbool(settings.get("weaver.wps_restapi"))
weaver_api_url = get_wps_restapi_base_url(settings) if weaver_api else None
weaver_api_oas_ui = weaver_api_url + sd.api_openapi_ui_service.path if weaver_api else None
weaver_api_swagger = weaver_api_url + sd.api_swagger_ui_service.path if weaver_api else None
weaver_api_doc = settings.get("weaver.wps_restapi_doc",
None) if weaver_api else None
weaver_api_ref = settings.get("weaver.wps_restapi_ref",
None) if weaver_api else None
weaver_api_spec = weaver_api_url + sd.openapi_json_service.path if weaver_api else None
weaver_wps = asbool(settings.get("weaver.wps"))
weaver_wps_url = get_wps_url(settings) if weaver_wps else None
weaver_conform_url = weaver_url + sd.api_conformance_service.path
weaver_process_url = weaver_url + sd.processes_service.path
weaver_jobs_url = weaver_url + sd.jobs_service.path
weaver_links = [{
"href": weaver_url,
"rel": "self",
"type": CONTENT_TYPE_APP_JSON,
"title": "This landing page."
}, {
"href": weaver_conform_url,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/conformance",
"type": CONTENT_TYPE_APP_JSON,
"title": "Conformance classes implemented by this service."
}, {
"href": __meta__.__license_url__,
"rel": "license",
"type": CONTENT_TYPE_TEXT_PLAIN,
"title": __meta__.__license_long__
}]
if weaver_api:
weaver_links.extend([
{
"href": weaver_api_url,
"rel": "service",
"type": CONTENT_TYPE_APP_JSON,
"title": "WPS REST API endpoint of this service."
},
{
"href": weaver_api_spec,
"rel": "service-desc",
"type": CONTENT_TYPE_APP_JSON,
"title": "OpenAPI specification of this service."
},
{
"href": weaver_api_oas_ui,
"rel": "service-doc",
"type": CONTENT_TYPE_TEXT_HTML,
"title":
"Human readable OpenAPI documentation of this service."
},
{
"href": weaver_api_spec,
"rel": "OpenAPI",
"type": CONTENT_TYPE_APP_JSON,
"title": "OpenAPI specification of this service."
},
{
"href": weaver_api_swagger,
"rel": "swagger-ui",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS REST API definition of this service."
},
{
"href": weaver_process_url,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/processes",
"type": CONTENT_TYPE_APP_JSON,
"title": "Processes offered by this service."
},
{
"href": sd.OGC_API_REPO_URL,
"rel": "ogcapi-processes-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "OGC API - Processes schema definitions repository."
},
{
"href":
weaver_jobs_url,
"rel":
"http://www.opengis.net/def/rel/ogc/1.0/job-list",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Job search and listing endpoint of executions registered under this service."
},
{
"href": sd.CWL_BASE_URL,
"rel": "cwl-home",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) homepage."
},
{
"href": sd.CWL_REPO_URL,
"rel": "cwl-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) repositories."
},
{
"href": sd.CWL_SPEC_URL,
"rel": "cwl-specification",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) specification."
},
{
"href": sd.CWL_USER_GUIDE_URL,
"rel": "cwl-user-guide",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "Common Workflow Language (CWL) user guide."
},
{
"href":
sd.CWL_CMD_TOOL_URL,
"rel":
"cwl-command-line-tool",
"type":
CONTENT_TYPE_TEXT_HTML,
"title":
"Common Workflow Language (CWL) CommandLineTool specification."
},
{
"href": sd.CWL_WORKFLOW_URL,
"rel": "cwl-workflow",
"type": CONTENT_TYPE_TEXT_HTML,
"title":
"Common Workflow Language (CWL) Workflow specification."
},
])
if weaver_api_ref:
# sample:
# https://app.swaggerhub.com/apis/geoprocessing/WPS/
weaver_links.append({
"href":
weaver_api_ref,
"rel":
"reference",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"API reference specification of this service."
})
if isinstance(weaver_api_doc, str):
# sample:
# https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/develop/docs/18-062.pdf
if "." in weaver_api_doc: # pylint: disable=E1135,unsupported-membership-test
ext_type = weaver_api_doc.split(".")[-1]
doc_type = "application/{}".format(ext_type)
else:
doc_type = CONTENT_TYPE_TEXT_PLAIN # default most basic type
weaver_links.append({
"href":
weaver_api_doc,
"rel":
"documentation",
"type":
doc_type,
"title":
"API reference documentation about this service."
})
else:
weaver_links.append({
"href":
__meta__.__documentation_url__,
"rel":
"documentation",
"type":
CONTENT_TYPE_TEXT_HTML,
"title":
"API reference documentation about this service."
})
if weaver_wps:
weaver_links.extend([
{
"href": weaver_wps_url,
"rel": "wps",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 1.0.0/2.0 XML endpoint of this service."
},
{
"href": "http://docs.opengeospatial.org/is/14-065/14-065.html",
"rel": "wps-specification",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS 1.0.0/2.0 definition of this service."
},
{
"href": "http://schemas.opengis.net/wps/",
"rel": "wps-schema-repository",
"type": CONTENT_TYPE_TEXT_HTML,
"title": "WPS 1.0.0/2.0 XML schemas repository."
},
{
"href": "http://schemas.opengis.net/wps/1.0.0/wpsAll.xsd",
"rel": "wps-schema-1",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 1.0.0 XML validation schemas entrypoint."
},
{
"href": "http://schemas.opengis.net/wps/2.0/wps.xsd",
"rel": "wps-schema-2",
"type": CONTENT_TYPE_TEXT_XML,
"title": "WPS 2.0 XML validation schemas entrypoint."
},
])
return {
"message":
"Weaver Information",
"configuration":
weaver_config,
"description":
__meta__.__description__,
"parameters": [
{
"name": "api",
"enabled": weaver_api,
"url": weaver_api_url,
"api": weaver_api_oas_ui
},
{
"name": "wps",
"enabled": weaver_wps,
"url": weaver_wps_url
},
],
"links":
weaver_links,
}
def wps_execute(self, version, accept):
wps_url = get_wps_url(self.settings)
if version == "1.0.0":
test_content = "Test file in Docker - WPS KVP"
wps_method = "GET"
elif version == "2.0.0":
test_content = "Test file in Docker - WPS XML"
wps_method = "POST"
else:
raise ValueError("Invalid WPS version: {}".format(version))
test_content += " {} request - Accept {}".format(wps_method, accept.split("/")[-1].upper())
with contextlib.ExitStack() as stack_exec:
# setup
dir_name = tempfile.gettempdir()
tmp_file = stack_exec.enter_context(tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt"))
tmp_file.write(test_content)
tmp_file.seek(0)
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
# execute
if version == "1.0.0":
wps_inputs = ["file={}@mimeType={}".format(tmp_file.name, CONTENT_TYPE_TEXT_PLAIN)]
wps_params = {
"service": "WPS",
"request": "Execute",
"version": version,
"identifier": self.process_id,
"DataInputs": wps_inputs,
}
wps_headers = {"Accept": accept}
wps_data = None
else:
wps_inputs = [("file", ComplexDataInput(tmp_file.name, mimeType=CONTENT_TYPE_TEXT_PLAIN))]
wps_outputs = [(self.out_key, True)] # as reference
wps_exec = WPSExecution(version=version, url=wps_url)
wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs)
wps_data = xml_util.tostring(wps_req)
wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML}
wps_params = None
resp = mocked_sub_requests(self.app, wps_method, wps_url,
params=wps_params, data=wps_data, headers=wps_headers, only_local=True)
assert resp.status_code in [200, 201], (
"Failed with: [{}]\nTest: [{}]\nReason:\n{}".format(resp.status_code, test_content, resp.text)
)
# parse response status
if accept == CONTENT_TYPE_APP_XML:
assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content
xml_body = xml_util.fromstring(str2bytes(resp.text))
status_url = xml_body.get("statusLocation")
job_id = status_url.split("/")[-1].split(".")[0]
elif accept == CONTENT_TYPE_APP_JSON:
assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content
status_url = resp.json["location"]
job_id = resp.json["jobID"]
assert status_url
assert job_id
if accept == CONTENT_TYPE_APP_XML:
wps_out_url = self.settings["weaver.wps_output_url"]
weaver_url = self.settings["weaver.url"]
assert status_url == f"{wps_out_url}/{job_id}.xml", "Status URL should be XML file for WPS-1 request"
# remap to employ JSON monitor method (could be done with XML parsing otherwise)
status_url = f"{weaver_url}/jobs/{job_id}"
# job monitoring
results = self.monitor_job(status_url)
outputs = self.get_outputs(status_url)
# validate XML status is updated accordingly
wps_xml_status = os.path.join(self.settings["weaver.wps_output_dir"], job_id + ".xml")
assert os.path.isfile(wps_xml_status)
with open(wps_xml_status, "r") as status_file:
assert "ProcessSucceeded" in status_file.read()
self.validate_outputs(job_id, results, outputs, test_content)