def parse_wps_process_config(config_entry):
# type: (Union[JSON, str]) -> Tuple[str, str, List[str], bool]
"""
Parses the available WPS provider or process entry to retrieve its relevant information.
:return: WPS provider name, WPS service URL, and list of process identifier(s).
:raise ValueError: if the entry cannot be parsed correctly.
"""
if isinstance(config_entry, dict):
svc_url = config_entry["url"]
svc_name = config_entry.get("name")
svc_proc = config_entry.get("id", [])
svc_vis = asbool(config_entry.get("visible", False))
elif isinstance(config_entry, str):
svc_url = config_entry
svc_name = None
svc_proc = []
svc_vis = False
else:
raise ValueError("Invalid service value: [{!s}].".format(config_entry))
url_p = urlparse(svc_url)
qs_p = parse_qs(url_p.query)
svc_url = get_url_without_query(url_p)
svc_name = svc_name or get_sane_name(url_p.hostname)
svc_proc = svc_proc or qs_p.get("identifier", []) # noqa
if not isinstance(svc_name, str):
raise ValueError("Invalid service value: [{!s}].".format(svc_name))
if not isinstance(svc_proc, list):
raise ValueError("Invalid process value: [{!s}].".format(svc_proc))
return svc_name, svc_url, svc_proc, svc_vis
def save_service(self, service, overwrite=True):
# type: (Service, bool) -> Service
"""
Stores an OWS service in mongodb.
"""
service_url = get_base_url(service.url)
# check if service is already registered
if self.collection.count_documents({"url": service_url}) > 0:
if overwrite:
self.collection.delete_one({"url": service_url})
else:
raise ServiceRegistrationError("service url already registered.")
service_name = get_sane_name(service.name, **self.sane_name_config)
if self.collection.count_documents({"name": service_name}) > 0:
if overwrite:
self.collection.delete_one({"name": service_name})
else:
raise ServiceRegistrationError("service name already registered.")
self.collection.insert_one(Service(
url=service_url,
name=service_name,
type=service.type,
public=service.public,
auth=service.auth).params())
return self.fetch_by_url(url=service_url)
def delete_process(self, process_id, visibility=None):
# type: (str, Optional[str]) -> bool
"""
Removes process from database, optionally filtered by visibility.
If ``visibility=None``, the process is deleted (if existing) regardless of its visibility value.
"""
sane_name = get_sane_name(process_id, **self.sane_name_config)
process = self.fetch_by_id(sane_name, visibility=visibility)
if not process:
raise ProcessNotFound("Process '{}' could not be found.".format(sane_name))
return bool(self.collection.delete_one({"identifier": sane_name}).deleted_count)
def test_get_sane_name_replace():
kw = {"assert_invalid": False, "max_len": 25}
assert get_sane_name("Hummingbird", **kw) == "Hummingbird"
assert get_sane_name("MapMint Demo Instance", **kw) == "MapMint_Demo_Instance"
assert get_sane_name(None, **kw) is None # noqa
assert get_sane_name("12", **kw) is None
assert get_sane_name(" ab c ", **kw) == "ab_c"
assert get_sane_name("a_much_to_long_name_for_this_test", **kw) == "a_much_to_long_name_for_t"
def fetch_by_id(self, process_id, visibility=None):
# type: (str, Optional[str]) -> Process
"""
Get process for given `process_id` from storage, optionally filtered by `visibility`.
If ``visibility=None``, the process is retrieved (if existing) regardless of its visibility value.
:param process_id: process identifier
:param visibility: one value amongst `weaver.visibility`.
:return: An instance of :class:`weaver.datatype.Process`.
"""
sane_name = get_sane_name(process_id, **self.sane_name_config)
process = self.collection.find_one({"identifier": sane_name})
if not process:
raise ProcessNotFound("Process '{}' could not be found.".format(sane_name))
process = Process(process)
if visibility is not None and process.visibility != visibility:
raise ProcessNotAccessible("Process '{}' cannot be accessed.".format(sane_name))
return process
def save_process(self, process, overwrite=True):
# type: (Union[Process, ProcessWPS], bool) -> Process
"""
Stores a process in storage.
:param process: An instance of :class:`weaver.datatype.Process`.
:param overwrite: Overwrite the matching process instance by name if conflicting.
"""
process_id = self._get_process_id(process)
sane_name = get_sane_name(process_id, **self.sane_name_config)
if self.collection.count_documents({"identifier": sane_name}) > 0:
if overwrite:
self.collection.delete_one({"identifier": sane_name})
else:
raise ProcessRegistrationError("Process '{}' already registered.".format(sane_name))
process.identifier = sane_name # must use property getter/setter to match both 'Process' types
self._add_process(process)
return self.fetch_by_id(sane_name)
def parse_wps_process_config(config_entry):
# type: (Union[JSON, str]) -> Tuple[str, str, List[str], bool]
"""
Parses the available WPS provider or process entry to retrieve its relevant information.
:return: WPS provider name, WPS service URL, and list of process identifier(s).
:raise ValueError: if the entry cannot be parsed correctly.
"""
if isinstance(config_entry, dict):
svc_url = config_entry["url"]
svc_name = config_entry.get("name")
svc_proc = config_entry.get("id", [])
svc_vis = asbool(config_entry.get("visible", False))
elif isinstance(config_entry, str):
svc_url = config_entry
svc_name = None
svc_proc = []
svc_vis = False
else:
raise ValueError("Invalid service value: [{!s}].".format(config_entry))
url_p = urlparse(svc_url)
qs_p = parse_qs(url_p.query)
svc_url = get_url_without_query(url_p)
# if explicit name was provided, validate it (assert fail if not),
# otherwise replace silently bad character since since is requested to be inferred
svc_name = get_sane_name(svc_name or url_p.hostname,
assert_invalid=bool(svc_name))
svc_proc = svc_proc or qs_p.get(
"identifier", []) # noqa # 'identifier=a,b,c' techically allowed
svc_proc = [proc.strip() for proc in svc_proc
if proc.strip()] # remote empty
if not isinstance(svc_name, str):
raise ValueError("Invalid service value: [{!s}].".format(svc_name))
if not isinstance(svc_proc, list):
raise ValueError("Invalid process value: [{!s}].".format(svc_proc))
return svc_name, svc_url, svc_proc, svc_vis
def register_wps_processes_from_config(wps_processes_file_path, container):
# type: (Optional[FileSystemPathType], AnySettingsContainer) -> None
"""
Loads a `wps_processes.yml` file and registers `WPS-1` providers processes to the
current `Weaver` instance as equivalent `WPS-2` processes.
References listed under ``processes`` are registered.
When the reference is a service (provider), registration of each WPS process is done individually
for each of the specified providers with ID ``[service]_[process]`` per listed process by ``GetCapabilities``.
.. versionadded:: 1.14.0
When references are specified using ``providers`` section instead of ``processes``, the registration
only saves the remote WPS provider endpoint to dynamically populate WPS processes on demand.
.. seealso::
- `weaver.wps_processes.yml.example` for additional file format details
"""
if wps_processes_file_path is None:
warnings.warn("No file specified for WPS-1 providers registration.",
RuntimeWarning)
wps_processes_file_path = get_weaver_config_file(
"", WEAVER_DEFAULT_WPS_PROCESSES_CONFIG)
elif wps_processes_file_path == "":
warnings.warn(
"Configuration file for WPS-1 providers registration explicitly defined as empty in settings. "
"Not loading anything.", RuntimeWarning)
return
# reprocess the path in case it is relative to default config directory
wps_processes_file_path = get_weaver_config_file(
wps_processes_file_path,
WEAVER_DEFAULT_WPS_PROCESSES_CONFIG,
generate_default_from_example=False)
if wps_processes_file_path == "":
warnings.warn("No file specified for WPS-1 providers registration.",
RuntimeWarning)
return
LOGGER.info("Using WPS-1 provider processes file: [%s]",
wps_processes_file_path)
try:
with open(wps_processes_file_path, "r") as f:
processes_config = yaml.safe_load(f)
processes = processes_config.get("processes") or []
providers = processes_config.get("providers") or []
if not processes and not providers:
LOGGER.warning("Nothing to process from file: [%s]",
wps_processes_file_path)
return
db = get_db(container)
process_store = db.get_store(StoreProcesses)
service_store = db.get_store(StoreServices)
# either 'service' references to register every underlying 'process' individually
# or explicit 'process' references to register by themselves
for cfg_service in processes:
svc_name, svc_url, svc_proc, svc_vis = parse_wps_process_config(
cfg_service)
# fetch data
LOGGER.info("Fetching WPS-1: [%s]", svc_url)
wps = WebProcessingService(url=svc_url)
if LooseVersion(wps.version) >= LooseVersion("2.0"):
LOGGER.warning("Invalid WPS-1 provider, version was [%s]",
wps.version)
continue
wps_processes = [wps.describeprocess(p)
for p in svc_proc] or wps.processes
for wps_process in wps_processes:
proc_id = "{}_{}".format(svc_name,
get_sane_name(wps_process.identifier))
try:
process_store.fetch_by_id(proc_id)
except ProcessNotFound:
pass
else:
LOGGER.warning(
"Process already registered: [%s]. Skipping...",
proc_id)
continue
proc_url = "{}?service=WPS&request=DescribeProcess&identifier={}&version={}" \
.format(svc_url, wps_process.identifier, wps.version)
svc_vis = VISIBILITY_PUBLIC if svc_vis else VISIBILITY_PRIVATE
payload = {
"processDescription": {
"process": {
"id": proc_id,
"visibility": svc_vis
}
},
"executionUnit": [{
"href": proc_url
}],
"deploymentProfileName":
"http://www.opengis.net/profiles/eoc/wpsApplication",
}
try:
resp = deploy_process_from_payload(payload, container)
if resp.status_code == HTTPOk.code:
LOGGER.info("Process registered: [%s]", proc_id)
else:
raise RuntimeError(
"Process registration failed: [{}]".format(
proc_id))
except Exception as ex:
LOGGER.exception(
"Exception during process registration: [%r]. Skipping...",
ex)
continue
# direct WPS providers to register
for cfg_service in providers:
svc_name, svc_url, _, svc_vis = parse_wps_process_config(
cfg_service)
LOGGER.info("Register WPS-1 provider: [%s]", svc_url)
WebProcessingService(
url=svc_url) # only attempt fetch to validate it exists
try:
service_store.fetch_by_name(svc_name)
except ServiceNotFound:
pass
else:
LOGGER.warning(
"Provider already registered: [%s]. Skipping...", svc_name)
continue
try:
service_store.save_service(
Service(name=svc_name, url=svc_url, public=svc_vis))
except Exception as ex:
LOGGER.exception(
"Exception during provider registration: [%r]. Skipping...",
ex)
continue
LOGGER.info("Finished processing configuration file [%s].",
wps_processes_file_path)
except Exception as exc:
msg = "Invalid WPS-1 providers configuration file [{!r}].".format(exc)
LOGGER.exception(msg)
raise RuntimeError(msg)
def register_wps_processes_static(service_url, service_name,
service_visibility, service_processes,
container):
# type: (str, str, bool, List[str], AnySettingsContainer) -> None
"""
Register WPS-1 :term:`Process` under a service :term:`Provider` as static references.
For a given WPS provider endpoint, either iterates over all available processes under it to register them one
by one, or limit itself only to those of the reduced set specified by :paramref:`service_processes`.
The registered `WPS-1` processes generate a **static** reference, meaning that metadata of each process as well
as any other modifications to the real remote reference will not be tracked, including validation of even their
actual existence, or modifications to inputs/outputs. The :term:`Application Package` will only point to it
assuming it remains valid.
Each of the deployed processes using *static* reference will be accessible directly under `Weaver` endpoints::
/processes/<service-name>_<process-id>
The service is **NOT** deployed as :term:`Provider` since the processes are registered directly.
.. seealso::
- :func:`register_wps_processes_dynamic`
:param service_url: WPS-1 service location (where ``GetCapabilities`` and ``DescribeProcess`` requests can be made).
:param service_name: Identifier to employ for generating the full process identifier.
:param service_visibility: Visibility flag of the provider.
:param service_processes: process IDs under the service to be registered, or all if empty.
:param container: settings to retrieve required configuration settings.
"""
db = get_db(container)
process_store = db.get_store(StoreProcesses) # type: StoreProcesses
LOGGER.info("Fetching WPS-1: [%s]", service_url)
wps = get_wps_client(service_url, container)
if LooseVersion(wps.version) >= LooseVersion("2.0"):
LOGGER.warning("Invalid WPS-1 provider, version was [%s]", wps.version)
return
wps_processes = [wps.describeprocess(p)
for p in service_processes] or wps.processes
for wps_process in wps_processes:
proc_id = "{}_{}".format(service_name,
get_sane_name(wps_process.identifier))
proc_url = "{}?service=WPS&request=DescribeProcess&identifier={}&version={}".format(
service_url, wps_process.identifier, wps.version)
svc_vis = VISIBILITY_PUBLIC if service_visibility else VISIBILITY_PRIVATE
try:
old_process = process_store.fetch_by_id(proc_id)
except ProcessNotFound:
pass
else:
if (old_process.id == proc_id
and old_process.processDescriptionURL == proc_url
and old_process.visibility == svc_vis):
LOGGER.warning("Process already registered: [%s]. Skipping...",
proc_id)
continue
LOGGER.warning(
"Process matches registered one: [%s]. Updating details...",
proc_id)
payload = {
"processDescription": {
"process": {
"id": proc_id,
"visibility": svc_vis
}
},
"executionUnit": [{
"href": proc_url
}],
"deploymentProfileName":
"http://www.opengis.net/profiles/eoc/wpsApplication",
}
try:
resp = deploy_process_from_payload(payload,
container,
overwrite=True)
if resp.status_code == HTTPOk.code:
LOGGER.info("Process registered: [%s]", proc_id)
else:
raise RuntimeError(
"Process registration failed: [{}]".format(proc_id))
except Exception as ex:
LOGGER.exception(
"Exception during process registration: [%r]. Skipping...", ex)
continue