def fetch_data_sources():
# type: () -> DataSourceConfig
global DATA_SOURCES # pylint: disable=W0603,global-statement
if DATA_SOURCES:
return DATA_SOURCES
settings = get_settings()
data_source_config = settings.get("weaver.data_sources", "")
if data_source_config:
data_source_config = get_weaver_config_file(
str(data_source_config), WEAVER_DEFAULT_DATA_SOURCES_CONFIG)
if not os.path.isabs(data_source_config):
data_source_config = os.path.normpath(
os.path.join(WEAVER_ROOT_DIR, data_source_config))
try:
with open(data_source_config, mode="r", encoding="utf-8") as f:
DATA_SOURCES = yaml.safe_load(f) # both JSON/YAML
except Exception as exc:
raise ValueError(
f"Data sources file [{data_source_config}] cannot be loaded due to error: [{exc!r}]."
)
if not DATA_SOURCES:
raise ValueError(
"No data sources found in setting 'weaver.data_sources'. Data source required for EMS."
)
return DATA_SOURCES
def main(global_config, **settings):
"""
Creates a Pyramid WSGI application for Weaver.
"""
setup_loggers(settings)
LOGGER.info("Initiating weaver application")
# validate and fix configuration
weaver_config = get_weaver_configuration(settings)
settings.update({"weaver.configuration": weaver_config})
# Parse extra_options and add each of them in the settings dict
LOGGER.info("Parsing extra options...")
settings.update(
parse_extra_options(settings.get("weaver.extra_options", "")))
# load requests options if found, otherwise skip
LOGGER.info("Checking for request options file...")
req_file = get_weaver_config_file(settings.get("weaver.request_options",
""),
WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG,
generate_default_from_example=False)
if req_file:
LOGGER.info("Loading request options...")
with open(req_file, "r") as f:
settings.update({"weaver.request_options": yaml.safe_load(f)})
else:
LOGGER.warning("No request options found.")
# add default caching regions if they were omitted in config file
if settings.get("weaver.celery", False):
LOGGER.info("Celery runner detected. Skipping cache options setup.")
else:
LOGGER.info("Adding default caching options...")
setup_cache(settings)
LOGGER.info("Setup celery configuration...")
local_config = Configurator(settings=settings)
if global_config.get("__file__") is not None:
local_config.include("pyramid_celery")
local_config.configure_celery(global_config["__file__"])
local_config.include("weaver")
LOGGER.info("Running database migration...")
db = get_db(settings)
db.run_migration()
if settings.get("weaver.celery", False):
LOGGER.info("Celery runner detected. Skipping process registration.")
else:
LOGGER.info("Registering builtin processes...")
register_builtin_processes(local_config)
LOGGER.info("Registering WPS-1 processes from configuration file...")
wps_processes_file = get_settings(local_config).get(
"weaver.wps_processes_file")
register_wps_processes_from_config(wps_processes_file, local_config)
return local_config.make_wsgi_app()
def get_settings_from_config_ini(config_ini_path=None,
ini_section_name="app:main"):
# type: (Optional[str], str) -> SettingsType
parser = ConfigParser()
parser.read(
[get_weaver_config_file(config_ini_path, WEAVER_DEFAULT_INI_CONFIG)])
settings = dict(parser.items(ini_section_name))
return settings
def main(global_config, **settings):
"""
Creates a Pyramid WSGI application for Weaver.
"""
setup_loggers(settings)
LOGGER.info("Initiating weaver application")
# validate and fix configuration
weaver_config = get_weaver_configuration(settings)
settings.update({"weaver.configuration": weaver_config})
# Parse extra_options and add each of them in the settings dict
settings.update(
parse_extra_options(settings.get("weaver.extra_options", "")))
# load requests options if found, otherwise skip
req_file = get_weaver_config_file(settings.get("weaver.request_options",
""),
WEAVER_DEFAULT_REQUEST_OPTIONS_CONFIG,
generate_default_from_example=False)
if req_file:
with open(req_file, "r") as f:
settings.update({"weaver.request_options": yaml.safe_load(f)})
local_config = Configurator(settings=settings)
if global_config.get("__file__") is not None:
local_config.include("pyramid_celery")
local_config.configure_celery(global_config["__file__"])
local_config.include("weaver")
LOGGER.info("Registering builtin processes...")
register_builtin_processes(local_config)
LOGGER.info("Registering WPS-1 processes from configuration file...")
wps_processes_file = get_settings(local_config).get(
"weaver.wps_processes_file")
register_wps_processes_from_config(wps_processes_file, local_config)
return local_config.make_wsgi_app()
def fetch_data_sources():
global DATA_SOURCES # pylint: disable=W0603,global-statement
if DATA_SOURCES:
return DATA_SOURCES
data_source_config = get_settings(app).get("weaver.data_sources", "")
if data_source_config:
data_source_config = get_weaver_config_file(
str(data_source_config), WEAVER_DEFAULT_DATA_SOURCES_CONFIG)
if not os.path.isabs(data_source_config):
data_source_config = os.path.normpath(
os.path.join(WEAVER_ROOT_DIR, data_source_config))
try:
with open(data_source_config) as f:
DATA_SOURCES = yaml.safe_load(f) # both JSON/YAML
except Exception as exc:
raise ValueError(
"Data sources file [{0}] cannot be loaded due to error: [{1!r}]."
.format(data_source_config, exc))
if not DATA_SOURCES:
raise ValueError(
"No data sources found in setting 'weaver.data_sources'.")
return DATA_SOURCES
def register_wps_processes_from_config(wps_processes_file_path, container):
# type: (Optional[FileSystemPathType], AnySettingsContainer) -> None
"""
Loads a `wps_processes.yml` file and registers `WPS-1` providers processes to the
current `Weaver` instance as equivalent `WPS-2` processes.
References listed under ``processes`` are registered.
When the reference is a service (provider), registration of each WPS process is done individually
for each of the specified providers with ID ``[service]_[process]`` per listed process by ``GetCapabilities``.
.. versionadded:: 1.14.0
When references are specified using ``providers`` section instead of ``processes``, the registration
only saves the remote WPS provider endpoint to dynamically populate WPS processes on demand.
.. seealso::
- `weaver.wps_processes.yml.example` for additional file format details
"""
if wps_processes_file_path is None:
warnings.warn("No file specified for WPS-1 providers registration.",
RuntimeWarning)
wps_processes_file_path = get_weaver_config_file(
"", WEAVER_DEFAULT_WPS_PROCESSES_CONFIG)
elif wps_processes_file_path == "":
warnings.warn(
"Configuration file for WPS-1 providers registration explicitly defined as empty in settings. "
"Not loading anything.", RuntimeWarning)
return
# reprocess the path in case it is relative to default config directory
wps_processes_file_path = get_weaver_config_file(
wps_processes_file_path,
WEAVER_DEFAULT_WPS_PROCESSES_CONFIG,
generate_default_from_example=False)
if wps_processes_file_path == "":
warnings.warn("No file specified for WPS-1 providers registration.",
RuntimeWarning)
return
LOGGER.info("Using WPS-1 provider processes file: [%s]",
wps_processes_file_path)
try:
with open(wps_processes_file_path, "r") as f:
processes_config = yaml.safe_load(f)
processes = processes_config.get("processes") or []
providers = processes_config.get("providers") or []
if not processes and not providers:
LOGGER.warning("Nothing to process from file: [%s]",
wps_processes_file_path)
return
db = get_db(container)
process_store = db.get_store(StoreProcesses)
service_store = db.get_store(StoreServices)
# either 'service' references to register every underlying 'process' individually
# or explicit 'process' references to register by themselves
for cfg_service in processes:
svc_name, svc_url, svc_proc, svc_vis = parse_wps_process_config(
cfg_service)
# fetch data
LOGGER.info("Fetching WPS-1: [%s]", svc_url)
wps = WebProcessingService(url=svc_url)
if LooseVersion(wps.version) >= LooseVersion("2.0"):
LOGGER.warning("Invalid WPS-1 provider, version was [%s]",
wps.version)
continue
wps_processes = [wps.describeprocess(p)
for p in svc_proc] or wps.processes
for wps_process in wps_processes:
proc_id = "{}_{}".format(svc_name,
get_sane_name(wps_process.identifier))
try:
process_store.fetch_by_id(proc_id)
except ProcessNotFound:
pass
else:
LOGGER.warning(
"Process already registered: [%s]. Skipping...",
proc_id)
continue
proc_url = "{}?service=WPS&request=DescribeProcess&identifier={}&version={}" \
.format(svc_url, wps_process.identifier, wps.version)
svc_vis = VISIBILITY_PUBLIC if svc_vis else VISIBILITY_PRIVATE
payload = {
"processDescription": {
"process": {
"id": proc_id,
"visibility": svc_vis
}
},
"executionUnit": [{
"href": proc_url
}],
"deploymentProfileName":
"http://www.opengis.net/profiles/eoc/wpsApplication",
}
try:
resp = deploy_process_from_payload(payload, container)
if resp.status_code == HTTPOk.code:
LOGGER.info("Process registered: [%s]", proc_id)
else:
raise RuntimeError(
"Process registration failed: [{}]".format(
proc_id))
except Exception as ex:
LOGGER.exception(
"Exception during process registration: [%r]. Skipping...",
ex)
continue
# direct WPS providers to register
for cfg_service in providers:
svc_name, svc_url, _, svc_vis = parse_wps_process_config(
cfg_service)
LOGGER.info("Register WPS-1 provider: [%s]", svc_url)
WebProcessingService(
url=svc_url) # only attempt fetch to validate it exists
try:
service_store.fetch_by_name(svc_name)
except ServiceNotFound:
pass
else:
LOGGER.warning(
"Provider already registered: [%s]. Skipping...", svc_name)
continue
try:
service_store.save_service(
Service(name=svc_name, url=svc_url, public=svc_vis))
except Exception as ex:
LOGGER.exception(
"Exception during provider registration: [%r]. Skipping...",
ex)
continue
LOGGER.info("Finished processing configuration file [%s].",
wps_processes_file_path)
except Exception as exc:
msg = "Invalid WPS-1 providers configuration file [{!r}].".format(exc)
LOGGER.exception(msg)
raise RuntimeError(msg)
def register_wps_processes_from_config(wps_processes_file_path, container):
# type: (Optional[FileSystemPathType], AnySettingsContainer) -> None
"""
Registers remote `WPS` providers and/or processes as specified from the configuration file.
Loads a `wps_processes.yml` file and registers `WPS-1` providers processes to the
current `Weaver` instance as equivalent `WPS-2` processes.
References listed under ``processes`` are registered.
When the reference is a service (provider), registration of each WPS process is done individually
for each of the specified providers with ID ``[service]_[process]`` per listed process by ``GetCapabilities``.
.. versionadded:: 1.14.0
When references are specified using ``providers`` section instead of ``processes``, the registration
only saves the remote WPS provider endpoint to dynamically populate WPS processes on demand.
.. seealso::
- `weaver.wps_processes.yml.example` for additional file format details
"""
if wps_processes_file_path is None:
warnings.warn("No file specified for WPS-1 providers registration.",
RuntimeWarning)
wps_processes_file_path = get_weaver_config_file(
"",
WEAVER_DEFAULT_WPS_PROCESSES_CONFIG,
generate_default_from_example=False)
elif wps_processes_file_path == "":
warnings.warn(
"Configuration file for WPS-1 providers registration explicitly defined as empty in settings. "
"Not loading anything.", RuntimeWarning)
return
# reprocess the path in case it is relative to default config directory
wps_processes_file_path = get_weaver_config_file(
wps_processes_file_path,
WEAVER_DEFAULT_WPS_PROCESSES_CONFIG,
generate_default_from_example=False)
if wps_processes_file_path == "":
warnings.warn("No file specified for WPS-1 providers registration.",
RuntimeWarning)
return
LOGGER.info("Using WPS-1 provider processes file: [%s]",
wps_processes_file_path)
try:
with open(wps_processes_file_path, "r") as f:
# if file is empty (not even processes/providers section), None is return instead of dict
processes_config = yaml.safe_load(f) or {}
if processes_config:
processes = processes_config.get("processes") or []
providers = processes_config.get("providers") or []
else:
processes = providers = None
if not processes and not providers:
LOGGER.warning("Nothing to process from file: [%s]",
wps_processes_file_path)
return
# either 'service' references to register every underlying 'process' individually
# or explicit 'process' references to register by themselves
for cfg_service in processes:
svc_name, svc_url, svc_proc, svc_vis = parse_wps_process_config(
cfg_service)
register_wps_processes_static(svc_url, svc_name, svc_vis, svc_proc,
container)
# direct WPS providers to register
for cfg_service in providers:
svc_name, svc_url, _, svc_vis = parse_wps_process_config(
cfg_service)
register_wps_processes_dynamic(svc_name, svc_url, svc_vis,
container)
LOGGER.info("Finished processing configuration file [%s].",
wps_processes_file_path)
except Exception as exc:
msg = "Invalid WPS-1 providers configuration file caused: [{!s}]({!s}).".format(
type(exc).__name__, exc)
LOGGER.exception(msg)
raise RuntimeError(msg)