def init(self, resource, probe_vars):
"""
Probe contains the actual Probe parameters (from Models/DB) for
requests and a list of response Checks with their
functions and parameters
:param resource:
:param probe_vars:
:return: None
"""
self._resource = resource
self._probe_vars = probe_vars
self._parameters = probe_vars.parameters
self._check_vars = probe_vars.check_vars
self.response = None
# Create ProbeResult object that gathers all results for single Probe
self.result = ProbeResult(self, self._probe_vars)
def init(self, resource, probe_vars):
"""
Probe contains the actual Probe parameters (from Models/DB) for
requests and a list of response Checks with their
functions and parameters
:param resource:
:param probe_vars:
:return: None
"""
self._resource = resource
self._probe_vars = probe_vars
self._parameters = probe_vars.parameters
self._check_vars = probe_vars.check_vars
self.response = None
# Create ProbeResult object that gathers all results for single Probe
self.result = ProbeResult(self, self._probe_vars)
class Probe(Plugin):
"""
Base class for specific implementations to run a Probe with Checks.
Most Probes can be implemented using REQUEST_TEMPLATES parameterized
via actualized PARAM_DEFS but specialized Probes may implement
their own Requests and Checks, for example by "drilling down"
through OWS services on an OGC OWS endpoint starting at the
Capabilities level or for specific WWW:LINK-based REST APIs.
"""
# Generic attributes, subclassses override
RESOURCE_TYPE = 'Not Applicable'
"""
Type of GHC Resource e.g. 'OGC:WMS', default not applicable.
"""
# Request attributes, defaults, subclasses override
REQUEST_METHOD = 'GET'
"""
HTTP request method capitalized, GET (default) or POST.
"""
REQUEST_HEADERS = {}
"""
`dict` of optional requests headers.
"""
REQUEST_TEMPLATE = ''
"""
Template in standard Python `str.format(*args)`. The variables
like {service} and {version} within a template are filled from
actual values for parameters defined in PARAM_DEFS and substituted
from values or constant values specified by user in GUI and stored
in DB.
"""
# Parameter definitions and possible Checks,
# subclassses override
PARAM_DEFS = {}
"""
Parameter definitions mostly for `REQUEST_TEMPLATE` but potential other
uses in specific Probe implementations. Format is `dict` where each key
is a parameter name and the value a `dict` of: `type`, `description`,
`required`, `default`, `range` (value range) and optional `value` item.
If `value` specified, this value becomes fixed (non-editable) unless
overridden in subclass.
"""
CHECKS_AVAIL = {}
"""
Available Check (classes) for this Probe in `dict` format.
Key is a Check class (string), values are optional (default `{}`).
In the (constant) value 'parameters' and other attributes for
Check.PARAM_DEFS can be specified, including `default` if this Check
should be added to Probe on creation.
"""
METADATA_CACHE = {}
"""
Cache for metadata, like capabilities documents or OWSLib Service
instances. Saves doing multiple requests/responses. In particular for
endpoints with 50+ Layers.
"""
def __init__(self):
Plugin.__init__(self)
#
# Lifecycle : optionally expand params from Resource metadata
def expand_params(self, resource):
"""
Called after creation. Use to expand PARAM_DEFS, e.g. from Resource
metadata like WMS Capabilities. See e.g. WmsGetMapV1 class.
:param resource:
:return: None
"""
pass
def get_metadata(self, resource, version='any'):
"""
Get metadata, specific per Resource type.
:param resource:
:param version:
:return: Metadata object
"""
return 'md'
def get_metadata_cached(self, resource, version='any'):
"""
Get metadata, specific per Resource type, get from cache
if cached.
:param resource:
:param version:
:return: Metadata object
"""
key = '%s_%s_%s' % (resource.url, resource.resource_type, version)
metadata = None
if key in Probe.METADATA_CACHE:
entry = Probe.METADATA_CACHE[key]
delta = datetime.datetime.utcnow() - entry['time']
metadata = entry['metadata']
# Don't keep cache forever, refresh every N mins
if delta.seconds > App.get_config()['GHC_METADATA_CACHE_SECS']:
entry = Probe.METADATA_CACHE.pop(key)
del entry
metadata = None
if not metadata:
# Get actual metadata, Resource-type specifc
metadata = self.get_metadata(resource, version)
if metadata and App.get_config()['GHC_METADATA_CACHE_SECS'] > 0:
# Store entry with time, for expiry later
entry = {
"metadata": metadata,
"time": datetime.datetime.utcnow()
}
Probe.METADATA_CACHE[key] = entry
return metadata
# Lifecycle
def init(self, resource, probe_vars):
"""
Probe contains the actual Probe parameters (from Models/DB) for
requests and a list of response Checks with their
functions and parameters
:param resource:
:param probe_vars:
:return: None
"""
self._resource = resource
self._probe_vars = probe_vars
self._parameters = probe_vars.parameters
self._check_vars = probe_vars.check_vars
self.response = None
# Create ProbeResult object that gathers all results for single Probe
self.result = ProbeResult(self, self._probe_vars)
#
# Lifecycle
def exit(self):
pass
def get_var_names(self):
var_names = Plugin.get_var_names(self)
var_names.extend([
'RESOURCE_TYPE', 'REQUEST_METHOD', 'REQUEST_HEADERS',
'REQUEST_TEMPLATE', 'CHECKS_AVAIL'
])
return var_names
def expand_check_vars(self, checks_avail):
for check_class in checks_avail:
check_avail = checks_avail[check_class]
check = Factory.create_obj(check_class)
check_vars = Plugin.copy(check.get_plugin_vars())
# Check if Probe class overrides Check Params
# mainly "value" entries.
if 'set_params' in check_avail:
set_params = check_avail['set_params']
for set_param in set_params:
if set_param in check_vars['PARAM_DEFS']:
param_orig = check_vars['PARAM_DEFS'][set_param]
param_override = set_params[set_param]
param_def = Plugin.merge(param_orig, param_override)
check_vars['PARAM_DEFS'][set_param] = param_def
checks_avail[check_class] = check_vars
return checks_avail
def get_checks_info_defaults(self):
checks_avail = self.get_checks_info()
checks_avail_default = {}
for check_class in checks_avail:
check_avail = checks_avail[check_class]
# Only include default Checks if specified
if 'default' in check_avail and check_avail['default']:
checks_avail_default[check_class] = check_avail
return checks_avail_default
def get_checks_info(self):
return Plugin.copy(Plugin.get_plugin_vars(self))['CHECKS_AVAIL']
def get_plugin_vars(self):
probe_vars = Plugin.copy(Plugin.get_plugin_vars(self))
probe_vars['CHECKS_AVAIL'] = \
self.expand_check_vars(probe_vars['CHECKS_AVAIL'])
return probe_vars
def log(self, text):
LOGGER.info(text)
def before_request(self):
""" Before running actual request to service"""
pass
def after_request(self):
""" After running actual request to service"""
pass
def get_request_headers(self):
return self.REQUEST_HEADERS
def perform_request(self):
""" Perform actual request to service"""
# Actualize request query string or POST body
# by substitution in template.
url_base = self._resource.url
request_string = None
if self.REQUEST_TEMPLATE:
request_string = self.REQUEST_TEMPLATE
if '?' in url_base and self.REQUEST_TEMPLATE[0] == '?':
self.REQUEST_TEMPLATE = '&' + self.REQUEST_TEMPLATE[1:]
if self._parameters:
request_parms = self._parameters
param_defs = self.get_param_defs()
# Expand string list array to comma separated string
for param in request_parms:
if param_defs[param]['type'] == 'stringlist':
request_parms[param] = ','.join(request_parms[param])
request_string = self.REQUEST_TEMPLATE.format(**request_parms)
self.log('Requesting: %s url=%s' % (self.REQUEST_METHOD, url_base))
if self.REQUEST_METHOD == 'GET':
# Default is plain URL, e.g. for WWW:LINK
url = url_base
if request_string:
# Query String: mainly OWS:* resources
url = "%s%s" % (url, request_string)
self.response = requests.get(url,
headers=self.get_request_headers())
elif self.REQUEST_METHOD == 'POST':
self.response = requests.post(url_base,
data=request_string,
headers=self.get_request_headers())
self.log('response: status=%d' % self.response.status_code)
if self.response.status_code / 100 in [4, 5]:
self.log('Error response: %s' % (str(self.response.text)))
def run_request(self):
""" Run actual request to service"""
try:
self.before_request()
self.result.start()
self.perform_request()
self.result.stop()
self.after_request()
except:
# We must never bailout because of Exception
# in Probe.
msg = "Probe Err: %s" % str(sys.exc_info())
LOGGER.error(msg)
self.result.set(False, msg)
def run_checks(self):
""" Do the checks on the response from request"""
# Config also determines which actual checks are performed
# from possible Checks in Probe. Checks are performed
# by Check instances.
for check_var in self._check_vars:
check = None
try:
check_class = check_var.check_class
check = Factory.create_obj(check_class)
except:
LOGGER.error("Cannot create Check class: %s %s" %
(check_class, str(sys.exc_info())))
if not check:
continue
try:
check.init(self, check_var)
check.perform()
except:
msg = "Check Err: %s" % str(sys.exc_info())
LOGGER.error(msg)
check.set_result(False, msg)
self.log('Check: fun=%s result=%s' %
(check_class, check._result.success))
self.result.add_result(check._result)
# Lifecycle
def calc_result(self):
""" Calculate overall result from the Result object"""
self.log("Result: %s" % str(self.result))
@staticmethod
def run(resource, probe_vars):
"""
Class method to create and run a single Probe
instance. Follows strict sequence of method calls.
Each method can be overridden in subclass.
"""
probe = None
try:
# Create Probe instance from module.class string
probe = Factory.create_obj(probe_vars.probe_class)
except:
LOGGER.error("Cannot create Probe class: %s %s" %
(probe_vars.probe_class, str(sys.exc_info())))
if not probe:
return
# Initialize with actual parameters
probe.init(resource, probe_vars)
# Perform request
probe.run_request()
# Perform the Probe's checks
probe.run_checks()
# Determine result
probe.calc_result()
# Lifecycle
probe.exit()
# Return result
return probe.result
class Probe(Plugin):
"""
Base class for specific implementations to run a Probe with Checks.
Most Probes can be implemented using REQUEST_TEMPLATES parameterized
via actualized PARAM_DEFS but specialized Probes may implement
their own Requests and Checks, for example by "drilling down"
through OWS services on an OGC OWS endpoint starting at the
Capabilities level or for specific WWW:LINK-based REST APIs.
"""
# Generic attributes, subclassses override
RESOURCE_TYPE = 'Not Applicable'
"""
Type of GHC Resource e.g. 'OGC:WMS', default not applicable.
"""
# Request attributes, defaults, subclasses override
REQUEST_METHOD = 'GET'
"""
HTTP request method capitalized, GET (default) or POST.
"""
REQUEST_HEADERS = {}
"""
`dict` of optional requests headers.
"""
REQUEST_TEMPLATE = ''
"""
Template in standard Python `str.format(*args)`. The variables
like {service} and {version} within a template are filled from
actual values for parameters defined in PARAM_DEFS and substituted
from values or constant values specified by user in GUI and stored
in DB.
"""
# Parameter definitions and possible Checks,
# subclassses override
PARAM_DEFS = {}
"""
Parameter definitions mostly for `REQUEST_TEMPLATE` but potential other
uses in specific Probe implementations. Format is `dict` where each key
is a parameter name and the value a `dict` of: `type`, `description`,
`required`, `default`, `range` (value range) and optional `value` item.
If `value` specified, this value becomes fixed (non-editable) unless
overridden in subclass.
"""
CHECKS_AVAIL = {}
"""
Available Check (classes) for this Probe in `dict` format.
Key is a Check class (string), values are optional (default `{}`).
In the (constant) value 'parameters' and other attributes for
Check.PARAM_DEFS can be specified, including `default` if this Check
should be added to Probe on creation.
"""
METADATA_CACHE = {}
"""
Cache for metadata, like capabilities documents or OWSLib Service
instances. Saves doing multiple requests/responses. In particular for
endpoints with 50+ Layers.
"""
def __init__(self):
Plugin.__init__(self)
#
# Lifecycle : optionally expand params from Resource metadata
def expand_params(self, resource):
"""
Called after creation. Use to expand PARAM_DEFS, e.g. from Resource
metadata like WMS Capabilities. See e.g. WmsGetMapV1 class.
:param resource:
:return: None
"""
pass
def get_metadata(self, resource, version='any'):
"""
Get metadata, specific per Resource type.
:param resource:
:param version:
:return: Metadata object
"""
return 'md'
def get_metadata_cached(self, resource, version='any'):
"""
Get metadata, specific per Resource type, get from cache
if cached.
:param resource:
:param version:
:return: Metadata object
"""
key = '%s_%s_%s' % (resource.url, resource.resource_type,
version)
metadata = None
if key in Probe.METADATA_CACHE:
entry = Probe.METADATA_CACHE[key]
delta = datetime.datetime.utcnow() - entry['time']
metadata = entry['metadata']
# Don't keep cache forever, refresh every N mins
if delta.seconds > App.get_config()['GHC_METADATA_CACHE_SECS']:
entry = Probe.METADATA_CACHE.pop(key)
del entry
metadata = None
if not metadata:
# Get actual metadata, Resource-type specifc
metadata = self.get_metadata(resource, version)
if metadata and App.get_config()['GHC_METADATA_CACHE_SECS'] > 0:
# Store entry with time, for expiry later
entry = {
"metadata": metadata,
"time": datetime.datetime.utcnow()
}
Probe.METADATA_CACHE[key] = entry
return metadata
# Lifecycle
def init(self, resource, probe_vars):
"""
Probe contains the actual Probe parameters (from Models/DB) for
requests and a list of response Checks with their
functions and parameters
:param resource:
:param probe_vars:
:return: None
"""
self._resource = resource
self._probe_vars = probe_vars
self._parameters = probe_vars.parameters
self._check_vars = probe_vars.check_vars
self.response = None
# Create ProbeResult object that gathers all results for single Probe
self.result = ProbeResult(self, self._probe_vars)
#
# Lifecycle
def exit(self):
pass
def get_var_names(self):
var_names = Plugin.get_var_names(self)
var_names.extend([
'RESOURCE_TYPE',
'REQUEST_METHOD',
'REQUEST_HEADERS',
'REQUEST_TEMPLATE',
'CHECKS_AVAIL'
])
return var_names
def expand_check_vars(self, checks_avail):
for check_class in checks_avail:
check_avail = checks_avail[check_class]
check = Factory.create_obj(check_class)
check_vars = Plugin.copy(check.get_plugin_vars())
# Check if Probe class overrides Check Params
# mainly "value" entries.
if 'set_params' in check_avail:
set_params = check_avail['set_params']
for set_param in set_params:
if set_param in check_vars['PARAM_DEFS']:
param_orig = check_vars['PARAM_DEFS'][set_param]
param_override = set_params[set_param]
param_def = Plugin.merge(param_orig, param_override)
check_vars['PARAM_DEFS'][set_param] = param_def
checks_avail[check_class] = check_vars
return checks_avail
def get_checks_info_defaults(self):
checks_avail = self.get_checks_info()
checks_avail_default = {}
for check_class in checks_avail:
check_avail = checks_avail[check_class]
# Only include default Checks if specified
if 'default' in check_avail and check_avail['default']:
checks_avail_default[check_class] = check_avail
return checks_avail_default
def get_checks_info(self):
return Plugin.copy(Plugin.get_plugin_vars(self))['CHECKS_AVAIL']
def get_plugin_vars(self):
probe_vars = Plugin.copy(Plugin.get_plugin_vars(self))
probe_vars['CHECKS_AVAIL'] = \
self.expand_check_vars(probe_vars['CHECKS_AVAIL'])
return probe_vars
def log(self, text):
LOGGER.info(text)
def before_request(self):
""" Before running actual request to service"""
pass
def after_request(self):
""" After running actual request to service"""
pass
def get_request_headers(self):
return self.REQUEST_HEADERS
def perform_request(self):
""" Perform actual request to service"""
# Actualize request query string or POST body
# by substitution in template.
url_base = self._resource.url
request_string = None
if self.REQUEST_TEMPLATE:
request_string = self.REQUEST_TEMPLATE
if '?' in url_base and self.REQUEST_TEMPLATE[0] == '?':
self.REQUEST_TEMPLATE = '&' + self.REQUEST_TEMPLATE[1:]
if self._parameters:
request_parms = self._parameters
param_defs = self.get_param_defs()
# Expand string list array to comma separated string
for param in request_parms:
if param_defs[param]['type'] == 'stringlist':
request_parms[param] = ','.join(request_parms[param])
request_string = self.REQUEST_TEMPLATE.format(**request_parms)
self.log('Requesting: %s url=%s' % (self.REQUEST_METHOD, url_base))
try:
headers = self.get_request_headers()
if self.REQUEST_METHOD == 'GET':
# Default is plain URL, e.g. for WWW:LINK
url = url_base
if request_string:
# Query String: mainly OWS:* resources
url = "%s%s" % (url, request_string)
self.response = requests.get(url,
headers=headers)
elif self.REQUEST_METHOD == 'POST':
self.response = requests.post(url_base,
data=request_string,
headers=headers)
except requests.exceptions.RequestException as e:
msg = "Request Err: %s %s" % (e.__class__.__name__, str(e))
self.result.set(False, msg)
if self.response:
self.log('response: status=%d' % self.response.status_code)
if self.response.status_code / 100 in [4, 5]:
self.log('Error response: %s' % (str(self.response.text)))
def run_request(self):
""" Run actual request to service"""
try:
self.before_request()
self.result.start()
try:
self.perform_request()
except Exception as e:
msg = "Perform_request Err: %s %s" % \
(e.__class__.__name__, str(e))
self.result.set(False, msg)
self.result.stop()
self.after_request()
except Exception as e:
# We must never bailout because of Exception
# in Probe.
msg = "Probe Err: %s %s" % (e.__class__.__name__, str(e))
LOGGER.error(msg)
self.result.set(False, msg)
def run_checks(self):
""" Do the checks on the response from request"""
# Do not run Checks if Probe already failed
if not self.result.success:
return
# Config also determines which actual checks are performed
# from possible Checks in Probe. Checks are performed
# by Check instances.
for check_var in self._check_vars:
check = None
try:
check_class = check_var.check_class
check = Factory.create_obj(check_class)
except Exception:
LOGGER.error("Cannot create Check class: %s %s"
% (check_class, str(sys.exc_info())))
if not check:
continue
try:
check.init(self, check_var)
check.perform()
except Exception:
msg = "Check Err: %s" % str(sys.exc_info())
LOGGER.error(msg)
check.set_result(False, msg)
self.log('Check: fun=%s result=%s' % (check_class,
check._result.success))
self.result.add_result(check._result)
# Lifecycle
def calc_result(self):
""" Calculate overall result from the Result object"""
self.log("Result: %s" % str(self.result))
@staticmethod
def run(resource, probe_vars):
"""
Class method to create and run a single Probe
instance. Follows strict sequence of method calls.
Each method can be overridden in subclass.
"""
probe = None
try:
# Create Probe instance from module.class string
probe = Factory.create_obj(probe_vars.probe_class)
except Exception:
LOGGER.error("Cannot create Probe class: %s %s"
% (probe_vars.probe_class, str(sys.exc_info())))
if not probe:
return
# Initialize with actual parameters
probe.init(resource, probe_vars)
# Perform request
probe.run_request()
# Perform the Probe's checks
probe.run_checks()
# Determine result
probe.calc_result()
# Lifecycle
probe.exit()
# Return result
return probe.result