def get_metadata_cached(self, resource, version='any'):
"""
Get metadata, specific per Resource type, get from cache
if cached.
:param resource:
:param version:
:return: Metadata object
"""
key = '%s_%s_%s' % (resource.url, resource.resource_type, version)
metadata = None
if key in Probe.METADATA_CACHE:
entry = Probe.METADATA_CACHE[key]
delta = datetime.datetime.utcnow() - entry['time']
metadata = entry['metadata']
# Don't keep cache forever, refresh every N mins
if delta.seconds > App.get_config()['GHC_METADATA_CACHE_SECS']:
entry = Probe.METADATA_CACHE.pop(key)
del entry
metadata = None
if not metadata:
# Get actual metadata, Resource-type specifc
metadata = self.get_metadata(resource, version)
if metadata and App.get_config()['GHC_METADATA_CACHE_SECS'] > 0:
# Store entry with time, for expiry later
entry = {
"metadata": metadata,
"time": datetime.datetime.utcnow()
}
Probe.METADATA_CACHE[key] = entry
return metadata
def perform_get_request(self, url):
""" Perform actual HTTP GET request to service"""
return self._session.get(
url,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
verify=App.get_config()['GHC_VERIFY_SSL'],
headers=self.get_request_headers())
def perform_post_request(self, url_base, request_string):
""" Perform actual HTTP POST request to service"""
return self._session.post(
url_base,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
verify=App.get_config()['GHC_VERIFY_SSL'],
data=request_string,
headers=self.get_request_headers())
def perform_request(self):
""" Perform actual request to service"""
# Actualize request query string or POST body
# by substitution in template.
url_base = self._resource.url
request_string = None
if self.REQUEST_TEMPLATE:
request_string = self.REQUEST_TEMPLATE
if '?' in url_base and self.REQUEST_TEMPLATE[0] == '?':
self.REQUEST_TEMPLATE = '&' + self.REQUEST_TEMPLATE[1:]
if self._parameters:
request_parms = self._parameters
param_defs = self.get_param_defs()
# Expand string list array to comma separated string
for param in request_parms:
if param_defs[param]['type'] == 'stringlist':
request_parms[param] = ','.join(request_parms[param])
request_string = self.REQUEST_TEMPLATE.format(**request_parms)
self.log('Requesting: %s url=%s' % (self.REQUEST_METHOD, url_base))
try:
headers = self.get_request_headers()
if self.REQUEST_METHOD == 'GET':
# Default is plain URL, e.g. for WWW:LINK
url = url_base
if request_string:
# Query String: mainly OWS:* resources
url = "%s%s" % (url, request_string)
self.response = requests.get(
url,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
headers=headers)
elif self.REQUEST_METHOD == 'POST':
self.response = requests.post(
url_base,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
data=request_string,
headers=headers)
except requests.exceptions.RequestException as e:
msg = "Request Err: %s %s" % (e.__class__.__name__, str(e))
self.result.set(False, msg)
if self.response:
self.log('response: status=%d' % self.response.status_code)
if self.response.status_code / 100 in [4, 5]:
self.log('Error response: %s' % (str(self.response.text)))
def perform_request(self):
""" Perform actual request to service"""
# Actualize request query string or POST body
# by substitution in template.
url_base = self._resource.url
request_string = None
if self.REQUEST_TEMPLATE:
request_string = self.REQUEST_TEMPLATE
if '?' in url_base and self.REQUEST_TEMPLATE[0] == '?':
self.REQUEST_TEMPLATE = '&' + self.REQUEST_TEMPLATE[1:]
if self._parameters:
request_parms = self._parameters
param_defs = self.get_param_defs()
# Expand string list array to comma separated string
for param in request_parms:
if param_defs[param]['type'] == 'stringlist':
request_parms[param] = ','.join(request_parms[param])
request_string = self.REQUEST_TEMPLATE.format(**request_parms)
self.log('Requesting: %s url=%s' % (self.REQUEST_METHOD, url_base))
try:
headers = self.get_request_headers()
if self.REQUEST_METHOD == 'GET':
# Default is plain URL, e.g. for WWW:LINK
url = url_base
if request_string:
# Query String: mainly OWS:* resources
url = "%s%s" % (url, request_string)
self.response = requests.get(
url,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
headers=headers)
elif self.REQUEST_METHOD == 'POST':
self.response = requests.post(
url_base,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
data=request_string,
headers=headers)
except requests.exceptions.RequestException as e:
msg = "Request Err: %s %s" % (e.__class__.__name__, str(e))
self.result.set(False, msg)
if self.response:
self.log('response: status=%d' % self.response.status_code)
if self.response.status_code / 100 in [4, 5]:
self.log('Error response: %s' % (str(self.response.text)))
def setUp(self):
# Need this for Resource Auth
App.get_config()['SECRET_KEY'] = 'mysecrettestkey'
self.db = DB
# do once per test
load_data('%s/data/fixtures.json' % TEST_DIR)
def perform_get_request(self, url):
""" Perform actual HTTP GET request to service"""
LOGGER.debug("request: %s" % str(url))
LOGGER.debug("headers: %s" % str(self.get_request_headers()))
return requests.get(
url,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
headers=self.get_request_headers())
def add_auth_header(self, headers_dict):
headers_dict = super(ArcGISTokenAuth,
self).add_auth_header(headers_dict)
# add referer header
referer_header = App.get_config()['GHC_SITE_URL']
headers_dict.update({'referer': str(referer_header)})
return headers_dict
def perform_post_request(self, url_base, request_string):
""" Perform actual HTTP POST request to service"""
LOGGER.debug("request: %s" % str(url_base))
LOGGER.debug("headers: %s" % str(self.get_request_headers()))
LOGGER.debug("data: %s" % str(request_string))
return requests.post(
url_base,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
data=request_string,
headers=self.get_request_headers())
def flush_runs():
APP = App.get_app()
retention_days = int(APP.config['GHC_RETENTION_DAYS'])
LOGGER.info('Flushing runs older than %d days' % retention_days)
all_runs = Run.query.all()
run_count = 0
for run in all_runs:
days_old = (datetime.utcnow() - run.checked_datetime).days
if days_old > retention_days:
run_count += 1
DB.session.delete(run)
db_commit()
LOGGER.info('Deleted %d Runs' % run_count)
DB.session.remove()
def get_metadata_cached(self, resource, version='any'):
"""
Get metadata, specific per Resource type, get from cache
if cached.
:param resource:
:param version:
:return: Metadata object
"""
key = '%s_%s_%s' % (resource.url, resource.resource_type,
version)
metadata = None
if key in Probe.METADATA_CACHE:
entry = Probe.METADATA_CACHE[key]
delta = datetime.datetime.utcnow() - entry['time']
metadata = entry['metadata']
# Don't keep cache forever, refresh every N mins
if delta.seconds > App.get_config()['GHC_METADATA_CACHE_SECS']:
entry = Probe.METADATA_CACHE.pop(key)
del entry
metadata = None
if not metadata:
# Get actual metadata, Resource-type specifc
metadata = self.get_metadata(resource, version)
if metadata and App.get_config()['GHC_METADATA_CACHE_SECS'] > 0:
# Store entry with time, for expiry later
entry = {
"metadata": metadata,
"time": datetime.datetime.utcnow()
}
Probe.METADATA_CACHE[key] = entry
return metadata
def get_plugins(baseclass='GeoHealthCheck.plugin.Plugin', filters=None):
"""
Class method to get list of Plugins of particular baseclass (optional),
default is all Plugins. filters is a list of tuples to filter out
Plugins with class var values: (class var, value),
e.g. `filters=[('RESOURCE_TYPE', 'OGC:*'),
('RESOURCE_TYPE', 'OGC:WMS')]`.
"""
result = []
baseclass = Factory.create_class(baseclass)
def add_result(plugin_name, class_obj):
if not filters:
result.append(plugin_name)
else:
vars = Factory.get_class_vars(class_obj)
for filter in filters:
if vars[filter[0]] == filter[1]:
result.append(plugin_name)
break
plugins = App.get_plugins()
for plugin_name in plugins:
try:
# Assume module first
module = Factory.create_module(plugin_name)
for name in dir(module):
class_obj = getattr(module, name)
# Must be a class object inheriting from baseclass
# but not the baseclass itself
if inspect.isclass(class_obj) \
and baseclass in inspect.getmro(class_obj) and \
baseclass != class_obj:
add_result('%s.%s' % (plugin_name, name), class_obj)
except Exception:
# Try for full classname
try:
class_obj = Factory.create_class(plugin_name)
if baseclass in inspect.getmro(class_obj) \
and baseclass != class_obj:
add_result(plugin_name, class_obj)
except Exception:
print('cannot create obj class=%s' % plugin_name)
return result
def get_plugins(baseclass='GeoHealthCheck.plugin.Plugin', filters=None):
"""
Class method to get list of Plugins of particular baseclass (optional),
default is all Plugins. filters is a list of tuples to filter out
Plugins with class var values: (class var, value),
e.g. `filters=[('RESOURCE_TYPE', 'OGC:*'),
('RESOURCE_TYPE', 'OGC:WMS')]`.
"""
result = []
baseclass = Factory.create_class(baseclass)
def add_result(plugin_name, class_obj):
if not filters:
result.append(plugin_name)
else:
vars = Factory.get_class_vars(class_obj)
for filter in filters:
if vars[filter[0]] == filter[1]:
result.append(plugin_name)
break
plugins = App.get_plugins()
for plugin_name in plugins:
try:
# Assume module first
module = Factory.create_module(plugin_name)
for name in dir(module):
class_obj = getattr(module, name)
# Must be a class object inheriting from baseclass
# but not the baseclass itself
if inspect.isclass(class_obj) \
and baseclass in inspect.getmro(class_obj) and \
baseclass != class_obj:
add_result('%s.%s' % (plugin_name, name), class_obj)
except Exception:
# Try for full classname
try:
class_obj = Factory.create_class(plugin_name)
if baseclass in inspect.getmro(class_obj) \
and baseclass != class_obj:
add_result(plugin_name, class_obj)
except Exception:
LOGGER.warn('cannot create obj class=%s' % plugin_name)
return result
title = 'OGC STA'
else:
title = ows.identification.title
if title is None:
title = '%s %s %s' % (resource_type, gettext('for'), url)
title = title.decode('utf-8')
except Exception as err:
title = 'Untitled'
msg = 'Getting metadata failed: %s' % str(err)
LOGGER.exception(msg)
message = msg
success = False
end_time = datetime.datetime.utcnow()
delta = end_time - start_time
response_time = '%s.%s' % (delta.seconds, delta.microseconds)
return [title, success, response_time, message, start_time]
if __name__ == '__main__':
import sys
from init import App
if len(sys.argv) < 3:
print('Usage: %s <resource_type> <url>' % sys.argv[0])
sys.exit(1)
# TODO: need APP.config here, None for now
print(sniff_test_resource(App.get_config(), sys.argv[1], sys.argv[2]))
def perform_get_request(self, url):
""" Perform actual HTTP GET request to service"""
return requests.get(
url,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
headers=self.get_request_headers())
from flask_login import (LoginManager, login_user, logout_user,
current_user, login_required)
from flask_migrate import Migrate
from __init__ import __version__
from healthcheck import sniff_test_resource, run_test_resource
from init import App
from enums import RESOURCE_TYPES
from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient
from factory import Factory
from util import render_template2, send_email
import views
# Module globals for convenience
LOGGER = logging.getLogger(__name__)
APP = App.get_app()
CONFIG = App.get_config()
DB = App.get_db()
BABEL = App.get_babel()
MIGRATE = Migrate(APP, DB)
LOGIN_MANAGER = LoginManager()
LOGIN_MANAGER.init_app(APP)
LANGUAGES = (
('en', 'English'),
('fr', 'Français'),
('de', 'German'),
('de_DE', 'German (Germany)'),
('nl_NL', 'Nederlands (Nederland)'),
import logging
import os
import random
import string
from datetime import datetime, timedelta
from models import Resource, ResourceLock, flush_runs
from healthcheck import run_resource
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.jobstores.base import JobLookupError
from apscheduler.events import \
EVENT_SCHEDULER_STARTED, EVENT_SCHEDULER_SHUTDOWN, \
EVENT_JOB_MISSED, EVENT_JOB_ERROR
from init import App
LOGGER = logging.getLogger(__name__)
DB = App.get_db()
# Create scheduler
scheduler = BackgroundScheduler()
# commit or rollback shorthand
def db_commit():
err = None
try:
DB.session.commit()
except Exception as err:
DB.session.rollback()
# finally:
# DB.session.close()
return err
from owslib.tms import TileMapService
from owslib.wfs import WebFeatureService
from owslib.wcs import WebCoverageService
from owslib.wps import WebProcessingService
from owslib.csw import CatalogueServiceWeb
from owslib.sos import SensorObservationService
from init import App
from enums import RESOURCE_TYPES
from models import Resource, Run
from probe import Probe
from result import ResourceResult
from notifications import notify
LOGGER = logging.getLogger(__name__)
APP = App.get_app()
DB = App.get_db()
# commit or rollback shorthand
def db_commit():
err = None
try:
DB.session.commit()
except Exception as err:
LOGGER.warning('Cannot commit to database {}'.format(err))
DB.session.rollback()
# finally:
# DB.session.close()
return err
import json
import logging
from plugin import Plugin
from factory import Factory
from util import encode, decode
from init import App
APP = App.get_app()
LOGGER = logging.getLogger(__name__)
class ResourceAuth(Plugin):
"""
Base class for specific Plugin implementations to perform
authentication on a Resource. Subclasses provide specific
auth methods like Basic Auth, Bearer Token etc.
"""
def __init__(self):
Plugin.__init__(self)
self.resource = None
self.auth_dict = None
# Lifecycle
def init(self, auth_dict=None):
"""
Initialize ResourceAuth with related Resource and auth dict.
:return:
"""
self.auth_dict = auth_dict
@staticmethod
try:
data = json.load(r)
except (TypeError, ValueError,), err:
msg = "Cannot decode response from GeoNode at {}: {}".format(base_url,
err)
raise ValueError(msg)
def update(val):
val['title'] = base_name.format(val['type'])
return val
return [update(d) for d in data['data']]
def geonode_make_tags(base_url):
url = urlparse(base_url)
tag_name = 'GeoNode: {}'.format(url.hostname)
return [tag_name]
if __name__ == '__main__':
import sys
logging.basicConfig(level=logging.INFO)
from init import App
if len(sys.argv) < 3:
print('Usage: %s <resource_type> <url>' % sys.argv[0])
sys.exit(1)
# TODO: need APP.config here, None for now
pprint(sniff_test_resource(App.get_config(), sys.argv[1], sys.argv[2]))
from owslib.tms import TileMapService
from owslib.wfs import WebFeatureService
from owslib.wcs import WebCoverageService
from owslib.wps import WebProcessingService
from owslib.csw import CatalogueServiceWeb
from owslib.sos import SensorObservationService
from init import App
from enums import RESOURCE_TYPES
from models import Resource, Run
from probe import Probe
from result import ResourceResult
from notifications import notify
LOGGER = logging.getLogger(__name__)
APP = App.get_app()
DB = App.get_db()
# commit or rollback shorthand
def db_commit():
err = None
try:
DB.session.commit()
except Exception as err:
DB.session.rollback()
# finally:
# DB.session.close()
return err
import logging
import os
import smtplib
import base64
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from urllib.parse import urlparse
from gettext import translation
from passlib.hash import pbkdf2_sha256
from factory import Factory
from init import App
from jinja2 import Environment, FileSystemLoader
APP = App.get_app()
CONFIG = App.get_config()
LOGGER = logging.getLogger(__name__)
def average(values):
"""calculates average from a list"""
try:
return float(sum(values) / len(values))
except ZeroDivisionError:
return 0
def format_checked_datetime(run, default='-'):
"""common formatting datetime fields"""
from flask_babel import gettext
from flask_login import (LoginManager, login_user, logout_user, current_user,
login_required)
from flask_migrate import Migrate
from __init__ import __version__
from init import App
from enums import RESOURCE_TYPES
from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient
from factory import Factory
from util import send_email, geocode
import views
# Module globals for convenience
LOGGER = logging.getLogger(__name__)
APP = App.get_app()
CONFIG = App.get_config()
DB = App.get_db()
BABEL = App.get_babel()
MIGRATE = Migrate(APP, DB)
LOGIN_MANAGER = LoginManager()
LOGIN_MANAGER.init_app(APP)
LANGUAGES = (('en', 'English'), ('fr', 'Français'), ('de', 'German'),
('de_DE', 'German (Germany)'), ('nl_NL',
'Nederlands (Nederland)'),
('es_BO', 'Español (Bolivia)'), ('hr_HR', 'Croatian (Croatia)'))
# Should GHC Runner be run within GHC webapp?
import logging
import os
import random
import string
from datetime import datetime, timedelta
from models import Resource, ResourceLock, flush_runs
from healthcheck import run_resource
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.jobstores.base import JobLookupError
from apscheduler.events import \
EVENT_SCHEDULER_STARTED, EVENT_SCHEDULER_SHUTDOWN, \
EVENT_JOB_MISSED, EVENT_JOB_ERROR
from init import App
LOGGER = logging.getLogger(__name__)
DB = App.get_db()
# Create scheduler
scheduler = BackgroundScheduler()
# commit or rollback shorthand
def db_commit():
err = None
try:
DB.session.commit()
except Exception as err:
DB.session.rollback()
# finally:
# DB.session.close()
return err