def get_metadata_cached(self, resource, version='any'):
"""
Get metadata, specific per Resource type, get from cache
if cached.
:param resource:
:param version:
:return: Metadata object
"""
key = '%s_%s_%s' % (resource.url, resource.resource_type, version)
metadata = None
if key in Probe.METADATA_CACHE:
entry = Probe.METADATA_CACHE[key]
delta = datetime.datetime.utcnow() - entry['time']
metadata = entry['metadata']
# Don't keep cache forever, refresh every N mins
if delta.seconds > App.get_config()['GHC_METADATA_CACHE_SECS']:
entry = Probe.METADATA_CACHE.pop(key)
del entry
metadata = None
if not metadata:
# Get actual metadata, Resource-type specifc
metadata = self.get_metadata(resource, version)
if metadata and App.get_config()['GHC_METADATA_CACHE_SECS'] > 0:
# Store entry with time, for expiry later
entry = {
"metadata": metadata,
"time": datetime.datetime.utcnow()
}
Probe.METADATA_CACHE[key] = entry
return metadata
def perform_get_request(self, url):
""" Perform actual HTTP GET request to service"""
return self._session.get(
url,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
verify=App.get_config()['GHC_VERIFY_SSL'],
headers=self.get_request_headers())
def perform_post_request(self, url_base, request_string):
""" Perform actual HTTP POST request to service"""
return self._session.post(
url_base,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
verify=App.get_config()['GHC_VERIFY_SSL'],
data=request_string,
headers=self.get_request_headers())
def perform_request(self):
""" Perform actual request to service"""
# Actualize request query string or POST body
# by substitution in template.
url_base = self._resource.url
request_string = None
if self.REQUEST_TEMPLATE:
request_string = self.REQUEST_TEMPLATE
if '?' in url_base and self.REQUEST_TEMPLATE[0] == '?':
self.REQUEST_TEMPLATE = '&' + self.REQUEST_TEMPLATE[1:]
if self._parameters:
request_parms = self._parameters
param_defs = self.get_param_defs()
# Expand string list array to comma separated string
for param in request_parms:
if param_defs[param]['type'] == 'stringlist':
request_parms[param] = ','.join(request_parms[param])
request_string = self.REQUEST_TEMPLATE.format(**request_parms)
self.log('Requesting: %s url=%s' % (self.REQUEST_METHOD, url_base))
try:
headers = self.get_request_headers()
if self.REQUEST_METHOD == 'GET':
# Default is plain URL, e.g. for WWW:LINK
url = url_base
if request_string:
# Query String: mainly OWS:* resources
url = "%s%s" % (url, request_string)
self.response = requests.get(
url,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
headers=headers)
elif self.REQUEST_METHOD == 'POST':
self.response = requests.post(
url_base,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
data=request_string,
headers=headers)
except requests.exceptions.RequestException as e:
msg = "Request Err: %s %s" % (e.__class__.__name__, str(e))
self.result.set(False, msg)
if self.response:
self.log('response: status=%d' % self.response.status_code)
if self.response.status_code / 100 in [4, 5]:
self.log('Error response: %s' % (str(self.response.text)))
def perform_request(self):
""" Perform actual request to service"""
# Actualize request query string or POST body
# by substitution in template.
url_base = self._resource.url
request_string = None
if self.REQUEST_TEMPLATE:
request_string = self.REQUEST_TEMPLATE
if '?' in url_base and self.REQUEST_TEMPLATE[0] == '?':
self.REQUEST_TEMPLATE = '&' + self.REQUEST_TEMPLATE[1:]
if self._parameters:
request_parms = self._parameters
param_defs = self.get_param_defs()
# Expand string list array to comma separated string
for param in request_parms:
if param_defs[param]['type'] == 'stringlist':
request_parms[param] = ','.join(request_parms[param])
request_string = self.REQUEST_TEMPLATE.format(**request_parms)
self.log('Requesting: %s url=%s' % (self.REQUEST_METHOD, url_base))
try:
headers = self.get_request_headers()
if self.REQUEST_METHOD == 'GET':
# Default is plain URL, e.g. for WWW:LINK
url = url_base
if request_string:
# Query String: mainly OWS:* resources
url = "%s%s" % (url, request_string)
self.response = requests.get(
url,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
headers=headers)
elif self.REQUEST_METHOD == 'POST':
self.response = requests.post(
url_base,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
data=request_string,
headers=headers)
except requests.exceptions.RequestException as e:
msg = "Request Err: %s %s" % (e.__class__.__name__, str(e))
self.result.set(False, msg)
if self.response:
self.log('response: status=%d' % self.response.status_code)
if self.response.status_code / 100 in [4, 5]:
self.log('Error response: %s' % (str(self.response.text)))
def setUp(self):
# Need this for Resource Auth
App.get_config()['SECRET_KEY'] = 'mysecrettestkey'
self.db = DB
# do once per test
load_data('%s/data/fixtures.json' % TEST_DIR)
def perform_get_request(self, url):
""" Perform actual HTTP GET request to service"""
LOGGER.debug("request: %s" % str(url))
LOGGER.debug("headers: %s" % str(self.get_request_headers()))
return requests.get(
url,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
headers=self.get_request_headers())
def add_auth_header(self, headers_dict):
headers_dict = super(ArcGISTokenAuth,
self).add_auth_header(headers_dict)
# add referer header
referer_header = App.get_config()['GHC_SITE_URL']
headers_dict.update({'referer': str(referer_header)})
return headers_dict
def perform_post_request(self, url_base, request_string):
""" Perform actual HTTP POST request to service"""
LOGGER.debug("request: %s" % str(url_base))
LOGGER.debug("headers: %s" % str(self.get_request_headers()))
LOGGER.debug("data: %s" % str(request_string))
return requests.post(
url_base,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
data=request_string,
headers=self.get_request_headers())
def get_metadata_cached(self, resource, version='any'):
"""
Get metadata, specific per Resource type, get from cache
if cached.
:param resource:
:param version:
:return: Metadata object
"""
key = '%s_%s_%s' % (resource.url, resource.resource_type,
version)
metadata = None
if key in Probe.METADATA_CACHE:
entry = Probe.METADATA_CACHE[key]
delta = datetime.datetime.utcnow() - entry['time']
metadata = entry['metadata']
# Don't keep cache forever, refresh every N mins
if delta.seconds > App.get_config()['GHC_METADATA_CACHE_SECS']:
entry = Probe.METADATA_CACHE.pop(key)
del entry
metadata = None
if not metadata:
# Get actual metadata, Resource-type specifc
metadata = self.get_metadata(resource, version)
if metadata and App.get_config()['GHC_METADATA_CACHE_SECS'] > 0:
# Store entry with time, for expiry later
entry = {
"metadata": metadata,
"time": datetime.datetime.utcnow()
}
Probe.METADATA_CACHE[key] = entry
return metadata
from flask_login import (LoginManager, login_user, logout_user, current_user,
login_required)
from flask_migrate import Migrate
from __init__ import __version__
from init import App
from enums import RESOURCE_TYPES
from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient
from factory import Factory
from util import send_email, geocode
import views
# Module globals for convenience
LOGGER = logging.getLogger(__name__)
APP = App.get_app()
CONFIG = App.get_config()
DB = App.get_db()
BABEL = App.get_babel()
MIGRATE = Migrate(APP, DB)
LOGIN_MANAGER = LoginManager()
LOGIN_MANAGER.init_app(APP)
LANGUAGES = (('en', 'English'), ('fr', 'Français'), ('de', 'German'),
('de_DE', 'German (Germany)'), ('nl_NL',
'Nederlands (Nederland)'),
('es_BO', 'Español (Bolivia)'), ('hr_HR', 'Croatian (Croatia)'))
# Should GHC Runner be run within GHC webapp?
if CONFIG['GHC_RUNNER_IN_WEBAPP'] is True:
title = 'OGC STA'
else:
title = ows.identification.title
if title is None:
title = '%s %s %s' % (resource_type, gettext('for'), url)
title = title.decode('utf-8')
except Exception as err:
title = 'Untitled'
msg = 'Getting metadata failed: %s' % str(err)
LOGGER.exception(msg)
message = msg
success = False
end_time = datetime.datetime.utcnow()
delta = end_time - start_time
response_time = '%s.%s' % (delta.seconds, delta.microseconds)
return [title, success, response_time, message, start_time]
if __name__ == '__main__':
import sys
from init import App
if len(sys.argv) < 3:
print('Usage: %s <resource_type> <url>' % sys.argv[0])
sys.exit(1)
# TODO: need APP.config here, None for now
print(sniff_test_resource(App.get_config(), sys.argv[1], sys.argv[2]))
current_user, login_required)
from flask_migrate import Migrate
from __init__ import __version__
from healthcheck import sniff_test_resource, run_test_resource
from init import App
from enums import RESOURCE_TYPES
from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient
from factory import Factory
from util import render_template2, send_email
import views
# Module globals for convenience
LOGGER = logging.getLogger(__name__)
APP = App.get_app()
CONFIG = App.get_config()
DB = App.get_db()
BABEL = App.get_babel()
MIGRATE = Migrate(APP, DB)
LOGIN_MANAGER = LoginManager()
LOGIN_MANAGER.init_app(APP)
LANGUAGES = (
('en', 'English'),
('fr', 'Français'),
('de', 'German'),
('de_DE', 'German (Germany)'),
('nl_NL', 'Nederlands (Nederland)'),
('es_BO', 'Español (Bolivia)'),
try:
data = json.load(r)
except (TypeError, ValueError,), err:
msg = "Cannot decode response from GeoNode at {}: {}".format(base_url,
err)
raise ValueError(msg)
def update(val):
val['title'] = base_name.format(val['type'])
return val
return [update(d) for d in data['data']]
def geonode_make_tags(base_url):
url = urlparse(base_url)
tag_name = 'GeoNode: {}'.format(url.hostname)
return [tag_name]
if __name__ == '__main__':
import sys
logging.basicConfig(level=logging.INFO)
from init import App
if len(sys.argv) < 3:
print('Usage: %s <resource_type> <url>' % sys.argv[0])
sys.exit(1)
# TODO: need APP.config here, None for now
pprint(sniff_test_resource(App.get_config(), sys.argv[1], sys.argv[2]))
def perform_get_request(self, url):
""" Perform actual HTTP GET request to service"""
return requests.get(
url,
timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'],
headers=self.get_request_headers())