def get_metadata_cached(self, resource, version='any'): """ Get metadata, specific per Resource type, get from cache if cached. :param resource: :param version: :return: Metadata object """ key = '%s_%s_%s' % (resource.url, resource.resource_type, version) metadata = None if key in Probe.METADATA_CACHE: entry = Probe.METADATA_CACHE[key] delta = datetime.datetime.utcnow() - entry['time'] metadata = entry['metadata'] # Don't keep cache forever, refresh every N mins if delta.seconds > App.get_config()['GHC_METADATA_CACHE_SECS']: entry = Probe.METADATA_CACHE.pop(key) del entry metadata = None if not metadata: # Get actual metadata, Resource-type specifc metadata = self.get_metadata(resource, version) if metadata and App.get_config()['GHC_METADATA_CACHE_SECS'] > 0: # Store entry with time, for expiry later entry = { "metadata": metadata, "time": datetime.datetime.utcnow() } Probe.METADATA_CACHE[key] = entry return metadata
def perform_get_request(self, url): """ Perform actual HTTP GET request to service""" return self._session.get( url, timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'], verify=App.get_config()['GHC_VERIFY_SSL'], headers=self.get_request_headers())
def perform_post_request(self, url_base, request_string): """ Perform actual HTTP POST request to service""" return self._session.post( url_base, timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'], verify=App.get_config()['GHC_VERIFY_SSL'], data=request_string, headers=self.get_request_headers())
def perform_request(self): """ Perform actual request to service""" # Actualize request query string or POST body # by substitution in template. url_base = self._resource.url request_string = None if self.REQUEST_TEMPLATE: request_string = self.REQUEST_TEMPLATE if '?' in url_base and self.REQUEST_TEMPLATE[0] == '?': self.REQUEST_TEMPLATE = '&' + self.REQUEST_TEMPLATE[1:] if self._parameters: request_parms = self._parameters param_defs = self.get_param_defs() # Expand string list array to comma separated string for param in request_parms: if param_defs[param]['type'] == 'stringlist': request_parms[param] = ','.join(request_parms[param]) request_string = self.REQUEST_TEMPLATE.format(**request_parms) self.log('Requesting: %s url=%s' % (self.REQUEST_METHOD, url_base)) try: headers = self.get_request_headers() if self.REQUEST_METHOD == 'GET': # Default is plain URL, e.g. for WWW:LINK url = url_base if request_string: # Query String: mainly OWS:* resources url = "%s%s" % (url, request_string) self.response = requests.get( url, timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'], headers=headers) elif self.REQUEST_METHOD == 'POST': self.response = requests.post( url_base, timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'], data=request_string, headers=headers) except requests.exceptions.RequestException as e: msg = "Request Err: %s %s" % (e.__class__.__name__, str(e)) self.result.set(False, msg) if self.response: self.log('response: status=%d' % self.response.status_code) if self.response.status_code / 100 in [4, 5]: self.log('Error response: %s' % (str(self.response.text)))
def perform_request(self): """ Perform actual request to service""" # Actualize request query string or POST body # by substitution in template. url_base = self._resource.url request_string = None if self.REQUEST_TEMPLATE: request_string = self.REQUEST_TEMPLATE if '?' in url_base and self.REQUEST_TEMPLATE[0] == '?': self.REQUEST_TEMPLATE = '&' + self.REQUEST_TEMPLATE[1:] if self._parameters: request_parms = self._parameters param_defs = self.get_param_defs() # Expand string list array to comma separated string for param in request_parms: if param_defs[param]['type'] == 'stringlist': request_parms[param] = ','.join(request_parms[param]) request_string = self.REQUEST_TEMPLATE.format(**request_parms) self.log('Requesting: %s url=%s' % (self.REQUEST_METHOD, url_base)) try: headers = self.get_request_headers() if self.REQUEST_METHOD == 'GET': # Default is plain URL, e.g. for WWW:LINK url = url_base if request_string: # Query String: mainly OWS:* resources url = "%s%s" % (url, request_string) self.response = requests.get( url, timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'], headers=headers) elif self.REQUEST_METHOD == 'POST': self.response = requests.post( url_base, timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'], data=request_string, headers=headers) except requests.exceptions.RequestException as e: msg = "Request Err: %s %s" % (e.__class__.__name__, str(e)) self.result.set(False, msg) if self.response: self.log('response: status=%d' % self.response.status_code) if self.response.status_code / 100 in [4, 5]: self.log('Error response: %s' % (str(self.response.text)))
def setUp(self): # Need this for Resource Auth App.get_config()['SECRET_KEY'] = 'mysecrettestkey' self.db = DB # do once per test load_data('%s/data/fixtures.json' % TEST_DIR)
def perform_get_request(self, url): """ Perform actual HTTP GET request to service""" LOGGER.debug("request: %s" % str(url)) LOGGER.debug("headers: %s" % str(self.get_request_headers())) return requests.get( url, timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'], headers=self.get_request_headers())
def add_auth_header(self, headers_dict): headers_dict = super(ArcGISTokenAuth, self).add_auth_header(headers_dict) # add referer header referer_header = App.get_config()['GHC_SITE_URL'] headers_dict.update({'referer': str(referer_header)}) return headers_dict
def perform_post_request(self, url_base, request_string): """ Perform actual HTTP POST request to service""" LOGGER.debug("request: %s" % str(url_base)) LOGGER.debug("headers: %s" % str(self.get_request_headers())) LOGGER.debug("data: %s" % str(request_string)) return requests.post( url_base, timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'], data=request_string, headers=self.get_request_headers())
def get_metadata_cached(self, resource, version='any'): """ Get metadata, specific per Resource type, get from cache if cached. :param resource: :param version: :return: Metadata object """ key = '%s_%s_%s' % (resource.url, resource.resource_type, version) metadata = None if key in Probe.METADATA_CACHE: entry = Probe.METADATA_CACHE[key] delta = datetime.datetime.utcnow() - entry['time'] metadata = entry['metadata'] # Don't keep cache forever, refresh every N mins if delta.seconds > App.get_config()['GHC_METADATA_CACHE_SECS']: entry = Probe.METADATA_CACHE.pop(key) del entry metadata = None if not metadata: # Get actual metadata, Resource-type specifc metadata = self.get_metadata(resource, version) if metadata and App.get_config()['GHC_METADATA_CACHE_SECS'] > 0: # Store entry with time, for expiry later entry = { "metadata": metadata, "time": datetime.datetime.utcnow() } Probe.METADATA_CACHE[key] = entry return metadata
from flask_login import (LoginManager, login_user, logout_user, current_user, login_required) from flask_migrate import Migrate from __init__ import __version__ from init import App from enums import RESOURCE_TYPES from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient from factory import Factory from util import send_email, geocode import views # Module globals for convenience LOGGER = logging.getLogger(__name__) APP = App.get_app() CONFIG = App.get_config() DB = App.get_db() BABEL = App.get_babel() MIGRATE = Migrate(APP, DB) LOGIN_MANAGER = LoginManager() LOGIN_MANAGER.init_app(APP) LANGUAGES = (('en', 'English'), ('fr', 'Français'), ('de', 'German'), ('de_DE', 'German (Germany)'), ('nl_NL', 'Nederlands (Nederland)'), ('es_BO', 'Español (Bolivia)'), ('hr_HR', 'Croatian (Croatia)')) # Should GHC Runner be run within GHC webapp? if CONFIG['GHC_RUNNER_IN_WEBAPP'] is True:
title = 'OGC STA' else: title = ows.identification.title if title is None: title = '%s %s %s' % (resource_type, gettext('for'), url) title = title.decode('utf-8') except Exception as err: title = 'Untitled' msg = 'Getting metadata failed: %s' % str(err) LOGGER.exception(msg) message = msg success = False end_time = datetime.datetime.utcnow() delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) return [title, success, response_time, message, start_time] if __name__ == '__main__': import sys from init import App if len(sys.argv) < 3: print('Usage: %s <resource_type> <url>' % sys.argv[0]) sys.exit(1) # TODO: need APP.config here, None for now print(sniff_test_resource(App.get_config(), sys.argv[1], sys.argv[2]))
current_user, login_required) from flask_migrate import Migrate from __init__ import __version__ from healthcheck import sniff_test_resource, run_test_resource from init import App from enums import RESOURCE_TYPES from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient from factory import Factory from util import render_template2, send_email import views # Module globals for convenience LOGGER = logging.getLogger(__name__) APP = App.get_app() CONFIG = App.get_config() DB = App.get_db() BABEL = App.get_babel() MIGRATE = Migrate(APP, DB) LOGIN_MANAGER = LoginManager() LOGIN_MANAGER.init_app(APP) LANGUAGES = ( ('en', 'English'), ('fr', 'Français'), ('de', 'German'), ('de_DE', 'German (Germany)'), ('nl_NL', 'Nederlands (Nederland)'), ('es_BO', 'Español (Bolivia)'),
try: data = json.load(r) except (TypeError, ValueError,), err: msg = "Cannot decode response from GeoNode at {}: {}".format(base_url, err) raise ValueError(msg) def update(val): val['title'] = base_name.format(val['type']) return val return [update(d) for d in data['data']] def geonode_make_tags(base_url): url = urlparse(base_url) tag_name = 'GeoNode: {}'.format(url.hostname) return [tag_name] if __name__ == '__main__': import sys logging.basicConfig(level=logging.INFO) from init import App if len(sys.argv) < 3: print('Usage: %s <resource_type> <url>' % sys.argv[0]) sys.exit(1) # TODO: need APP.config here, None for now pprint(sniff_test_resource(App.get_config(), sys.argv[1], sys.argv[2]))
def perform_get_request(self, url): """ Perform actual HTTP GET request to service""" return requests.get( url, timeout=App.get_config()['GHC_PROBE_HTTP_TIMEOUT_SECS'], headers=self.get_request_headers())