def update_config(self, config): self.config = {} # general options self.config['enabled'] = asbool(config.get('appenlight', True)) self.config['server_name'] = config.get('appenlight.server_name') \ or socket.getfqdn() if PY3: default_client = 'python3' else: default_client = 'python' self.config['client'] = config.get('appenlight.client', default_client) self.config['api_key'] = config.get('appenlight.api_key') if not self.config['api_key']: self.config['enabled'] = False logging.warning("Disabling appenlight client, no api key") self.config['transport'] = config.get( 'appenlight.transport') or \ 'appenlight_client.transports.requests:HTTPTransport' self.config['transport_config'] = config.get( 'appenlight.transport_config') or \ 'https://api.appenlight.com?threaded=1&timeout=5' self.config['reraise_exceptions'] = asbool( config.get('appenlight.reraise_exceptions', True)) self.config['slow_requests'] = asbool( config.get('appenlight.slow_requests', True)) self.config['slow_request_time'] = float( config.get('appenlight.slow_request_time', 1)) if self.config['slow_request_time'] < 0.01: self.config['slow_request_time'] = 0.01 self.config['slow_request_time'] = datetime.timedelta( seconds=self.config['slow_request_time']) self.config['logging'] = asbool(config.get('appenlight.logging', True)) self.config['logging_attach_exc_text'] = asbool( config.get('appenlight.logging_attach_exc_text', True)) self.config['logging_level'] = config.get('appenlight.logging.level', 'WARNING').lower() self.config['logging_on_error'] = asbool( config.get('appenlight.logging_on_error', False)) self.config['logging_max_thread_logs'] = int( config.get('appenlight.logging.max_thread_logs', 10000)) self.config['report_404'] = asbool(config.get('appenlight.report_404', False)) self.config['report_local_vars'] = asbool( config.get('appenlight.report_local_vars', True)) self.config['report_local_vars_skip_existing'] = asbool( config.get('appenlight.report_local_vars_skip_existing', True)) self.config['report_errors'] = asbool( config.get('appenlight.report_errors', True)) self.config['buffer_flush_interval'] = int( config.get('appenlight.buffer_flush_interval', 5)) self.config['buffer_clear_on_send'] = asbool( config.get('appenlight.buffer_clear_on_send', False)) self.config['force_send'] = asbool(config.get('appenlight.force_send', False)) request_keys_blacklist = [ 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf', 'session', 'pass', 'config', 'settings', 'environ', 'xsrf', 'auth'] self.config['request_keys_blacklist'] = request_keys_blacklist req_blacklist = aslist( config.get('appenlight.request_keys_blacklist', config.get('appenlight.bad_request_keys')), ',') self.config['request_keys_blacklist'].extend( filter(lambda x: x, req_blacklist) ) self.config['cookie_keys_whitelist'] = [] cookie_whitelist = aslist( config.get('appenlight.cookie_keys_whitelist', config.get('appenlight.cookie_keys_whitelist')), ',') self.config['cookie_keys_whitelist'].extend( filter(lambda x: x, cookie_whitelist) ) if config.get('appenlight.bad_request_keys'): log.warning('appenlight.bad_request_keys is deprecated use ' 'request_keys_blacklist') # pragma: nocover self.config['environ_keys_whitelist'] = [ 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE', 'HTTP_REFERER'] environ_whitelist = aslist( config.get('appenlight.environ_keys_whitelist'), ',') self.config['environ_keys_whitelist'].extend( filter(lambda x: x, environ_whitelist)) self.config['log_namespace_blacklist'] = [ 'appenlight_client.client', 'appenlight_client.transports.requests'] log_blacklist = aslist( config.get('appenlight.log_namespace_blacklist'), ',') self.config['log_namespace_blacklist'].extend(filter( lambda x: x, log_blacklist)) self.config['filter_callable'] = config.get( 'appenlight.filter_callable') if self.config['buffer_flush_interval'] < 1: self.config['buffer_flush_interval'] = 1 self.config['buffer_flush_interval'] = datetime.timedelta( seconds=self.config['buffer_flush_interval']) # register slow call metrics if self.config['slow_requests'] and self.config['enabled']: self.config['timing'] = config.get('appenlight.timing', {}) for k, v in config.items(): if k.startswith('appenlight.timing'): try: self.config['timing'][k[18:]] = float(v) except (TypeError, ValueError) as e: self.config['timing'][k[18:]] = False self.hooks_blacklist = aslist( config.get('appenlight.hooks_blacklist'), ',') self.config['ignore_slow_paths'] = \ config.get('appenlight.ignore_slow_paths', []) self.config['ignore_paths'] = \ config.get('appenlight.ignore_paths', [])
def __init__(self, config=None, register_timing=True): """ at minimum client expects following keys to be present:: appenlight.api_key = YOUR_API_KEY """ self.config = {} # general options self.config['enabled'] = asbool(config.get('appenlight', True)) self.config['server_name'] = config.get('appenlight.server_name') \ or socket.getfqdn() if PY3: default_client = 'python3' else: default_client = 'python' self.config['client'] = config.get('appenlight.client', default_client) self.config['api_key'] = config.get('appenlight.api_key') if not self.config['api_key']: self.config['enabled'] = False logging.warning("Disabling appenlight client, no api key") self.config['transport'] = config.get('appenlight.transport', 'appenlight_client.transports.requests:HTTPTransport') self.config['transport_config'] = config.get('appenlight.transport_config', 'https://api.appenlight.com?threaded=1&timeout=5') self.config['reraise_exceptions'] = asbool( config.get('appenlight.reraise_exceptions', True)) self.config['slow_requests'] = asbool( config.get('appenlight.slow_requests', True)) self.config['slow_request_time'] = float( config.get('appenlight.slow_request_time', 1)) if self.config['slow_request_time'] < 0.01: self.config['slow_request_time'] = 0.01 self.config['slow_request_time'] = datetime.timedelta( seconds=self.config['slow_request_time']) self.config['logging'] = asbool(config.get('appenlight.logging', True)) self.config['logging_on_error'] = asbool( config.get('appenlight.logging_on_error', False)) self.config['report_404'] = asbool(config.get('appenlight.report_404', False)) self.config['report_local_vars'] = asbool( config.get('appenlight.report_local_vars', False)) self.config['report_errors'] = asbool( config.get('appenlight.report_errors', True)) self.config['buffer_flush_interval'] = int( config.get('appenlight.buffer_flush_interval', 5)) self.config['buffer_clear_on_send'] = asbool( config.get('appenlight.buffer_clear_on_send', False)) self.config['force_send'] = asbool(config.get('appenlight.force_send', False)) self.config['request_keys_blacklist'] = ['password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf', 'session', 'pass', 'config', 'settings', 'environ', 'xsrf', 'auth'] req_blacklist = aslist(config.get('appenlight.request_keys_blacklist', config.get( 'appenlight.bad_request_keys')), ',') self.config['request_keys_blacklist'].extend( filter(lambda x: x, req_blacklist) ) if config.get('appenlight.bad_request_keys'): log.warning('appenlight.bad_request_keys is deprecated use ' 'request_keys_blacklist') # pragma: nocover self.config['environ_keys_whitelist'] = [ 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE', 'HTTP_REFERER'] environ_whitelist = aslist( config.get('appenlight.environ_keys_whitelist'), ',') self.config['environ_keys_whitelist'].extend( filter(lambda x: x, environ_whitelist)) self.config['log_namespace_blacklist'] = ['appenlight_client.client', 'appenlight_client.transports.requests'] log_blacklist = aslist( config.get('appenlight.log_namespace_blacklist'), ',') self.config['log_namespace_blacklist'].extend(filter( lambda x: x, log_blacklist)) self.filter_callable = config.get('appenlight.filter_callable') if self.filter_callable: try: parts = self.filter_callable.split(':') _tmp = __import__(parts[0], globals(), locals(), [parts[1], ], 0) self.filter_callable = getattr(_tmp, parts[1]) except ImportError as e: self.filter_callable = self.data_filter msg = 'Could not import filter callable, using default, %s' % e log.error(msg) else: self.filter_callable = self.data_filter if self.config['buffer_flush_interval'] < 1: self.config['buffer_flush_interval'] = 1 self.config['buffer_flush_interval'] = datetime.timedelta( seconds=self.config['buffer_flush_interval']) # register logging import appenlight_client.logger if self.config['logging'] and self.config['enabled']: self.log_handler = appenlight_client.logger.register_logging() level = LEVELS.get(config.get('appenlight.logging.level', 'WARNING').lower(), logging.WARNING) self.log_handler.setLevel(level) # register slow call metrics if self.config['slow_requests'] and self.config['enabled']: self.config['timing'] = config.get('appenlight.timing', {}) for k, v in config.items(): if k.startswith('appenlight.timing'): try: self.config['timing'][k[18:]] = float(v) except (TypeError, ValueError) as e: self.config['timing'][k[18:]] = False import appenlight_client.timing appenlight_client.timing.register_timing(self.config) self.hooks = ['hook_pylons'] self.hooks_blacklist = aslist(config.get('appenlight.hooks_blacklist'), ',') # register hooks if self.config['enabled']: self.register_hooks() self.config['endpoints'] = { "reports": '/api/reports', "logs": '/api/logs', "metrics": '/api/metrics' } self.report_queue = [] self.report_queue_lock = threading.RLock() self.log_queue = [] self.log_queue_lock = threading.RLock() self.request_stats = {} self.request_stats_lock = threading.RLock() self.uuid = uuid.uuid4() self.last_submit = datetime.datetime.utcnow() - datetime.timedelta( seconds=50) self.last_request_stats_submit = datetime.datetime.utcnow() - datetime.timedelta( seconds=50) try: parts = self.config['transport'].split(':') _tmp = __import__(parts[0], globals(), locals(), [parts[1], ], 0) selected_transport = getattr(_tmp, parts[1]) except ImportError as e: from appenlight_client.transports.requests import HTTPTransport as selected_transport msg = 'Could not import transport %s, using default, %s' % (self.config['transport'], e) log.error(msg) self.transport = selected_transport(self.config['transport_config'], self.config)