def convert_record_to_dict(record, client_config):
if record.name in client_config.get('log_namespace_blacklist', []):
return None
if not getattr(record, 'created'):
time_string = datetime.datetime.utcnow().isoformat()
else:
time_string = time.strftime(
'%Y-%m-%dT%H:%M:%S.', time.gmtime(record.created)) + (
'%0.3f' % record.msecs).replace('.', '').zfill(6)
try:
message = record.getMessage()
tags_list = []
log_dict = {
'log_level': record.levelname,
"namespace": record.name,
'server': client_config.get('server_name', 'unknown'),
'date': time_string,
'request_id': None
}
if PY3:
log_dict['message'] = '%s' % message
else:
msg = message.encode('utf8') if isinstance(message,
unicode) else message
log_dict['message'] = '%s' % msg
# TODO: Based on docs, that attribute exists if a formatter
# already formatted the traceback, not sure if it is always
# there.
if client_config.get('logging_attach_exc_text'):
exc_text = getattr(record, 'exc_text', '')
if exc_text:
log_dict['message'] += '\n%s' % exc_text
# populate tags from extra
for k, v in vars(record).iteritems():
if k not in EXCLUDED_LOG_VARS:
try:
tags_list.append(parse_tag(k, v))
if k == 'ae_primary_key':
log_dict['primary_key'] = unicode(v)
if k == 'ae_permanent':
try:
log_dict['permanent'] = asbool(v)
except Exception:
log_dict['permanent'] = True
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
if tags_list:
log_dict['tags'] = tags_list
return log_dict
except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e:
# handle some weird case where record.getMessage() fails
log.warning(e)
def convert_record_to_dict(record, client_config):
if record.name in client_config.get('log_namespace_blacklist', []):
return None
if not getattr(record, 'created'):
time_string = datetime.datetime.utcnow().isoformat()
else:
time_string = time.strftime(
'%Y-%m-%dT%H:%M:%S.',
time.gmtime(record.created)) + ('%0.3f' % record.msecs).replace('.','').zfill(6)
try:
message = record.getMessage()
tags_list = []
log_dict = {'log_level': record.levelname,
"namespace": record.name,
'server': client_config.get('server_name', 'unknown'),
'date': time_string,
'request_id': None}
if PY3:
log_dict['message'] = '%s' % message
else:
msg = message.encode('utf8') if isinstance(message,
unicode) else message
log_dict['message'] = '%s' % msg
# TODO: Based on docs, that attribute exists if a formatter
# already formatted the traceback, not sure if it is always
# there.
if client_config.get('logging_attach_exc_text'):
exc_text = getattr(record, 'exc_text', '')
if exc_text:
log_dict['message'] += '\n%s' % exc_text
# populate tags from extra
for k, v in six.iteritems(vars(record)):
if k not in EXCLUDED_LOG_VARS:
try:
tags_list.append(parse_tag(k, v))
if k == 'ae_primary_key':
log_dict['primary_key'] = six.text_type(v)
if k == 'ae_permanent':
try:
log_dict['permanent'] = asbool(v)
except Exception:
log_dict['permanent'] = True
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
if tags_list:
log_dict['tags'] = tags_list
return log_dict
except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e:
# handle some weird case where record.getMessage() fails
log.warning(e)
def convert_record_to_dict(record, client_config):
if record.channel in client_config.get('log_namespace_blacklist', []):
return None
if not getattr(record, 'time'):
time_string = datetime.datetime.utcnow().isoformat()
else:
time_string = record.time.isoformat()
try:
message = record.msg
tags_list = []
log_dict = {
'log_level': record.level_name,
"namespace": record.channel,
'server': client_config.get('server_name', 'unknown'),
'date': time_string,
'request_id': None
}
if PY3:
log_dict['message'] = '%s' % message
else:
msg = message.encode('utf8') if isinstance(message,
unicode) else message
log_dict['message'] = '%s' % msg
if client_config.get('logging_attach_exc_text'):
pass
# populate tags from extra
for k, v in record.extra.iteritems():
if k not in EXCLUDED_LOG_VARS:
try:
tags_list.append(parse_tag(k, v))
if k == 'ae_primary_key':
log_dict['primary_key'] = unicode(v)
if k == 'ae_permanent':
try:
log_dict['permanent'] = asbool(v)
except Exception:
log_dict['permanent'] = True
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
if tags_list:
log_dict['tags'] = tags_list
return log_dict
except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e:
# handle some weird case where record.getMessage() fails
log.warning(e)
def convert_record_to_dict(record, client_config):
if record.channel in client_config.get('log_namespace_blacklist', []):
return None
if not getattr(record, 'time'):
time_string = datetime.datetime.utcnow().isoformat()
else:
time_string = record.time.isoformat()
try:
message = record.msg
tags_list = []
log_dict = {'log_level': record.level_name,
"namespace": record.channel,
'server': client_config.get('server_name', 'unknown'),
'date': time_string,
'request_id': None}
if PY3:
log_dict['message'] = '%s' % message
else:
msg = message.encode('utf8') if isinstance(message,
unicode) else message
log_dict['message'] = '%s' % msg
if client_config.get('logging_attach_exc_text'):
pass
# populate tags from extra
for k, v in record.extra.iteritems():
if k not in EXCLUDED_LOG_VARS:
try:
tags_list.append(parse_tag(k, v))
if k == 'ae_primary_key':
log_dict['primary_key'] = unicode(v)
if k == 'ae_permanent':
try:
log_dict['permanent'] = asbool(v)
except Exception:
log_dict['permanent'] = True
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
if tags_list:
log_dict['tags'] = tags_list
return log_dict
except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e:
# handle some weird case where record.getMessage() fails
log.warning(e)
def __init__(self, config=None, register_timing=True):
"""
at minimum client expects following keys to be present::
appenlight.api_key = YOUR_API_KEY
"""
self.config = {}
# general options
self.config['enabled'] = asbool(config.get('appenlight', True))
self.config['server_name'] = config.get('appenlight.server_name') \
or socket.getfqdn()
if PY3:
default_client = 'python3'
else:
default_client = 'python'
self.config['client'] = config.get('appenlight.client', default_client)
self.config['api_key'] = config.get('appenlight.api_key')
if not self.config['api_key']:
self.config['enabled'] = False
logging.warning("Disabling appenlight client, no api key")
self.config['transport'] = config.get('appenlight.transport',
'appenlight_client.transports.requests:HTTPTransport')
self.config['transport_config'] = config.get('appenlight.transport_config', 'https://api.appenlight.com?threaded=1&timeout=5')
self.config['reraise_exceptions'] = asbool(
config.get('appenlight.reraise_exceptions', True))
self.config['slow_requests'] = asbool(
config.get('appenlight.slow_requests', True))
self.config['slow_request_time'] = float(
config.get('appenlight.slow_request_time', 1))
if self.config['slow_request_time'] < 0.01:
self.config['slow_request_time'] = 0.01
self.config['slow_request_time'] = datetime.timedelta(
seconds=self.config['slow_request_time'])
self.config['logging'] = asbool(config.get('appenlight.logging', True))
self.config['logging_on_error'] = asbool(
config.get('appenlight.logging_on_error', False))
self.config['report_404'] = asbool(config.get('appenlight.report_404',
False))
self.config['report_local_vars'] = asbool(
config.get('appenlight.report_local_vars', False))
self.config['report_errors'] = asbool(
config.get('appenlight.report_errors', True))
self.config['buffer_flush_interval'] = int(
config.get('appenlight.buffer_flush_interval', 5))
self.config['buffer_clear_on_send'] = asbool(
config.get('appenlight.buffer_clear_on_send', False))
self.config['force_send'] = asbool(config.get('appenlight.force_send',
False))
self.config['request_keys_blacklist'] = ['password', 'passwd', 'pwd',
'auth_tkt', 'secret', 'csrf',
'session', 'pass', 'config',
'settings', 'environ', 'xsrf',
'auth']
req_blacklist = aslist(config.get('appenlight.request_keys_blacklist',
config.get(
'appenlight.bad_request_keys')),
',')
self.config['request_keys_blacklist'].extend(
filter(lambda x: x, req_blacklist)
)
if config.get('appenlight.bad_request_keys'):
log.warning('appenlight.bad_request_keys is deprecated use '
'request_keys_blacklist') # pragma: nocover
self.config['environ_keys_whitelist'] = [
'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE',
'HTTP_REFERER']
environ_whitelist = aslist(
config.get('appenlight.environ_keys_whitelist'), ',')
self.config['environ_keys_whitelist'].extend(
filter(lambda x: x, environ_whitelist))
self.config['log_namespace_blacklist'] = ['appenlight_client.client', 'appenlight_client.transports.requests']
log_blacklist = aslist(
config.get('appenlight.log_namespace_blacklist'), ',')
self.config['log_namespace_blacklist'].extend(filter(
lambda x: x, log_blacklist))
self.filter_callable = config.get('appenlight.filter_callable')
if self.filter_callable:
try:
parts = self.filter_callable.split(':')
_tmp = __import__(parts[0], globals(), locals(),
[parts[1], ], 0)
self.filter_callable = getattr(_tmp, parts[1])
except ImportError as e:
self.filter_callable = self.data_filter
msg = 'Could not import filter callable, using default, %s' % e
log.error(msg)
else:
self.filter_callable = self.data_filter
if self.config['buffer_flush_interval'] < 1:
self.config['buffer_flush_interval'] = 1
self.config['buffer_flush_interval'] = datetime.timedelta(
seconds=self.config['buffer_flush_interval'])
# register logging
import appenlight_client.logger
if self.config['logging'] and self.config['enabled']:
self.log_handler = appenlight_client.logger.register_logging()
level = LEVELS.get(config.get('appenlight.logging.level',
'WARNING').lower(), logging.WARNING)
self.log_handler.setLevel(level)
# register slow call metrics
if self.config['slow_requests'] and self.config['enabled']:
self.config['timing'] = config.get('appenlight.timing', {})
for k, v in config.items():
if k.startswith('appenlight.timing'):
try:
self.config['timing'][k[18:]] = float(v)
except (TypeError, ValueError) as e:
self.config['timing'][k[18:]] = False
import appenlight_client.timing
appenlight_client.timing.register_timing(self.config)
self.hooks = ['hook_pylons']
self.hooks_blacklist = aslist(config.get('appenlight.hooks_blacklist'),
',')
# register hooks
if self.config['enabled']:
self.register_hooks()
self.config['endpoints'] = {
"reports": '/api/reports',
"logs": '/api/logs',
"metrics": '/api/metrics'
}
self.report_queue = []
self.report_queue_lock = threading.RLock()
self.log_queue = []
self.log_queue_lock = threading.RLock()
self.request_stats = {}
self.request_stats_lock = threading.RLock()
self.uuid = uuid.uuid4()
self.last_submit = datetime.datetime.utcnow() - datetime.timedelta(
seconds=50)
self.last_request_stats_submit = datetime.datetime.utcnow() - datetime.timedelta(
seconds=50)
try:
parts = self.config['transport'].split(':')
_tmp = __import__(parts[0], globals(), locals(),
[parts[1], ], 0)
selected_transport = getattr(_tmp, parts[1])
except ImportError as e:
from appenlight_client.transports.requests import HTTPTransport as selected_transport
msg = 'Could not import transport %s, using default, %s' % (self.config['transport'], e)
log.error(msg)
self.transport = selected_transport(self.config['transport_config'],
self.config)
def update_config(self, config):
self.config = {}
# general options
self.config['enabled'] = asbool(config.get('appenlight', True))
self.config['server_name'] = config.get('appenlight.server_name') \
or socket.getfqdn()
if PY3:
default_client = 'python3'
else:
default_client = 'python'
self.config['client'] = config.get('appenlight.client', default_client)
self.config['api_key'] = config.get('appenlight.api_key')
if not self.config['api_key']:
self.config['enabled'] = False
logging.warning("Disabling appenlight client, no api key")
self.config['transport'] = config.get(
'appenlight.transport') or \
'appenlight_client.transports.requests:HTTPTransport'
self.config['transport_config'] = config.get(
'appenlight.transport_config') or \
'https://api.appenlight.com?threaded=1&timeout=5'
self.config['reraise_exceptions'] = asbool(
config.get('appenlight.reraise_exceptions', True))
self.config['slow_requests'] = asbool(
config.get('appenlight.slow_requests', True))
self.config['slow_request_time'] = float(
config.get('appenlight.slow_request_time', 1))
if self.config['slow_request_time'] < 0.01:
self.config['slow_request_time'] = 0.01
self.config['slow_request_time'] = datetime.timedelta(
seconds=self.config['slow_request_time'])
self.config['logging'] = asbool(config.get('appenlight.logging', True))
self.config['logging_attach_exc_text'] = asbool(
config.get('appenlight.logging_attach_exc_text', True))
self.config['logging_level'] = config.get('appenlight.logging.level',
'WARNING').lower()
self.config['logging_on_error'] = asbool(
config.get('appenlight.logging_on_error', False))
self.config['logging_max_thread_logs'] = int(
config.get('appenlight.logging.max_thread_logs', 10000))
self.config['report_404'] = asbool(config.get('appenlight.report_404',
False))
self.config['report_local_vars'] = asbool(
config.get('appenlight.report_local_vars', True))
self.config['report_local_vars_skip_existing'] = asbool(
config.get('appenlight.report_local_vars_skip_existing', True))
self.config['report_errors'] = asbool(
config.get('appenlight.report_errors', True))
self.config['buffer_flush_interval'] = int(
config.get('appenlight.buffer_flush_interval', 5))
self.config['buffer_clear_on_send'] = asbool(
config.get('appenlight.buffer_clear_on_send', False))
self.config['force_send'] = asbool(config.get('appenlight.force_send',
False))
request_keys_blacklist = [
'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf',
'session', 'pass', 'config', 'settings', 'environ', 'xsrf', 'auth']
self.config['request_keys_blacklist'] = request_keys_blacklist
req_blacklist = aslist(
config.get('appenlight.request_keys_blacklist',
config.get('appenlight.bad_request_keys')), ',')
self.config['request_keys_blacklist'].extend(
filter(lambda x: x, req_blacklist)
)
self.config['cookie_keys_whitelist'] = []
cookie_whitelist = aslist(
config.get('appenlight.cookie_keys_whitelist',
config.get('appenlight.cookie_keys_whitelist')), ',')
self.config['cookie_keys_whitelist'].extend(
filter(lambda x: x, cookie_whitelist)
)
if config.get('appenlight.bad_request_keys'):
log.warning('appenlight.bad_request_keys is deprecated use '
'request_keys_blacklist') # pragma: nocover
self.config['environ_keys_whitelist'] = [
'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE',
'HTTP_REFERER']
environ_whitelist = aslist(
config.get('appenlight.environ_keys_whitelist'), ',')
self.config['environ_keys_whitelist'].extend(
filter(lambda x: x, environ_whitelist))
self.config['log_namespace_blacklist'] = [
'appenlight_client.client',
'appenlight_client.transports.requests']
log_blacklist = aslist(
config.get('appenlight.log_namespace_blacklist'), ',')
self.config['log_namespace_blacklist'].extend(filter(
lambda x: x, log_blacklist))
self.config['filter_callable'] = config.get(
'appenlight.filter_callable')
if self.config['buffer_flush_interval'] < 1:
self.config['buffer_flush_interval'] = 1
self.config['buffer_flush_interval'] = datetime.timedelta(
seconds=self.config['buffer_flush_interval'])
# register slow call metrics
if self.config['slow_requests'] and self.config['enabled']:
self.config['timing'] = config.get('appenlight.timing', {})
for k, v in config.items():
if k.startswith('appenlight.timing'):
try:
self.config['timing'][k[18:]] = float(v)
except (TypeError, ValueError) as e:
self.config['timing'][k[18:]] = False
self.hooks_blacklist = aslist(
config.get('appenlight.hooks_blacklist'), ',')
self.config['ignore_slow_paths'] = \
config.get('appenlight.ignore_slow_paths', [])
self.config['ignore_paths'] = \
config.get('appenlight.ignore_paths', [])
def py_log(self, environ, records=None, r_uuid=None, created_report=None):
if not records:
records = self.log_handlers_get_records()
self.log_handlers_clear_records()
if not environ.get('appenlight.force_logs') and \
(self.config['logging_on_error'] and created_report is None):
return False
count = 0
for record in records:
count += 1
if record.name in self.config['log_namespace_blacklist']:
continue
if not getattr(record, 'created'):
time_string = datetime.datetime.utcnow().isoformat()
else:
time_string = time.strftime(
'%Y-%m-%dT%H:%M:%S.', time.gmtime(record.created)) + (
'%0.3f' % record.msecs).replace('.', '').zfill(6)
try:
message = record.getMessage()
tags_list = []
log_dict = {
'log_level': record.levelname,
"namespace": record.name,
'server': self.config['server_name'],
'date': time_string,
'request_id': r_uuid
}
if PY3:
log_dict['message'] = '%s' % message
else:
msg = message.encode('utf8') if isinstance(
message, unicode) else message
log_dict['message'] = '%s' % msg
# TODO: Based on docs, that attribute exists if a formatter
# already formatted the traceback, not sure if it is always
# there.
if self.config['logging_attach_exc_text']:
exc_text = getattr(record, 'exc_text', '')
if exc_text:
log_dict['message'] += '\n%s' % exc_text
# populate tags from extra
for k, v in vars(record).iteritems():
if k not in EXCLUDED_LOG_VARS:
try:
tags_list.append(parse_tag(k, v))
if k == 'ae_primary_key':
log_dict['primary_key'] = unicode(v)
if k == 'ae_permanent':
try:
log_dict['permanent'] = asbool(v)
except Exception:
log_dict['permanent'] = True
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
if tags_list:
log_dict['tags'] = tags_list
self.transport.feed_log(log_dict)
except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e:
# handle some weird case where record.getMessage() fails
log.warning(e)
log.debug('add %s log entries to queue' % count)
return True
def update_config(self, config):
self.config = {}
# general options
self.config['enabled'] = asbool(config.get('appenlight', True))
self.config['server_name'] = config.get('appenlight.server_name') \
or socket.getfqdn()
if PY3:
default_client = 'python3'
else:
default_client = 'python'
self.config['client'] = config.get('appenlight.client', default_client)
self.config['api_key'] = config.get('appenlight.api_key')
if not self.config['api_key']:
self.config['enabled'] = False
logging.warning("Disabling appenlight client, no api key")
self.config['transport'] = config.get(
'appenlight.transport') or \
'appenlight_client.transports.requests:HTTPTransport'
self.config['transport_config'] = config.get(
'appenlight.transport_config') or \
'https://api.appenlight.com?threaded=1&timeout=5'
self.config['reraise_exceptions'] = asbool(
config.get('appenlight.reraise_exceptions', True))
self.config['slow_requests'] = asbool(
config.get('appenlight.slow_requests', True))
self.config['slow_request_time'] = float(
config.get('appenlight.slow_request_time', 1))
if self.config['slow_request_time'] < 0.01:
self.config['slow_request_time'] = 0.01
self.config['slow_request_time'] = datetime.timedelta(
seconds=self.config['slow_request_time'])
self.config['logging'] = asbool(config.get('appenlight.logging', True))
self.config['logging_attach_exc_text'] = asbool(
config.get('appenlight.logging_attach_exc_text', True))
self.config['logging_level'] = config.get('appenlight.logging.level',
'WARNING').lower()
self.config['logging_on_error'] = asbool(
config.get('appenlight.logging_on_error', False))
self.config['logging_max_thread_logs'] = int(
config.get('appenlight.logging.max_thread_logs', 10000))
self.config['report_404'] = asbool(config.get('appenlight.report_404',
False))
self.config['report_local_vars'] = asbool(
config.get('appenlight.report_local_vars', True))
self.config['report_local_vars_skip_existing'] = asbool(
config.get('appenlight.report_local_vars_skip_existing', True))
self.config['report_errors'] = asbool(
config.get('appenlight.report_errors', True))
self.config['buffer_flush_interval'] = int(
config.get('appenlight.buffer_flush_interval', 5))
self.config['buffer_clear_on_send'] = asbool(
config.get('appenlight.buffer_clear_on_send', False))
self.config['force_send'] = asbool(config.get('appenlight.force_send',
False))
request_keys_blacklist = [
'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf',
'session', 'pass', 'config', 'settings', 'environ', 'xsrf', 'auth']
self.config['request_keys_blacklist'] = request_keys_blacklist
req_blacklist = aslist(
config.get('appenlight.request_keys_blacklist',
config.get('appenlight.bad_request_keys')), ',')
self.config['request_keys_blacklist'].extend(
filter(lambda x: x, req_blacklist)
)
self.config['cookie_keys_whitelist'] = []
cookie_whitelist = aslist(
config.get('appenlight.cookie_keys_whitelist',
config.get('appenlight.cookie_keys_whitelist')), ',')
self.config['cookie_keys_whitelist'].extend(
filter(lambda x: x, cookie_whitelist)
)
if config.get('appenlight.bad_request_keys'):
log.warning('appenlight.bad_request_keys is deprecated use '
'request_keys_blacklist') # pragma: nocover
self.config['environ_keys_whitelist'] = [
'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE',
'HTTP_REFERER']
environ_whitelist = aslist(
config.get('appenlight.environ_keys_whitelist'), ',')
self.config['environ_keys_whitelist'].extend(
filter(lambda x: x, environ_whitelist))
self.config['log_namespace_blacklist'] = [
'appenlight_client.client',
'appenlight_client.transports.requests']
log_blacklist = aslist(
config.get('appenlight.log_namespace_blacklist'), ',')
self.config['log_namespace_blacklist'].extend(filter(
lambda x: x, log_blacklist))
self.config['filter_callable'] = config.get(
'appenlight.filter_callable')
if self.config['buffer_flush_interval'] < 1:
self.config['buffer_flush_interval'] = 1
self.config['buffer_flush_interval'] = datetime.timedelta(
seconds=self.config['buffer_flush_interval'])
# register slow call metrics
if self.config['slow_requests'] and self.config['enabled']:
self.config['timing'] = config.get('appenlight.timing', {})
for k, v in config.items():
if k.startswith('appenlight.timing'):
try:
self.config['timing'][k[18:]] = float(v)
except (TypeError, ValueError) as e:
self.config['timing'][k[18:]] = False
self.hooks_blacklist = aslist(
config.get('appenlight.hooks_blacklist'), ',')
self.config['ignore_slow_paths'] = \
config.get('appenlight.ignore_slow_paths', [])
self.config['ignore_paths'] = \
config.get('appenlight.ignore_paths', [])