def convert_record_to_dict(record, client_config):
if record.name in client_config.get('log_namespace_blacklist', []):
return None
if not getattr(record, 'created'):
time_string = datetime.datetime.utcnow().isoformat()
else:
time_string = time.strftime(
'%Y-%m-%dT%H:%M:%S.', time.gmtime(record.created)) + (
'%0.3f' % record.msecs).replace('.', '').zfill(6)
try:
message = record.getMessage()
tags_list = []
log_dict = {
'log_level': record.levelname,
"namespace": record.name,
'server': client_config.get('server_name', 'unknown'),
'date': time_string,
'request_id': None
}
if PY3:
log_dict['message'] = '%s' % message
else:
msg = message.encode('utf8') if isinstance(message,
unicode) else message
log_dict['message'] = '%s' % msg
# TODO: Based on docs, that attribute exists if a formatter
# already formatted the traceback, not sure if it is always
# there.
if client_config.get('logging_attach_exc_text'):
exc_text = getattr(record, 'exc_text', '')
if exc_text:
log_dict['message'] += '\n%s' % exc_text
# populate tags from extra
for k, v in vars(record).iteritems():
if k not in EXCLUDED_LOG_VARS:
try:
tags_list.append(parse_tag(k, v))
if k == 'ae_primary_key':
log_dict['primary_key'] = unicode(v)
if k == 'ae_permanent':
try:
log_dict['permanent'] = asbool(v)
except Exception:
log_dict['permanent'] = True
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
if tags_list:
log_dict['tags'] = tags_list
return log_dict
except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e:
# handle some weird case where record.getMessage() fails
log.warning(e)
def convert_record_to_dict(record, client_config):
if record.name in client_config.get('log_namespace_blacklist', []):
return None
if not getattr(record, 'created'):
time_string = datetime.datetime.utcnow().isoformat()
else:
time_string = time.strftime(
'%Y-%m-%dT%H:%M:%S.',
time.gmtime(record.created)) + ('%0.3f' % record.msecs).replace('.','').zfill(6)
try:
message = record.getMessage()
tags_list = []
log_dict = {'log_level': record.levelname,
"namespace": record.name,
'server': client_config.get('server_name', 'unknown'),
'date': time_string,
'request_id': None}
if PY3:
log_dict['message'] = '%s' % message
else:
msg = message.encode('utf8') if isinstance(message,
unicode) else message
log_dict['message'] = '%s' % msg
# TODO: Based on docs, that attribute exists if a formatter
# already formatted the traceback, not sure if it is always
# there.
if client_config.get('logging_attach_exc_text'):
exc_text = getattr(record, 'exc_text', '')
if exc_text:
log_dict['message'] += '\n%s' % exc_text
# populate tags from extra
for k, v in six.iteritems(vars(record)):
if k not in EXCLUDED_LOG_VARS:
try:
tags_list.append(parse_tag(k, v))
if k == 'ae_primary_key':
log_dict['primary_key'] = six.text_type(v)
if k == 'ae_permanent':
try:
log_dict['permanent'] = asbool(v)
except Exception:
log_dict['permanent'] = True
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
if tags_list:
log_dict['tags'] = tags_list
return log_dict
except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e:
# handle some weird case where record.getMessage() fails
log.warning(e)
def convert_record_to_dict(record, client_config):
if record.channel in client_config.get('log_namespace_blacklist', []):
return None
if not getattr(record, 'time'):
time_string = datetime.datetime.utcnow().isoformat()
else:
time_string = record.time.isoformat()
try:
message = record.msg
tags_list = []
log_dict = {
'log_level': record.level_name,
"namespace": record.channel,
'server': client_config.get('server_name', 'unknown'),
'date': time_string,
'request_id': None
}
if PY3:
log_dict['message'] = '%s' % message
else:
msg = message.encode('utf8') if isinstance(message,
unicode) else message
log_dict['message'] = '%s' % msg
if client_config.get('logging_attach_exc_text'):
pass
# populate tags from extra
for k, v in record.extra.iteritems():
if k not in EXCLUDED_LOG_VARS:
try:
tags_list.append(parse_tag(k, v))
if k == 'ae_primary_key':
log_dict['primary_key'] = unicode(v)
if k == 'ae_permanent':
try:
log_dict['permanent'] = asbool(v)
except Exception:
log_dict['permanent'] = True
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
if tags_list:
log_dict['tags'] = tags_list
return log_dict
except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e:
# handle some weird case where record.getMessage() fails
log.warning(e)
def convert_record_to_dict(record, client_config):
if record.channel in client_config.get('log_namespace_blacklist', []):
return None
if not getattr(record, 'time'):
time_string = datetime.datetime.utcnow().isoformat()
else:
time_string = record.time.isoformat()
try:
message = record.msg
tags_list = []
log_dict = {'log_level': record.level_name,
"namespace": record.channel,
'server': client_config.get('server_name', 'unknown'),
'date': time_string,
'request_id': None}
if PY3:
log_dict['message'] = '%s' % message
else:
msg = message.encode('utf8') if isinstance(message,
unicode) else message
log_dict['message'] = '%s' % msg
if client_config.get('logging_attach_exc_text'):
pass
# populate tags from extra
for k, v in record.extra.iteritems():
if k not in EXCLUDED_LOG_VARS:
try:
tags_list.append(parse_tag(k, v))
if k == 'ae_primary_key':
log_dict['primary_key'] = unicode(v)
if k == 'ae_permanent':
try:
log_dict['permanent'] = asbool(v)
except Exception:
log_dict['permanent'] = True
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
if tags_list:
log_dict['tags'] = tags_list
return log_dict
except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e:
# handle some weird case where record.getMessage() fails
log.warning(e)
def process_environ(self, environ, traceback=None, include_params=False,
http_status=200):
# form friendly to json encode
parsed_environ = {}
appenlight_info = {}
if ('PATH_INFO' in environ or 'SERVER_NAME' in environ or
'wsgi.url_scheme' in environ):
# dummy data for Request to work
if 'wsgi.url_scheme' not in environ:
environ['wsgi.url_scheme'] = 'http'
environ['HTTP_HOST'] = ""
try:
req = Request(environ)
except Exception:
req = None
else:
req = None
reserved_list = ('appenlight.client',
'appenlight.force_send',
'appenlight.log',
'appenlight.report',
'appenlight.force_logs',
'appenlight.tags',
'appenlight.extra',
'appenlight.post_vars')
for key, value in environ.items():
if key.startswith('appenlight.') and key not in reserved_list:
appenlight_info[key[11:]] = unicode(value)
elif key == 'appenlight.tags':
appenlight_info['tags'] = []
for k, v in value.iteritems():
try:
appenlight_info['tags'].append(parse_tag(k, v))
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
elif key == 'appenlight.extra':
appenlight_info['extra'] = []
for k, v in value.iteritems():
try:
appenlight_info['extra'].append(parse_tag(k, v))
except Exception as e:
log.info(
u'Couldn\'t convert attached extra value %s' % e)
else:
whitelisted = key.startswith('HTTP') or key in self.config[
'environ_keys_whitelist']
if http_status not in [404, '404'] and whitelisted:
try:
if isinstance(value, str):
if PY3:
parsed_environ[key] = value
else:
parsed_environ[key] = value.decode('utf8')
else:
parsed_environ[key] = unicode(value)
except Exception:
pass
# provide better details for 500's
try:
parsed_environ['HTTP_METHOD'] = req.method
except Exception:
pass
if include_params and req:
try:
parsed_environ['COOKIES'] = dict(
[(k, v,) for k, v in req.cookies.items()
if k in self.config['cookie_keys_whitelist']])
except Exception:
parsed_environ['COOKIES'] = {}
try:
parsed_environ['GET'] = dict(
[(k, req.GET.getall(k),) for k in req.GET])
except Exception:
parsed_environ['GET'] = {}
try:
# handle situation where something already did seek()
# on wsgi.input
parsed_environ['POST'] = {}
post_vars = req.environ.get('appenlight.post_vars', None)
if post_vars is None:
post_vars = req.POST.mixed()
# handle werkzeug and multiple post values
if hasattr(post_vars, 'to_dict'):
post_vars = post_vars.to_dict(flat=False).items()
# handle django request object
elif hasattr(post_vars, 'lists'):
post_vars = post_vars.lists()
# handle everything else that should be dict-like
else:
post_vars = post_vars.items()
# if django request object use "lists()" to get multiple values
for k, v in post_vars:
try:
if isinstance(v, basestring):
parsed_environ['POST'][k] = v
else:
try:
parsed_environ['POST'][k] = [
unicode(val) for val in v]
except Exception:
parsed_environ['POST'][k] = unicode(v)
except Exception as e:
pass
except Exception as e:
parsed_environ['POST'] = {}
# figure out real ip
if environ.get("HTTP_X_FORWARDED_FOR"):
remote_addr = environ.get("HTTP_X_FORWARDED_FOR").split(',')[
0].strip()
else:
remote_addr = (
environ.get("HTTP_X_REAL_IP") or environ.get('REMOTE_ADDR'))
parsed_environ['HTTP_USER_AGENT'] = environ.get("HTTP_USER_AGENT", '')
parsed_environ['REMOTE_ADDR'] = remote_addr
if 'message' in appenlight_info:
appenlight_info['extra'].append(
('message', appenlight_info['message']), )
try:
username = appenlight_info.get('username',
environ.get('REMOTE_USER', u''))
appenlight_info['username'] = u'%s' % username
except (UnicodeEncodeError, UnicodeDecodeError):
appenlight_info['username'] = "******"
try:
appenlight_info['URL'] = req.url
except (UnicodeEncodeError, UnicodeDecodeError):
appenlight_info['URL'] = '/invalid-encoded-url'
except Exception:
appenlight_info['URL'] = 'unknown'
return parsed_environ, appenlight_info
def py_log(self, environ, records=None, r_uuid=None, created_report=None):
if not records:
records = self.log_handlers_get_records()
self.log_handlers_clear_records()
if not environ.get('appenlight.force_logs') and \
(self.config['logging_on_error'] and created_report is None):
return False
count = 0
for record in records:
count += 1
if record.name in self.config['log_namespace_blacklist']:
continue
if not getattr(record, 'created'):
time_string = datetime.datetime.utcnow().isoformat()
else:
time_string = time.strftime(
'%Y-%m-%dT%H:%M:%S.', time.gmtime(record.created)) + (
'%0.3f' % record.msecs).replace('.', '').zfill(6)
try:
message = record.getMessage()
tags_list = []
log_dict = {
'log_level': record.levelname,
"namespace": record.name,
'server': self.config['server_name'],
'date': time_string,
'request_id': r_uuid
}
if PY3:
log_dict['message'] = '%s' % message
else:
msg = message.encode('utf8') if isinstance(
message, unicode) else message
log_dict['message'] = '%s' % msg
# TODO: Based on docs, that attribute exists if a formatter
# already formatted the traceback, not sure if it is always
# there.
if self.config['logging_attach_exc_text']:
exc_text = getattr(record, 'exc_text', '')
if exc_text:
log_dict['message'] += '\n%s' % exc_text
# populate tags from extra
for k, v in vars(record).iteritems():
if k not in EXCLUDED_LOG_VARS:
try:
tags_list.append(parse_tag(k, v))
if k == 'ae_primary_key':
log_dict['primary_key'] = unicode(v)
if k == 'ae_permanent':
try:
log_dict['permanent'] = asbool(v)
except Exception:
log_dict['permanent'] = True
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
if tags_list:
log_dict['tags'] = tags_list
self.transport.feed_log(log_dict)
except (TypeError, UnicodeDecodeError, UnicodeEncodeError) as e:
# handle some weird case where record.getMessage() fails
log.warning(e)
log.debug('add %s log entries to queue' % count)
return True
def process_environ(self, environ, traceback=None, include_params=False,
http_status=200):
# form friendly to json encode
parsed_environ = {}
appenlight_info = {}
if ('PATH_INFO' in environ or 'SERVER_NAME' in environ or
'wsgi.url_scheme' in environ):
# dummy data for Request to work
if 'wsgi.url_scheme' not in environ:
environ['wsgi.url_scheme'] = 'http'
environ['HTTP_HOST'] = ""
try:
req = Request(environ)
except Exception:
req = None
else:
req = None
reserved_list = ('appenlight.client',
'appenlight.force_send',
'appenlight.log',
'appenlight.report',
'appenlight.force_logs',
'appenlight.tags',
'appenlight.extra',
'appenlight.post_vars')
for key, value in environ.items():
if key.startswith('appenlight.') and key not in reserved_list:
appenlight_info[key[11:]] = six.text_type(value)
elif key == 'appenlight.tags':
appenlight_info['tags'] = []
for k, v in six.iteritems(value):
try:
appenlight_info['tags'].append(parse_tag(k, v))
except Exception as e:
log.info(u'Couldn\'t convert attached tag %s' % e)
elif key == 'appenlight.extra':
appenlight_info['extra'] = []
for k, v in six.iteritems(value):
try:
appenlight_info['extra'].append(parse_tag(k, v))
except Exception as e:
log.info(
u'Couldn\'t convert attached extra value %s' % e)
else:
whitelisted = key.startswith('HTTP') or key in self.config[
'environ_keys_whitelist']
if http_status not in [404, '404'] and whitelisted:
try:
if isinstance(value, str):
if PY3:
parsed_environ[key] = value
else:
parsed_environ[key] = value.decode('utf8')
else:
parsed_environ[key] = six.text_type(value)
except Exception:
pass
# provide better details for 500's
try:
parsed_environ['HTTP_METHOD'] = req.method
except Exception:
pass
if include_params and req:
try:
parsed_environ['COOKIES'] = dict(
[(k, v,) for k, v in req.cookies.items()
if k in self.config['cookie_keys_whitelist']])
except Exception:
parsed_environ['COOKIES'] = {}
try:
parsed_environ['GET'] = dict(
[(k, req.GET.getall(k),) for k in req.GET])
except Exception:
parsed_environ['GET'] = {}
try:
# handle situation where something already did seek()
# on wsgi.input
parsed_environ['POST'] = {}
post_vars = req.environ.get('appenlight.post_vars', None)
if post_vars is None:
post_vars = req.POST.mixed()
# handle werkzeug and multiple post values
if hasattr(post_vars, 'to_dict'):
post_vars = post_vars.to_dict(flat=False).items()
# handle django request object
elif hasattr(post_vars, 'lists'):
post_vars = post_vars.lists()
# handle everything else that should be dict-like
else:
post_vars = post_vars.items()
# if django request object use "lists()" to get multiple values
for k, v in post_vars:
try:
if isinstance(v, basestring):
parsed_environ['POST'][k] = v
else:
try:
parsed_environ['POST'][k] = [
six.text_type(val) for val in v]
except Exception:
parsed_environ['POST'][k] = six.text_type(v)
except Exception as e:
pass
except Exception as e:
parsed_environ['POST'] = {}
# figure out real ip
if environ.get("HTTP_X_FORWARDED_FOR"):
remote_addr = environ.get("HTTP_X_FORWARDED_FOR").split(',')[
0].strip()
else:
remote_addr = (
environ.get("HTTP_X_REAL_IP") or environ.get('REMOTE_ADDR'))
parsed_environ['HTTP_USER_AGENT'] = environ.get("HTTP_USER_AGENT", '')
parsed_environ['REMOTE_ADDR'] = remote_addr
if 'message' in appenlight_info:
appenlight_info['extra'].append(
('message', appenlight_info['message']), )
try:
username = appenlight_info.get('username',
environ.get('REMOTE_USER', u''))
appenlight_info['username'] = u'%s' % username
except (UnicodeEncodeError, UnicodeDecodeError):
appenlight_info['username'] = "******"
try:
appenlight_info['URL'] = req.url
except (UnicodeEncodeError, UnicodeDecodeError):
appenlight_info['URL'] = '/invalid-encoded-url'
except Exception:
appenlight_info['URL'] = 'unknown'
return parsed_environ, appenlight_info
