def _compare_urls(self, url1, url2):
if _starts_with_auth_re.match(url1):
url1 = '//%s' % url1
if _starts_with_auth_re.match(url2):
url2 = '//%s' % url2
parsed_urls = map(urlparse, (url1, url2))
qs_params = map(lambda x: parse_qs(x.query), parsed_urls)
num_params = map(len, qs_params)
param_names = map(lambda x: set(x.keys()), qs_params)
self.assertEqual(*num_params)
self.assertDictEqual(*qs_params)
self.assertSetEqual(*param_names)
for facet in ('scheme', 'netloc', 'path', 'params', 'username', 'password', 'hostname', 'port'):
comp = map(lambda x: getattr(x, facet), parsed_urls)
self.assertEqual(*comp)
def _compare_urls(self, url1, url2):
if _starts_with_auth_re.match(url1):
url1 = '//%s' % url1
if _starts_with_auth_re.match(url2):
url2 = '//%s' % url2
parsed_urls = map(urlparse, (url1, url2))
qs_params = map(lambda x: parse_qs(x.query), parsed_urls)
num_params = map(len, qs_params)
param_names = map(lambda x: set(x.keys()), qs_params)
self.assertEqual(*num_params)
self.assertDictEqual(*qs_params)
self.assertSetEqual(*param_names)
for facet in ('scheme', 'netloc', 'path', 'params', 'username',
'password', 'hostname', 'port'):
comp = map(lambda x: getattr(x, facet), parsed_urls)
self.assertEqual(*comp)
def redact(self, url_string):
_redact = super(ScrubUrlTransform, self).redact
missing_colon_double_slash = False
if _starts_with_auth_re.match(url_string):
missing_colon_double_slash = True
url_string = '//%s' % url_string
try:
url_parts = urlsplit(url_string)
qs_params = parse_qs(url_parts.query, keep_blank_values=True)
except:
# This isn't a URL, return url_string which is a no-op
# for this transform
return url_string
netloc = url_parts.netloc
# If there's no netloc, give up
if not netloc:
return url_string
for qs_param, vals in iteritems(qs_params):
if qs_param.lower() in self.params_to_scrub:
vals2 = map(_redact, vals)
qs_params[qs_param] = vals2
scrubbed_qs = urlencode(qs_params, doseq=True)
if self.scrub_username and url_parts.username:
redacted_username = _redact(url_parts.username)
netloc = netloc.replace(url_parts.username, redacted_username)
if self.scrub_password and url_parts.password:
redacted_pw = _redact(url_parts.password)
netloc = netloc.replace(url_parts.password, redacted_pw)
scrubbed_url = (url_parts.scheme,
netloc,
url_parts.path,
scrubbed_qs,
url_parts.fragment)
scrubbed_url_string = urlunsplit(scrubbed_url)
if missing_colon_double_slash:
scrubbed_url_string = scrubbed_url_string.lstrip('://')
return scrubbed_url_string
def __init__(self,
suffixes=None,
scrub_username=False,
scrub_password=True,
params_to_scrub=None,
redact_char='-',
randomize_len=True):
super(ScrubUrlTransform, self).__init__(suffixes=suffixes,
redact_char=redact_char,
randomize_len=randomize_len)
self.scrub_username = scrub_username
self.scrub_password = scrub_password
self.params_to_scrub = set(map(lambda x: x.lower(), params_to_scrub))
def redact(self, url_string):
_redact = super(ScrubUrlTransform, self).redact
missing_colon_double_slash = False
if _starts_with_auth_re.match(url_string):
missing_colon_double_slash = True
url_string = '//%s' % url_string
try:
url_parts = urlsplit(url_string)
qs_params = parse_qs(url_parts.query)
except:
# This isn't a URL, return url_string which is a no-op
# for this transform
return url_string
netloc = url_parts.netloc
# If there's no netloc, give up
if not netloc:
return url_string
for qs_param, vals in iteritems(qs_params):
if qs_param.lower() in self.params_to_scrub:
vals2 = map(_redact, vals)
qs_params[qs_param] = vals2
scrubbed_qs = urlencode(qs_params, doseq=True)
if self.scrub_username and url_parts.username:
redacted_username = _redact(url_parts.username)
netloc = netloc.replace(url_parts.username, redacted_username)
if self.scrub_password and url_parts.password:
redacted_pw = _redact(url_parts.password)
netloc = netloc.replace(url_parts.password, redacted_pw)
scrubbed_url = (url_parts.scheme,
netloc,
url_parts.path,
scrubbed_qs,
url_parts.fragment)
scrubbed_url_string = urlunsplit(scrubbed_url)
if missing_colon_double_slash:
scrubbed_url_string = scrubbed_url_string.lstrip('://')
return scrubbed_url_string
def __init__(self,
suffixes=None,
scrub_username=False,
scrub_password=True,
params_to_scrub=None,
redact_char='-',
randomize_len=True):
super(ScrubUrlTransform, self).__init__(suffixes=suffixes,
redact_char=redact_char,
randomize_len=randomize_len)
self.scrub_username = scrub_username
self.scrub_password = scrub_password
self.params_to_scrub = set(map(lambda x: x.lower(), params_to_scrub))
def _add_locals_data(data, exc_info):
if not SETTINGS['locals']['enabled']:
return
frames = data['body']['trace']['frames']
cur_tb = exc_info[2]
frame_num = 0
num_frames = len(frames)
while cur_tb:
cur_frame = frames[frame_num]
tb_frame = cur_tb.tb_frame
cur_tb = cur_tb.tb_next
if not isinstance(tb_frame, types.FrameType):
# this can happen if the traceback or frame is wrapped in some way,
# for example by `ExceptionInfo` in
# https://github.com/celery/billiard/blob/master/billiard/einfo.py
log.warning('Traceback frame not a types.FrameType. Ignoring.')
frame_num += 1
continue
# Create placeholders for args/kwargs/locals
args = []
kw = {}
_locals = {}
try:
arginfo = inspect.getargvalues(tb_frame)
local_vars = arginfo.locals
argspec = None
func = _get_func_from_frame(tb_frame)
if func:
if inspect.isfunction(func) or inspect.ismethod(func):
argspec = inspect.getargspec(func)
elif inspect.isclass(func):
init_func = getattr(func, '__init__', None)
if init_func:
argspec = inspect.getargspec(init_func)
# Get all of the named args
#
# args can be a nested list of args in the case where there
# are anonymous tuple args provided.
# e.g. in Python 2 you can:
# def func((x, (a, b), z)):
# return x + a + b + z
#
# func((1, (1, 2), 3))
named_args = _flatten_nested_lists(arginfo.args)
# Fill in all of the named args
for named_arg in named_args:
if named_arg in local_vars:
args.append(_transform(local_vars[named_arg], key=(named_arg,)))
# Add any varargs
if arginfo.varargs is not None:
args.extend(local_vars[arginfo.varargs])
# Fill in all of the kwargs
if arginfo.keywords is not None:
kw.update(local_vars[arginfo.keywords])
if argspec and argspec.defaults:
# Put any of the args that have defaults into kwargs
num_defaults = len(argspec.defaults)
if num_defaults:
# The last len(argspec.defaults) args in arginfo.args should be added
# to kwargs and removed from args
kw.update(dict(zip(arginfo.args[-num_defaults:], args[-num_defaults:])))
args = args[:-num_defaults]
# Optionally fill in locals for this frame
if local_vars and _check_add_locals(cur_frame, frame_num, num_frames):
_locals.update(local_vars.items())
args = args
kw = kw
_locals = _locals
except Exception as e:
log.exception('Error while extracting arguments from frame. Ignoring.')
# Finally, serialize each arg/kwarg/local separately so that we only report
# CircularReferences for each variable, instead of for the entire payload
# as would be the case if we serialized that payload in one-shot.
if args:
cur_frame['args'] = map(_serialize_frame_data, args)
if kw:
cur_frame['kwargs'] = dict((k, _serialize_frame_data(v)) for k, v in iteritems(kw))
if _locals:
cur_frame['locals'] = dict((k, _serialize_frame_data(v)) for k, v in iteritems(_locals))
frame_num += 1
def _add_locals_data(data, exc_info):
if not SETTINGS['locals']['enabled']:
return
frames = data['body']['trace']['frames']
cur_tb = exc_info[2]
frame_num = 0
num_frames = len(frames)
while cur_tb:
cur_frame = frames[frame_num]
tb_frame = cur_tb.tb_frame
cur_tb = cur_tb.tb_next
if not isinstance(tb_frame, types.FrameType):
# this can happen if the traceback or frame is wrapped in some way,
# for example by `ExceptionInfo` in
# https://github.com/celery/billiard/blob/master/billiard/einfo.py
log.warning('Traceback frame not a types.FrameType. Ignoring.')
frame_num += 1
continue
# Create placeholders for args/kwargs/locals
args = []
kw = {}
_locals = {}
try:
arginfo = inspect.getargvalues(tb_frame)
local_vars = arginfo.locals
argspec = None
func = _get_func_from_frame(tb_frame)
if func:
if inspect.isfunction(func) or inspect.ismethod(func):
argspec = inspect.getargspec(func)
elif inspect.isclass(func):
init_func = getattr(func, '__init__', None)
if init_func:
argspec = inspect.getargspec(init_func)
# Get all of the named args
#
# args can be a nested list of args in the case where there
# are anonymous tuple args provided.
# e.g. in Python 2 you can:
# def func((x, (a, b), z)):
# return x + a + b + z
#
# func((1, (1, 2), 3))
named_args = _flatten_nested_lists(arginfo.args)
# Fill in all of the named args
for named_arg in named_args:
if named_arg in local_vars:
args.append(
_transform(local_vars[named_arg], key=(named_arg, )))
# Add any varargs
if arginfo.varargs is not None:
args.extend(local_vars[arginfo.varargs])
# Fill in all of the kwargs
if arginfo.keywords is not None:
kw.update(local_vars[arginfo.keywords])
if argspec and argspec.defaults:
# Put any of the args that have defaults into kwargs
num_defaults = len(argspec.defaults)
if num_defaults:
# The last len(argspec.defaults) args in arginfo.args should be added
# to kwargs and removed from args
kw.update(
dict(
zip(arginfo.args[-num_defaults:],
args[-num_defaults:])))
args = args[:-num_defaults]
# Optionally fill in locals for this frame
if local_vars and _check_add_locals(cur_frame, frame_num,
num_frames):
_locals.update(local_vars.items())
args = args
kw = kw
_locals = _locals
except Exception as e:
log.exception(
'Error while extracting arguments from frame. Ignoring.')
# Finally, serialize each arg/kwarg/local separately so that we only report
# CircularReferences for each variable, instead of for the entire payload
# as would be the case if we serialized that payload in one-shot.
if args:
cur_frame['args'] = map(_serialize_frame_data, args)
if kw:
cur_frame['kwargs'] = dict(
(k, _serialize_frame_data(v)) for k, v in iteritems(kw))
if _locals:
cur_frame['locals'] = dict(
(k, _serialize_frame_data(v)) for k, v in iteritems(_locals))
frame_num += 1