def __init__(self, safe_repr=True, keys=None, **sizes):
super(ShortenerTransform, self).__init__()
self.safe_repr = safe_repr
self.keys = keys
self._repr = reprlib.Repr()
for name, size in iteritems(sizes):
setattr(self._repr, name, size)
def __init__(self, safe_repr=True, keys=None, **sizes):
super(ShortenerTransform, self).__init__()
self.safe_repr = safe_repr
self.keys = keys
self._repr = reprlib.Repr()
for name, size in iteritems(sizes):
setattr(self._repr, name, size)
def traverse(obj,
key=(),
string_handler=_default_handlers[STRING],
tuple_handler=_default_handlers[TUPLE],
namedtuple_handler=_default_handlers[NAMEDTUPLE],
list_handler=_default_handlers[LIST],
set_handler=_default_handlers[SET],
mapping_handler=_default_handlers[MAPPING],
default_handler=_default_handlers[DEFAULT],
circular_reference_handler=_default_handlers[CIRCULAR],
allowed_circular_reference_types=None,
memo=None,
**custom_handlers):
memo = memo or {}
obj_id = id(obj)
obj_type = get_type(obj)
ref_key = memo.get(obj_id)
if ref_key:
if not allowed_circular_reference_types or not isinstance(obj, allowed_circular_reference_types):
return circular_reference_handler(obj, key=key, ref_key=ref_key)
memo[obj_id] = key
kw = {
'string_handler': string_handler,
'tuple_handler': tuple_handler,
'namedtuple_handler': namedtuple_handler,
'list_handler': list_handler,
'set_handler': set_handler,
'mapping_handler': mapping_handler,
'default_handler': default_handler,
'circular_reference_handler': circular_reference_handler,
'allowed_circular_reference_types': allowed_circular_reference_types,
'memo': memo
}
kw.update(custom_handlers)
if obj_type is STRING:
return string_handler(obj, key=key)
elif obj_type is TUPLE:
return tuple_handler(tuple(traverse(elem, key=key + (i,), **kw) for i, elem in enumerate(obj)), key=key)
elif obj_type is NAMEDTUPLE:
return namedtuple_handler(obj._make(traverse(v, key=key + (k,), **kw) for k, v in iteritems(obj._asdict())), key=key)
elif obj_type is LIST:
return list_handler(list(traverse(elem, key=key + (i,), **kw) for i, elem in enumerate(obj)), key=key)
elif obj_type is SET:
return set_handler(set(traverse(elem, key=key + (i,), **kw) for i, elem in enumerate(obj)), key=key)
elif obj_type is MAPPING:
return mapping_handler(dict((k, traverse(v, key=key + (k,), **kw)) for k, v in iteritems(obj)), key=key)
elif obj_type is DEFAULT:
for handler_type, handler in iteritems(custom_handlers):
if isinstance(obj, handler_type):
return handler(obj, key=key)
return default_handler(obj, key=key)
def _build_payload(data):
"""
Returns the full payload as a string.
"""
for k, v in iteritems(data):
data[k] = _transform(v, key=(k, ))
payload = {'access_token': SETTINGS['access_token'], 'data': data}
return json.dumps(payload)
def _get_max_size(self, obj):
for name, _type in iteritems(_type_name_mapping):
# Special case for dicts since we are using collections.abc.Mapping
# to provide better type checking for dict-like objects
if name == 'mapping':
name = 'dict'
if _type and isinstance(obj, _type):
return getattr(self._repr, 'max%s' % name)
return self._repr.maxother
def _get_max_size(self, obj):
for name, _type in iteritems(_type_name_mapping):
# Special case for dicts since we are using collections.Mapping
# to provide better type checking for dict-like objects
if name == 'mapping':
name = 'dict'
if _type and isinstance(obj, _type):
return getattr(self._repr, 'max%s' % name)
return self._repr.maxother
def redact(self, url_string):
_redact = super(ScrubUrlTransform, self).redact
missing_colon_double_slash = False
if _starts_with_auth_re.match(url_string):
missing_colon_double_slash = True
url_string = '//%s' % url_string
try:
url_parts = urlsplit(url_string)
qs_params = parse_qs(url_parts.query, keep_blank_values=True)
except:
# This isn't a URL, return url_string which is a no-op
# for this transform
return url_string
netloc = url_parts.netloc
# If there's no netloc, give up
if not netloc:
return url_string
for qs_param, vals in iteritems(qs_params):
if qs_param.lower() in self.params_to_scrub:
vals2 = map(_redact, vals)
qs_params[qs_param] = vals2
scrubbed_qs = urlencode(qs_params, doseq=True)
if self.scrub_username and url_parts.username:
redacted_username = _redact(url_parts.username)
netloc = netloc.replace(url_parts.username, redacted_username)
if self.scrub_password and url_parts.password:
redacted_pw = _redact(url_parts.password)
netloc = netloc.replace(url_parts.password, redacted_pw)
scrubbed_url = (url_parts.scheme,
netloc,
url_parts.path,
scrubbed_qs,
url_parts.fragment)
scrubbed_url_string = urlunsplit(scrubbed_url)
if missing_colon_double_slash:
scrubbed_url_string = scrubbed_url_string.lstrip('://')
return scrubbed_url_string
def _build_payload(self, data):
"""
Returns the full payload as a string.
"""
for k, v in iteritems(data):
data[k] = self._transform(v, key=(k,))
payload = {
'access_token': self.settings['access_token'],
'data': data
}
return json.dumps(payload)
def _build_payload(data):
"""
Returns the full payload as a string.
"""
for k, v in iteritems(data):
data[k] = _transform(v, key=(k,))
payload = {
'access_token': SETTINGS['access_token'],
'data': data
}
return json.dumps(payload)
def redact(self, url_string):
_redact = super(ScrubUrlTransform, self).redact
missing_colon_double_slash = False
if _starts_with_auth_re.match(url_string):
missing_colon_double_slash = True
url_string = '//%s' % url_string
try:
url_parts = urlsplit(url_string)
qs_params = parse_qs(url_parts.query)
except:
# This isn't a URL, return url_string which is a no-op
# for this transform
return url_string
netloc = url_parts.netloc
# If there's no netloc, give up
if not netloc:
return url_string
for qs_param, vals in iteritems(qs_params):
if qs_param.lower() in self.params_to_scrub:
vals2 = map(_redact, vals)
qs_params[qs_param] = vals2
scrubbed_qs = urlencode(qs_params, doseq=True)
if self.scrub_username and url_parts.username:
redacted_username = _redact(url_parts.username)
netloc = netloc.replace(url_parts.username, redacted_username)
if self.scrub_password and url_parts.password:
redacted_pw = _redact(url_parts.password)
netloc = netloc.replace(url_parts.password, redacted_pw)
scrubbed_url = (url_parts.scheme,
netloc,
url_parts.path,
scrubbed_qs,
url_parts.fragment)
scrubbed_url_string = urlunsplit(scrubbed_url)
if missing_colon_double_slash:
scrubbed_url_string = scrubbed_url_string.lstrip('://')
return scrubbed_url_string
def transform_dict(self, o, key=None):
ret = {}
for k, v in iteritems(o):
if isinstance(k, string_types) or isinstance(k, binary_type):
if python_major_version() < 3:
if isinstance(k, unicode):
new_k = self.transform_unicode(k)
else:
new_k = self.transform_py2_str(k)
else:
if isinstance(k, bytes):
new_k = self.transform_py3_bytes(k)
else:
new_k = self.transform_unicode(k)
else:
new_k = text(k)
ret[new_k] = v
return super(SerializableTransform, self).transform_dict(ret, key=key)
def transform_dict(self, o, key=None):
ret = {}
for k, v in iteritems(o):
if isinstance(k, string_types) or isinstance(k, binary_type):
if python_major_version() < 3:
if isinstance(k, unicode):
new_k = self.transform_unicode(k)
else:
new_k = self.transform_py2_str(k)
else:
if isinstance(k, bytes):
new_k = self.transform_py3_bytes(k)
else:
new_k = self.transform_unicode(k)
else:
new_k = text(k)
ret[new_k] = v
return super(SerializableTransform, self).transform_dict(ret, key=key)
def _add_locals_data(data, exc_info):
if not SETTINGS['locals']['enabled']:
return
frames = data['body']['trace']['frames']
cur_tb = exc_info[2]
frame_num = 0
num_frames = len(frames)
while cur_tb:
cur_frame = frames[frame_num]
tb_frame = cur_tb.tb_frame
cur_tb = cur_tb.tb_next
if not isinstance(tb_frame, types.FrameType):
# this can happen if the traceback or frame is wrapped in some way,
# for example by `ExceptionInfo` in
# https://github.com/celery/billiard/blob/master/billiard/einfo.py
log.warning('Traceback frame not a types.FrameType. Ignoring.')
frame_num += 1
continue
# Create placeholders for argspec/varargspec/keywordspec/locals
argspec = None
varargspec = None
keywordspec = None
_locals = {}
try:
arginfo = inspect.getargvalues(tb_frame)
# Optionally fill in locals for this frame
if arginfo.locals and _check_add_locals(cur_frame, frame_num, num_frames):
# Get all of the named args
#
# args can be a nested list of args in the case where there
# are anonymous tuple args provided.
# e.g. in Python 2 you can:
# def func((x, (a, b), z)):
# return x + a + b + z
#
# func((1, (1, 2), 3))
argspec = _flatten_nested_lists(arginfo.args)
if arginfo.varargs is not None:
varargspec = arginfo.varargs
if SETTINGS['locals']['scrub_varargs']:
temp_varargs = list(arginfo.locals[varargspec])
for i, arg in enumerate(temp_varargs):
temp_varargs[i] = REDACT_REF
arginfo.locals[varargspec] = tuple(temp_varargs)
if arginfo.keywords is not None:
keywordspec = arginfo.keywords
_locals.update(arginfo.locals.items())
except Exception:
log.exception('Error while extracting arguments from frame. Ignoring.')
# Finally, serialize each arg/kwarg/local separately so that we only report
# CircularReferences for each variable, instead of for the entire payload
# as would be the case if we serialized that payload in one-shot.
if argspec:
cur_frame['argspec'] = argspec
if varargspec:
cur_frame['varargspec'] = varargspec
if keywordspec:
cur_frame['keywordspec'] = keywordspec
if _locals:
cur_frame['locals'] = dict((k, _serialize_frame_data(v)) for k, v in iteritems(_locals))
frame_num += 1
def traverse(obj,
key=(),
string_handler=_default_handlers[STRING],
tuple_handler=_default_handlers[TUPLE],
namedtuple_handler=_default_handlers[NAMEDTUPLE],
list_handler=_default_handlers[LIST],
set_handler=_default_handlers[SET],
mapping_handler=_default_handlers[MAPPING],
default_handler=_default_handlers[DEFAULT],
circular_reference_handler=_default_handlers[CIRCULAR],
allowed_circular_reference_types=None,
memo=None,
**custom_handlers):
memo = memo or {}
obj_id = id(obj)
obj_type = get_type(obj)
ref_key = memo.get(obj_id)
if ref_key:
if not allowed_circular_reference_types or not isinstance(
obj, allowed_circular_reference_types):
return circular_reference_handler(obj, key=key, ref_key=ref_key)
memo[obj_id] = key
kw = {
'string_handler': string_handler,
'tuple_handler': tuple_handler,
'namedtuple_handler': namedtuple_handler,
'list_handler': list_handler,
'set_handler': set_handler,
'mapping_handler': mapping_handler,
'default_handler': default_handler,
'circular_reference_handler': circular_reference_handler,
'allowed_circular_reference_types': allowed_circular_reference_types,
'memo': memo
}
kw.update(custom_handlers)
try:
if obj_type is STRING:
return string_handler(obj, key=key)
elif obj_type is TUPLE:
return tuple_handler(tuple(
traverse(elem, key=key + (i, ), **kw)
for i, elem in enumerate(obj)),
key=key)
elif obj_type is NAMEDTUPLE:
return namedtuple_handler(obj._make(
traverse(v, key=key + (k, ), **kw)
for k, v in iteritems(obj._asdict())),
key=key)
elif obj_type is LIST:
return list_handler(list(
traverse(elem, key=key + (i, ), **kw)
for i, elem in enumerate(obj)),
key=key)
elif obj_type is SET:
return set_handler(set(
traverse(elem, key=key + (i, ), **kw)
for i, elem in enumerate(obj)),
key=key)
elif obj_type is MAPPING:
return mapping_handler(dict((k, traverse(v, key=key + (k, ), **kw))
for k, v in iteritems(obj)),
key=key)
elif obj_type is DEFAULT:
for handler_type, handler in iteritems(custom_handlers):
if isinstance(obj, handler_type):
return handler(obj, key=key)
except:
# use the default handler for unknown object types
log.debug(
"Exception while traversing object using type-specific "
"handler. Switching to default handler.",
exc_info=True)
return default_handler(obj, key=key)
def _add_locals_data(data, exc_info):
if not SETTINGS['locals']['enabled']:
return
frames = data['body']['trace']['frames']
cur_tb = exc_info[2]
frame_num = 0
num_frames = len(frames)
while cur_tb:
cur_frame = frames[frame_num]
tb_frame = cur_tb.tb_frame
cur_tb = cur_tb.tb_next
if not isinstance(tb_frame, types.FrameType):
# this can happen if the traceback or frame is wrapped in some way,
# for example by `ExceptionInfo` in
# https://github.com/celery/billiard/blob/master/billiard/einfo.py
log.warning('Traceback frame not a types.FrameType. Ignoring.')
frame_num += 1
continue
# Create placeholders for args/kwargs/locals
args = []
kw = {}
_locals = {}
try:
arginfo = inspect.getargvalues(tb_frame)
local_vars = arginfo.locals
argspec = None
func = _get_func_from_frame(tb_frame)
if func:
if inspect.isfunction(func) or inspect.ismethod(func):
argspec = inspect.getargspec(func)
elif inspect.isclass(func):
init_func = getattr(func, '__init__', None)
if init_func:
argspec = inspect.getargspec(init_func)
# Get all of the named args
#
# args can be a nested list of args in the case where there
# are anonymous tuple args provided.
# e.g. in Python 2 you can:
# def func((x, (a, b), z)):
# return x + a + b + z
#
# func((1, (1, 2), 3))
named_args = _flatten_nested_lists(arginfo.args)
# Fill in all of the named args
for named_arg in named_args:
if named_arg in local_vars:
args.append(_transform(local_vars[named_arg], key=(named_arg,)))
# Add any varargs
if arginfo.varargs is not None:
args.extend(local_vars[arginfo.varargs])
# Fill in all of the kwargs
if arginfo.keywords is not None:
kw.update(local_vars[arginfo.keywords])
if argspec and argspec.defaults:
# Put any of the args that have defaults into kwargs
num_defaults = len(argspec.defaults)
if num_defaults:
# The last len(argspec.defaults) args in arginfo.args should be added
# to kwargs and removed from args
kw.update(dict(zip(arginfo.args[-num_defaults:], args[-num_defaults:])))
args = args[:-num_defaults]
# Optionally fill in locals for this frame
if local_vars and _check_add_locals(cur_frame, frame_num, num_frames):
_locals.update(local_vars.items())
args = args
kw = kw
_locals = _locals
except Exception as e:
log.exception('Error while extracting arguments from frame. Ignoring.')
# Finally, serialize each arg/kwarg/local separately so that we only report
# CircularReferences for each variable, instead of for the entire payload
# as would be the case if we serialized that payload in one-shot.
if args:
cur_frame['args'] = map(_serialize_frame_data, args)
if kw:
cur_frame['kwargs'] = dict((k, _serialize_frame_data(v)) for k, v in iteritems(kw))
if _locals:
cur_frame['locals'] = dict((k, _serialize_frame_data(v)) for k, v in iteritems(_locals))
frame_num += 1
def _add_locals_data(trace_data, exc_info):
if not SETTINGS['locals']['enabled']:
return
frames = trace_data['frames']
cur_tb = exc_info[2]
frame_num = 0
num_frames = len(frames)
while cur_tb:
cur_frame = frames[frame_num]
tb_frame = cur_tb.tb_frame
cur_tb = cur_tb.tb_next
if not isinstance(tb_frame, types.FrameType):
# this can happen if the traceback or frame is wrapped in some way,
# for example by `ExceptionInfo` in
# https://github.com/celery/billiard/blob/master/billiard/einfo.py
log.warning('Traceback frame not a types.FrameType. Ignoring.')
frame_num += 1
continue
# Create placeholders for argspec/varargspec/keywordspec/locals
argspec = None
varargspec = None
keywordspec = None
_locals = {}
try:
arginfo = inspect.getargvalues(tb_frame)
# Optionally fill in locals for this frame
if arginfo.locals and _check_add_locals(cur_frame, frame_num, num_frames):
# Get all of the named args
#
# args can be a nested list of args in the case where there
# are anonymous tuple args provided.
# e.g. in Python 2 you can:
# def func((x, (a, b), z)):
# return x + a + b + z
#
# func((1, (1, 2), 3))
argspec = _flatten_nested_lists(arginfo.args)
if arginfo.varargs is not None:
varargspec = arginfo.varargs
if SETTINGS['locals']['scrub_varargs']:
temp_varargs = list(arginfo.locals[varargspec])
for i, arg in enumerate(temp_varargs):
temp_varargs[i] = REDACT_REF
arginfo.locals[varargspec] = tuple(temp_varargs)
if arginfo.keywords is not None:
keywordspec = arginfo.keywords
_locals.update(arginfo.locals.items())
except Exception:
log.exception('Error while extracting arguments from frame. Ignoring.')
# Finally, serialize each arg/kwarg/local separately so that we only report
# CircularReferences for each variable, instead of for the entire payload
# as would be the case if we serialized that payload in one-shot.
if argspec:
cur_frame['argspec'] = argspec
if varargspec:
cur_frame['varargspec'] = varargspec
if keywordspec:
cur_frame['keywordspec'] = keywordspec
if _locals:
cur_frame['locals'] = dict((k, _serialize_frame_data(v)) for k, v in iteritems(_locals))
frame_num += 1
def _add_locals_data(data, exc_info):
if not SETTINGS['locals']['enabled']:
return
frames = data['body']['trace']['frames']
cur_tb = exc_info[2]
frame_num = 0
num_frames = len(frames)
while cur_tb:
cur_frame = frames[frame_num]
tb_frame = cur_tb.tb_frame
cur_tb = cur_tb.tb_next
if not isinstance(tb_frame, types.FrameType):
# this can happen if the traceback or frame is wrapped in some way,
# for example by `ExceptionInfo` in
# https://github.com/celery/billiard/blob/master/billiard/einfo.py
log.warning('Traceback frame not a types.FrameType. Ignoring.')
frame_num += 1
continue
# Create placeholders for args/kwargs/locals
args = []
kw = {}
_locals = {}
try:
arginfo = inspect.getargvalues(tb_frame)
local_vars = arginfo.locals
argspec = None
func = _get_func_from_frame(tb_frame)
if func:
if inspect.isfunction(func) or inspect.ismethod(func):
argspec = inspect.getargspec(func)
elif inspect.isclass(func):
init_func = getattr(func, '__init__', None)
if init_func:
argspec = inspect.getargspec(init_func)
# Get all of the named args
#
# args can be a nested list of args in the case where there
# are anonymous tuple args provided.
# e.g. in Python 2 you can:
# def func((x, (a, b), z)):
# return x + a + b + z
#
# func((1, (1, 2), 3))
named_args = _flatten_nested_lists(arginfo.args)
# Fill in all of the named args
for named_arg in named_args:
if named_arg in local_vars:
args.append(
_transform(local_vars[named_arg], key=(named_arg, )))
# Add any varargs
if arginfo.varargs is not None:
args.extend(local_vars[arginfo.varargs])
# Fill in all of the kwargs
if arginfo.keywords is not None:
kw.update(local_vars[arginfo.keywords])
if argspec and argspec.defaults:
# Put any of the args that have defaults into kwargs
num_defaults = len(argspec.defaults)
if num_defaults:
# The last len(argspec.defaults) args in arginfo.args should be added
# to kwargs and removed from args
kw.update(
dict(
zip(arginfo.args[-num_defaults:],
args[-num_defaults:])))
args = args[:-num_defaults]
# Optionally fill in locals for this frame
if local_vars and _check_add_locals(cur_frame, frame_num,
num_frames):
_locals.update(local_vars.items())
args = args
kw = kw
_locals = _locals
except Exception as e:
log.exception(
'Error while extracting arguments from frame. Ignoring.')
# Finally, serialize each arg/kwarg/local separately so that we only report
# CircularReferences for each variable, instead of for the entire payload
# as would be the case if we serialized that payload in one-shot.
if args:
cur_frame['args'] = map(_serialize_frame_data, args)
if kw:
cur_frame['kwargs'] = dict(
(k, _serialize_frame_data(v)) for k, v in iteritems(kw))
if _locals:
cur_frame['locals'] = dict(
(k, _serialize_frame_data(v)) for k, v in iteritems(_locals))
frame_num += 1
