def filter_traceback_and_stack(e):
out = None
# Scan the traceback and collect relevant frames.
for f, lineno in reversed(list(traceback.walk_tb(e.__traceback__))):
if include_frame(f):
out = types.TracebackType(out, f, f.f_lasti, lineno) # pytype: disable=wrong-arg-count
# Continue up the call stack.
#
# We would like to avoid stepping too far up, e.g. past the exec/eval point of
# a REPL such as IPython. To that end, we stop past the first contiguous bunch
# of module-level frames, if we reach any such frames at all. This is a
# heuristic that might stop in advance of the REPL boundary. For example, if
# the call stack includes module-level frames from the current module A, and
# the current module A was imported from within a function F elsewhere, then
# the stack trace we produce will be truncated at F's frame.
reached_module_level = False
for f, lineno in traceback.walk_stack(e.__traceback__.tb_frame):
if ignore_known_hidden_frame(f):
continue
if reached_module_level and f.f_code.co_name != '<module>':
break
if include_frame(f):
out = types.TracebackType(out, f, f.f_lasti, lineno) # pytype: disable=wrong-arg-count
if f.f_code.co_name == '<module>':
reached_module_level = True
return out
def generateTracebackFrom(exc, sourceFile):
"""Trim an exception's traceback to the last line of Scenic code."""
# find last stack frame in the source file
tbexc = traceback.TracebackException.from_exception(exc)
last = None
tbs = []
currentTb = exc.__traceback__
for depth, frame in enumerate(tbexc.stack):
assert currentTb is not None
tbs.append(currentTb)
currentTb = currentTb.tb_next
if frame.filename == sourceFile:
last = depth
assert last is not None
# create new trimmed traceback
lastTb = tbs[last]
lastLine = lastTb.tb_lineno
tbs = tbs[:last]
try:
currentTb = types.TracebackType(None, lastTb.tb_frame,
lastTb.tb_lasti, lastLine)
except TypeError:
# Python 3.6 does not allow creation of traceback objects, so we just
# return the original traceback
return exc.__traceback__, lastLine
for tb in reversed(tbs):
currentTb = types.TracebackType(currentTb, tb.tb_frame,
tb.tb_lasti, tb.tb_lineno)
return currentTb, lastLine
def _process_traceback_frames(tb):
new_tb = None
tb_list = list(traceback.walk_tb(tb))
for f, line_no in reversed(tb_list):
if include_frame(f.f_code.co_filename):
new_tb = types.TracebackType(new_tb, f, f.f_lasti, line_no)
if new_tb is None and tb_list:
f, line_no = tb_list[-1]
new_tb = types.TracebackType(new_tb, f, f.f_lasti, line_no)
return new_tb
def filter_traceback(tb):
out = None
# Scan the traceback and collect relevant frames.
frames = list(traceback.walk_tb(tb))
for f, lineno in reversed(frames):
if include_frame(f):
out = types.TracebackType(out, f, f.f_lasti, lineno) # pytype: disable=wrong-arg-count
if out is None and len(frames) > 0:
f, lineno = frames[-1]
out = types.TracebackType(out, f, f.f_lasti, lineno)
return out
def _process_traceback_frames(tb):
"""Iterate through traceback frames and return a new, filtered traceback."""
last_tb = None
tb_list = list(traceback.walk_tb(tb))
for f, line_no in reversed(tb_list):
if include_frame(f.f_code.co_filename):
last_tb = types.TracebackType(last_tb, f, f.f_lasti, line_no)
if last_tb is None and tb_list:
# If no frames were kept during filtering, create a new traceback
# from the outermost function.
f, line_no = tb_list[-1]
last_tb = types.TracebackType(last_tb, f, f.f_lasti, line_no)
return last_tb
def _frames_to_traceback(
frames: t.List[types.FrameType]) -> t.Optional[types.TracebackType]:
"Translate a list of frames (which can be obtained from the inspect module) to a traceback"
tb = None
for frame in frames:
tb = types.TracebackType(tb, frame, frame.f_lasti, frame.f_lineno)
return tb
def connect(signal: pyqtSignal, callback: Callable):
"""
By default calling ``signal.connect(callback)`` will dispose of context information.
Practically, this leads to single line tracebacks when ``signal.emit()`` is invoked.
This is very hard to debug.
This function wraps the ``connect()`` call to give you additional traceback information, if the callback does crash.
:param signal: the signal to ``connect()`` to.
:param callback: the callback to connect (will be called after ``signal.emit(...)``.
"""
# Step 1: At time of calling this function: get the stack frames.
# We reconstruct the stack as a ``traceback`` object.
source = None
for frame in list(inspect.stack())[1:]:
source = types.TracebackType(source, frame.frame, frame.index or 0,
frame.lineno)
# Step 2: construct a lightweight StackSummary object which does not contain
# actual frames or locals, to avoid memory leak
try:
summary = traceback.StackSummary.extract(traceback.walk_tb(source),
capture_locals=False)
finally:
del source
# Step 3: Wrap the ``callback`` and inject our creation stack if an error occurs.
# The BaseException instead of Exception is intentional: this makes sure interrupts of infinite loops show
# the source callback stack for debugging.
def trackback_wrapper(*args, **kwargs):
try:
callback(*args, **kwargs)
except BaseException as exc:
traceback_str = '\n' + ''.join(summary.format())
raise exc from CreationTraceback(traceback_str)
try:
setattr(callback, "tb_wrapper", trackback_wrapper)
except AttributeError:
# This is not a free function, but either an external library or a method bound to an instance.
if not hasattr(callback, "tb_wrapper") and hasattr(
callback, "__self__") and hasattr(callback, "__func__"):
# methods are finicky: you can't set attributes on them.
# Instead, we inject the handlers for each method in a dictionary on the instance.
bound_obj = callback.__self__
if not hasattr(bound_obj, "tb_mapping"):
setattr(bound_obj, "tb_mapping", {})
bound_obj.tb_mapping[
callback.__func__.__name__] = trackback_wrapper
else:
logging.warning(
"Unable to hook up connect() info to %s. Probably a 'builtin_function_or_method'.",
repr(callback))
# Step 3: Connect our wrapper to the signal.
signal.connect(trackback_wrapper)
def strip_modelkit_traceback_frames(exc: BaseException):
"""
Walk the traceback and remove frames that originate from within modelkit
Return an exception with the filtered traceback
"""
tb = None
for tb_frame, _ in reversed(list(traceback.walk_tb(exc.__traceback__))):
if not is_modelkit_internal_frame(tb_frame):
tb = types.TracebackType(tb, tb_frame, tb_frame.f_lasti,
tb_frame.f_lineno)
return exc.with_traceback(tb)
def generateTracebackFrom(exc, lineMap, sourceFile, full=False):
"""Adjust an exception's traceback to point to the correct line of original Scenic code."""
# find last stack frame in the source file
tbexc = traceback.TracebackException.from_exception(exc)
last = None
tbs = []
lms = []
currentTb = exc.__traceback__
for depth, frame in enumerate(tbexc.stack):
assert currentTb is not None
tbs.append(currentTb)
currentTb = currentTb.tb_next
if frame.filename == sourceFile:
last = depth
lms.append(lineMap)
else:
lms.append(None)
if full:
last = depth
assert last is not None
# create new trimmed traceback with corrected line numbers
lastTb = tbs[last]
lastLine = lastTb.tb_lineno
if lms[last] is not None:
lastLine = lms[last][lastLine]
tbs = tbs[:last]
lms = lms[:last]
try:
currentTb = types.TracebackType(None, lastTb.tb_frame, lastTb.tb_lasti,
lastLine)
except TypeError:
# Python 3.6 does not allow creation of traceback objects, so we just
# return the original traceback
return exc.__traceback__, lastLine
for tb, lm in zip(reversed(tbs), reversed(lms)):
line = lm[tb.tb_lineno] if lm else tb.tb_lineno
currentTb = types.TracebackType(currentTb, tb.tb_frame, tb.tb_lasti,
line)
return currentTb, lastLine
def _shallow_exception(exception, message=None, off_the_top=0):
exp = type(exception)
tb = None
while True:
try:
frame = sys._getframe(off_the_top)
off_the_top += 1
except ValueError as exc:
break
tb = types.TracebackType(tb, frame, frame.f_lasti, frame.f_lineno)
raise exp(message if message is not None else exception.args
).with_traceback(tb)
def make_traceback(depth=1):
'''
Create a phony traceback
Useful for testing calls that need a traceback object, such as
``sys.excepthook``
'''
tb = None
while True:
try:
frame = sys._getframe(depth)
depth += 1
except ValueError:
break
tb = types.TracebackType(tb, frame, frame.f_lasti, frame.f_lineno)
return tb
def check_arg(argument, allowed_types, condition=NO_DEFAULT, or_none=False):
"""Check if `argument` has the expected type and value.
**Note:** the exception's traceback is manipulated, so that the back frame
points to the ``check_arg()`` line, instead of the actual ``raise``.
Args:
argument (any): value of the argument to check
allowed_types (type or tuple of types)
condition (bool, optional): additional condition that must be true
or_none (bool, optional): defaults to false
Returns:
None
Raises:
TypeError: if `argument` is of an unexpected type
ValueError: if `argument` does not fulfill the optional condition
AssertionError:
if `check_arg` was called with a wrong syntax, i.e. `allowed_types`
does not contain types, e.g. if `1` was passed instead of `int`.
Examples::
def foo(name, amount, options=None):
check_arg(name, str)
check_arg(amount, (int, float), amount > 0)
check_arg(options, dict, or_none=True)
"""
try:
_check_arg(argument, allowed_types, condition, accept_none=or_none)
except (TypeError, ValueError) as e:
if sys.version_info < (3, 7):
raise
# Strip last frames, so the exception's stacktrace points to the call
_exc_type, _exc_value, traceback = sys.exc_info()
back_frame = traceback.tb_frame.f_back
back_tb = types.TracebackType(
tb_next=None,
tb_frame=back_frame,
tb_lasti=back_frame.f_lasti,
tb_lineno=back_frame.f_lineno,
)
raise e.with_traceback(back_tb)
def assert_always(condition, msg=None):
"""`assert` even in production code."""
try:
if not condition:
raise AssertionError(msg) if msg is not None else AssertionError
except AssertionError as e:
if sys.version_info < (3, 7):
raise
# Strip last frames, so the exception's stacktrace points to the call
# Credits: https://stackoverflow.com/a/58821552/19166
_exc_type, _exc_value, traceback = sys.exc_info()
back_frame = traceback.tb_frame.f_back
back_tb = types.TracebackType(
tb_next=None,
tb_frame=back_frame,
tb_lasti=back_frame.f_lasti,
tb_lineno=back_frame.f_lineno,
)
raise e.with_traceback(back_tb)
def raiseLessPipe(ex, includeMe=True):
tb = None
frame = inspect.currentframe(
) # do not use `frameInfos = inspect.stack(0)` as it is much much slower
# discard the frames for add_traceback
if not includeMe:
if frame.f_code.co_name == 'raiseLessPipe':
frame = frame.f_back
while True:
try:
# frame = sys._getframe(depth)
frame = frame.f_back
if not frame: break
except ValueError as e:
break
fullname = frame.f_globals['__name__'] + '.' + frame.f_code.co_name
ignore = ['IPython', 'ipykernel', 'pydevd', 'coppertop.pipe', '_pydev_imps._pydev_execfile', 'tornado', \
'runpy', 'asyncio', 'traitlets']
# print(fullname)
if not [fullname for i in ignore if fullname.startswith(i)]:
# print('-------- '+fullname)
tb = types.TracebackType(tb, frame, frame.f_lasti, frame.f_lineno)
else:
pass
# print(fullname)
if fullname == '__main__.<module>': break
hasPydev = False
hasIPython = False
while True:
# frame = sys._getframe(depth)
frame = frame.f_back
if not frame: break
fullname = frame.f_globals['__name__'] + '.' + frame.f_code.co_name
# print(fullname)
if fullname.startswith("pydevd"): hasPydev = True
if fullname.startswith("IPython"): hasIPython = True
if hasPydev or hasIPython:
raise ex.with_traceback(tb)
else:
raise ex from ex.with_traceback(tb)
def raise_error(self, message):
traceback = None
depth = 2
while True:
try:
frame = sys._getframe( # pylint: disable=protected-access
depth
)
except ValueError:
break
traceback = types.TracebackType(
traceback, frame, frame.f_lasti, frame.f_lineno
)
depth += 1
events = "\n" + "\n".join(str(e) for e in self.domain_events)
raise AssertionError(
f"{message}\n\nAll events: {str(events)}"
).with_traceback(traceback)
def test_constructor(self):
other_tb = get_tb()
frame = sys._getframe()
tb = types.TracebackType(other_tb, frame, 1, 2)
self.assertEqual(tb.tb_next, other_tb)
self.assertEqual(tb.tb_frame, frame)
self.assertEqual(tb.tb_lasti, 1)
self.assertEqual(tb.tb_lineno, 2)
tb = types.TracebackType(None, frame, 1, 2)
self.assertEqual(tb.tb_next, None)
with self.assertRaises(TypeError):
types.TracebackType("no", frame, 1, 2)
with self.assertRaises(TypeError):
types.TracebackType(other_tb, "no", 1, 2)
with self.assertRaises(TypeError):
types.TracebackType(other_tb, frame, "no", 2)
with self.assertRaises(TypeError):
types.TracebackType(other_tb, frame, 1, "nuh-uh")
def try_simplify_traceback(tb):
"""
Simplify the traceback. It removes the tracebacks in the current package, and only
shows the traceback that is related to the thirdparty and user-specified codes.
Returns
-------
TracebackType or None
Simplified traceback instance. It returns None if it fails to simplify.
Notes
-----
This keeps the tracebacks once it sees they are from a different file even
though the following tracebacks are from the current package.
Examples
--------
>>> import importlib
>>> import sys
>>> import traceback
>>> import tempfile
>>> with tempfile.TemporaryDirectory() as tmp_dir:
... with open("%s/dummy_module.py" % tmp_dir, "w") as f:
... _ = f.write(
... 'def raise_stop_iteration():\\n'
... ' raise StopIteration()\\n\\n'
... 'def simple_wrapper(f):\\n'
... ' def wrapper(*a, **k):\\n'
... ' return f(*a, **k)\\n'
... ' return wrapper\\n')
... f.flush()
... spec = importlib.util.spec_from_file_location(
... "dummy_module", "%s/dummy_module.py" % tmp_dir)
... dummy_module = importlib.util.module_from_spec(spec)
... spec.loader.exec_module(dummy_module)
>>> def skip_doctest_traceback(tb):
... import pyspark
... root = os.path.dirname(pyspark.__file__)
... pairs = zip(walk_tb(tb), traceback.extract_tb(tb))
... for cur_tb, cur_frame in pairs:
... if cur_frame.filename.startswith(root):
... return cur_tb
Regular exceptions should show the file name of the current package as below.
>>> exc_info = None
>>> try:
... fail_on_stopiteration(dummy_module.raise_stop_iteration)()
... except Exception as e:
... tb = sys.exc_info()[-1]
... e.__cause__ = None
... exc_info = "".join(
... traceback.format_exception(type(e), e, tb))
>>> print(exc_info) # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
Traceback (most recent call last):
File ...
...
File "/.../pyspark/util.py", line ...
...
RuntimeError: ...
>>> "pyspark/util.py" in exc_info
True
If the traceback is simplified with this method, it hides the current package file name:
>>> exc_info = None
>>> try:
... fail_on_stopiteration(dummy_module.raise_stop_iteration)()
... except Exception as e:
... tb = try_simplify_traceback(sys.exc_info()[-1])
... e.__cause__ = None
... exc_info = "".join(
... traceback.format_exception(
... type(e), e, try_simplify_traceback(skip_doctest_traceback(tb))))
>>> print(exc_info) # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
RuntimeError: ...
>>> "pyspark/util.py" in exc_info
False
In the case below, the traceback contains the current package in the middle.
In this case, it just hides the top occurrence only.
>>> exc_info = None
>>> try:
... fail_on_stopiteration(dummy_module.simple_wrapper(
... fail_on_stopiteration(dummy_module.raise_stop_iteration)))()
... except Exception as e:
... tb = sys.exc_info()[-1]
... e.__cause__ = None
... exc_info_a = "".join(
... traceback.format_exception(type(e), e, tb))
... exc_info_b = "".join(
... traceback.format_exception(
... type(e), e, try_simplify_traceback(skip_doctest_traceback(tb))))
>>> exc_info_a.count("pyspark/util.py")
2
>>> exc_info_b.count("pyspark/util.py")
1
"""
if "pypy" in platform.python_implementation().lower():
# Traceback modification is not supported with PyPy in PySpark.
return None
if sys.version_info[:2] < (3, 7):
# Traceback creation is not supported Python < 3.7.
# See https://bugs.python.org/issue30579.
return None
import pyspark
root = os.path.dirname(pyspark.__file__)
tb_next = None
new_tb = None
pairs = zip(walk_tb(tb), traceback.extract_tb(tb))
last_seen = []
for cur_tb, cur_frame in pairs:
if not cur_frame.filename.startswith(root):
# Filter the stacktrace from the PySpark source itself.
last_seen = [(cur_tb, cur_frame)]
break
for cur_tb, cur_frame in reversed(list(itertools.chain(last_seen, pairs))):
# Once we have seen the file names outside, don't skip.
new_tb = types.TracebackType(
tb_next=tb_next,
tb_frame=cur_tb.tb_frame,
tb_lasti=cur_tb.tb_frame.f_lasti,
tb_lineno=cur_tb.tb_frame.f_lineno
if cur_tb.tb_frame.f_lineno is not None else -1,
)
tb_next = new_tb
return new_tb
