def test_walk_tb(self):
try:
1/0
except Exception:
_, _, tb = sys.exc_info()
s = list(traceback.walk_tb(tb))
self.assertEqual(len(s), 1)
def filtered_traceback_format(tb_exception, chain=True):
if chain:
if tb_exception.__cause__ is not None:
for line in filtered_traceback_format(tb_exception.__cause__,
chain=chain):
yield line
yield tb._cause_message
elif (tb_exception.__context__ is not None and
not tb_exception.__suppress_context__):
for line in filtered_traceback_format(tb_exception.__context__,
chain=chain):
yield line
yield tb._context_message
yield 'Traceback (most recent calls WITHOUT Sacred internals):\n'
current_tb = tb_exception.exc_traceback
while current_tb is not None:
if not _is_sacred_frame(current_tb.tb_frame):
stack = tb.StackSummary.extract(tb.walk_tb(current_tb),
limit=1,
lookup_lines=True,
capture_locals=False)
for line in stack.format():
yield line
current_tb = current_tb.tb_next
for line in tb_exception.format_exception_only():
yield line
def printNotImplementedError(cls, ex):
from traceback import walk_tb
Init.init()
frame, _ = [x for x in walk_tb(ex.__traceback__)][-1]
filename = frame.f_code.co_filename
funcName = frame.f_code.co_name
print("{RED}Not implemented:{NOCOLOR} {function} in file '{filename}': {message}".format(function=funcName, filename=filename, message=str(ex), **Init.Foreground))
Exit.exit(1)
def printNotImplementedError(cls, ex):
from traceback import walk_tb
Init.init()
frame, _ = [x for x in walk_tb(ex.__traceback__)][-1]
filename = frame.f_code.co_filename
funcName = frame.f_code.co_name
print("{RED}NOT IMPLEMENTED:{NOCOLOR} {function} in file '{filename}': {message!s}".format(function=funcName, filename=filename, message=ex, **Init.Foreground))
print(("{RED}" + ("-" * 80) + "{NOCOLOR}").format(**Init.Foreground))
print(("{RED}Please report this bug at GitHub: https://github.com/VLSI-EDA/PoC/issues{NOCOLOR}").format(**Init.Foreground))
print(("{RED}" + ("-" * 80) + "{NOCOLOR}").format(**Init.Foreground))
Exit.exit(1)
def test_limit(self):
def recurse(n):
if n:
recurse(n-1)
else:
1/0
try:
recurse(10)
except Exception:
exc_info = sys.exc_info()
exc = traceback.TracebackException(*exc_info, limit=5)
expected_stack = traceback.StackSummary.extract(
traceback.walk_tb(exc_info[2]), limit=5)
self.assertEqual(expected_stack, exc.stack)
def test_smoke(self):
try:
1/0
except Exception:
exc_info = sys.exc_info()
exc = traceback.TracebackException(*exc_info)
expected_stack = traceback.StackSummary.extract(
traceback.walk_tb(exc_info[2]))
self.assertEqual(None, exc.__cause__)
self.assertEqual(None, exc.__context__)
self.assertEqual(False, exc.__suppress_context__)
self.assertEqual(expected_stack, exc.stack)
self.assertEqual(exc_info[0], exc.exc_type)
self.assertEqual(str(exc_info[1]), str(exc))
def printException(cls, ex):
from traceback import print_tb, walk_tb
Init.init()
print("{RED}FATAL: An unknown or unhandled exception reached the topmost exception handler!{NOCOLOR}".format(message=ex.__str__(), **Init.Foreground))
print("{YELLOW} Exception type:{NOCOLOR} {type}".format(type=ex.__class__.__name__, **Init.Foreground))
print("{YELLOW} Exception message:{NOCOLOR} {message}".format(message=ex.__str__(), **Init.Foreground))
frame,sourceLine = [x for x in walk_tb(ex.__traceback__)][-1]
filename = frame.f_code.co_filename
funcName = frame.f_code.co_name
print("{YELLOW} Caused by:{NOCOLOR} {function} in file '{filename}' at line {line}".format(function=funcName, filename=filename, line=sourceLine, **Init.Foreground))
print("-" * 80)
print_tb(ex.__traceback__)
print("-" * 80)
Exit.exit(1)
def printException(cls, ex):
from traceback import print_tb, walk_tb
Init.init()
print("{RED}FATAL: An unknown or unhandled exception reached the topmost exception handler!{NOCOLOR}".format(**Init.Foreground))
print("{YELLOW} Exception type:{NOCOLOR} {type}".format(type=ex.__class__.__name__, **Init.Foreground))
print("{YELLOW} Exception message:{NOCOLOR} {message!s}".format(message=ex, **Init.Foreground))
if (ex.__cause__ is not None):
print("{YELLOW} Caused by type:{NOCOLOR} {type}".format(message=ex.__cause__.__class__.__name__, **Init.Foreground))
print("{YELLOW} Caused by message:{NOCOLOR} {message!s}".format(message=ex.__cause__, **Init.Foreground))
frame,sourceLine = [x for x in walk_tb(ex.__traceback__)][-1]
filename = frame.f_code.co_filename
funcName = frame.f_code.co_name
print("{YELLOW} Caused by:{NOCOLOR} {function} in file '{filename}' at line {line}".format(function=funcName, filename=filename, line=sourceLine, **Init.Foreground))
print(("{RED}" + ("-" * 80) + "{NOCOLOR}").format(**Init.Foreground))
print_tb(ex.__traceback__)
print(("{RED}" + ("-" * 80) + "{NOCOLOR}").format(**Init.Foreground))
print(("{RED}Please report this bug at GitHub: https://github.com/VLSI-EDA/PoC/issues{NOCOLOR}").format(**Init.Foreground))
print(("{RED}" + ("-" * 80) + "{NOCOLOR}").format(**Init.Foreground))
Exit.exit(1)
def test_cause(self):
try:
try:
1/0
finally:
exc_info_context = sys.exc_info()
exc_context = traceback.TracebackException(*exc_info_context)
cause = Exception("cause")
raise Exception("uh oh") from cause
except Exception:
exc_info = sys.exc_info()
exc = traceback.TracebackException(*exc_info)
expected_stack = traceback.StackSummary.extract(
traceback.walk_tb(exc_info[2]))
exc_cause = traceback.TracebackException(Exception, cause, None)
self.assertEqual(exc_cause, exc.__cause__)
self.assertEqual(exc_context, exc.__context__)
self.assertEqual(True, exc.__suppress_context__)
self.assertEqual(expected_stack, exc.stack)
self.assertEqual(exc_info[0], exc.exc_type)
self.assertEqual(str(exc_info[1]), str(exc))
def test_from_exception(self):
# Check all the parameters are accepted.
def foo():
1/0
try:
foo()
except Exception as e:
exc_info = sys.exc_info()
self.expected_stack = traceback.StackSummary.extract(
traceback.walk_tb(exc_info[2]), limit=1, lookup_lines=False,
capture_locals=True)
self.exc = traceback.TracebackException.from_exception(
e, limit=1, lookup_lines=False, capture_locals=True)
expected_stack = self.expected_stack
exc = self.exc
self.assertEqual(None, exc.__cause__)
self.assertEqual(None, exc.__context__)
self.assertEqual(False, exc.__suppress_context__)
self.assertEqual(expected_stack, exc.stack)
self.assertEqual(exc_info[0], exc.exc_type)
self.assertEqual(str(exc_info[1]), str(exc))
def test_traceback():
try:
assert_that('foo').is_equal_to('bar')
fail('should have raised error')
except AssertionError as ex:
assert_that(str(ex)).is_equal_to('Expected <foo> to be equal to <bar>, but was not.')
assert_that(ex).is_type_of(AssertionError)
# extract all stack frames from the traceback
_, _, tb = sys.exc_info()
assert_that(tb).is_not_none()
# walk_tb added in 3.5
if sys.version_info[0] == 3 and sys.version_info[1] >= 5:
frames = [(f.f_code.co_filename, f.f_code.co_name, lineno) for f,lineno in traceback.walk_tb(tb)]
assert_that(frames).is_length(3)
assert_that(frames[0][0]).ends_with('test_traceback.py')
assert_that(frames[0][1]).is_equal_to('test_traceback')
assert_that(frames[0][2]).is_equal_to(35)
assert_that(frames[1][0]).ends_with('assertpy.py')
assert_that(frames[1][1]).is_equal_to('is_equal_to')
assert_that(frames[1][2]).is_greater_than(160)
assert_that(frames[2][0]).ends_with('assertpy.py')
assert_that(frames[2][1]).is_equal_to('_err')
assert_that(frames[2][2]).is_greater_than(1000)
def compile_flow_subgraph(self, node_uids, requested_outputs=None, use_different_thetas=False, use_unique_input_names=False):
""" Compile and return one callable for the given flow_module_uids.
If use_different_thetas is True, the callable expects an argument names "thetas".
Thetas are expected to be sorted in the same way collect_thetas() would return them.
Parameters
----------
node_uids : list
the uids of the members of this graph
requested_outputs : list, optional
list of tuples (node_uid, out_name) to filter the callable's return-values. defaults to None, returning all outputs
use_different_thetas : boolean, optional
if true, return a callable that expects a parameter "thetas" that will be used instead of existing thetas. defaults to False
use_unique_input_names : boolen, optional
if true, the returned callable expects input-kwargs to be prefixe by node_uid: "UID_NAME". defaults to False, using only the name of the input
Returns
-------
callable : function
the compiled function for this subgraph
dangling_inputs : list
list of tuples (node_uid, input) that the callable expectes as inputs
dangling_outputs : list
list of tuples (node_uid, input) that the callable will return as output
"""
subgraph = [self.get_node(uid) for uid in self.flow_toposort if uid in node_uids]
# split the nodes into symbolic/non-symbolic paths
paths = self.split_flow_graph_into_implementation_paths(subgraph)
dangling_inputs = []
dangling_outputs = []
thunks = []
for path_idx, path in enumerate(paths):
thunk = {
'implementation': path['implementation'],
'function': None,
'node': None,
'outputs': [],
'input_sources': [],
'dangling_outputs': [],
'list_outputs': [],
'members': path['members']
}
member_uids = [n.uid for n in path['members']]
outexpressions = {}
inputs = []
outputs = []
skip = False
# index for outputs of this thunk, considering unpacked list outputs
thunk_flattened_output_index = 0
for node in path['members']:
buildargs = []
# collect the inputs for this Flowmodule:
for in_idx, in_name in enumerate(node.inputs):
if not node.inputmap[in_name] or node.inputmap[in_name][0] not in member_uids:
# this input is not satisfied from within this path
in_expr = create_tensor(node.definition['inputdims'][in_idx], T.config.floatX, name="%s_%s" % (node.uid, in_name))
inputs.append(in_expr)
if not node.inputmap[in_name] or node.inputmap[in_name][0] not in node_uids:
# it's not even satisfied by another path within the subgraph,
# and needs to be provided as input to the emerging callable
if use_unique_input_names:
thunk['input_sources'].append(('kwargs', -1, "%s_%s" % (node.uid, in_name)))
else:
thunk['input_sources'].append(('kwargs', -1, in_name))
dangling_inputs.append((node.uid, in_name))
else:
# this input will be satisfied by another path within the subgraph
source_uid, source_name = node.inputmap[in_name]
for idx, p in enumerate(paths):
if self.get_node(source_uid) in p['members']:
# record which thunk, and which index of its output-array satisfies this input
thunk['input_sources'].append(('path', idx, thunks[idx]['outputs'].index((source_uid, source_name))))
buildargs.append(in_expr)
else:
# this input is satisfied within this path
source_uid, source_name = node.inputmap[in_name]
buildargs.append(outexpressions[source_uid][self.get_node(source_uid).outputs.index(source_name)])
# build the outexpression
try:
if len(node.outputs) <= 1:
original_outex = [node.build(*buildargs)]
elif node.implementation == 'python':
func = node.build(*buildargs)
original_outex = [func] * len(node.outputs)
else:
original_outex = node.build(*buildargs)
except Exception as err:
import traceback as tb
frame = [f[0] for f in tb.walk_tb(err.__traceback__) if f[0].f_code.co_filename == node.definition.get('path', '')]
lineno = "<unknown>" if len(frame) == 0 else str(frame[0].f_lineno)
self.logger.error("Error in Flowmodule %s at line %s: %s: %s" % (str(node), lineno, err.__class__.__name__, str(err)))
post_mortem()
skip = True
break
outexpressions[node.uid] = original_outex
flattened_outex = []
outputlengths = []
flattened_markers = []
# check if this node has a list as one of its return values:
for idx, ex in enumerate(original_outex):
if type(ex) == list:
# if so, flatten the outputs, and mark the offset and length of the flattened output
# so that we can later reconstruct the nested output-structure
flattened_markers.append((len(outputs) + idx, len(ex)))
outputlengths.append(len(ex))
for item in ex:
flattened_outex.append(item)
else:
flattened_outex.append(ex)
outputlengths.append(1)
# offset for indexing the flattened_outexpression by output_index
node_flattened_output_offset = 0
# go thorugh the nodes outputs, and see how they will be used:
for out_idx, out_name in enumerate(node.outputs):
dangling = ['external']
if node.outputmap[out_name]:
# if this output is used, we have to see where every connection goes
# iterate through every connection, and note if it's used path-internally,
# subgraph-internally, or will produce an output of the emerging callable
dangling = []
for pair in node.outputmap[out_name]:
if pair[0] in member_uids:
# path-internally satisfied
dangling.append(False)
elif pair[0] in node_uids:
# internal dangling aka subgraph-internally satisfied
dangling.append("internal")
else:
# externally dangling aka this will be a final output
dangling.append("external")
# now, handle internally or externally dangling outputs if there are any:
if set(dangling) != {False}:
thunk['outputs'].append((node.uid, out_name))
if outputlengths[out_idx] > 1:
# if this is output should produce a list, note this, for later de-flattenation
# and append the flattened output to the output-collection
thunk['list_outputs'].append((thunk_flattened_output_index, outputlengths[out_idx]))
for i in range(outputlengths[out_idx]):
outputs.append(flattened_outex[out_idx + node_flattened_output_offset + i])
node_flattened_output_offset += outputlengths[out_idx] - 1
else:
outputs.append(flattened_outex[out_idx + node_flattened_output_offset])
if "external" in dangling:
# this output will be a final one:
if requested_outputs is None or (node.uid, out_name) in requested_outputs:
dangling_outputs.append((node.uid, out_name))
thunk['dangling_outputs'].append(thunk_flattened_output_index)
thunk_flattened_output_index += outputlengths[out_idx]
if skip:
# thunk borked, skip
continue
# now, set the function of this thunk. Either compile a theano function
# or assign the python function.
if not use_different_thetas:
if thunk['implementation'] == 'theano':
thunk['function'] = theano.function(inputs=inputs, outputs=outputs)
else:
thunk['node'] = path['members'][0]
thunk['function'] = outexpressions[thunk['node'].uid][0]
else:
sharedvars = self.collect_thetas(node_uids)
dummies = [create_tensor(var.ndim, T.config.floatX, name="Theta_%s" % var.name) for var in sharedvars]
if thunk['implementation'] == 'theano':
givens = list(zip(sharedvars, dummies))
thunk['function'] = theano.function(inputs=inputs + dummies, outputs=outputs, givens=givens)
else:
thunk['node'] = path['members'][0]
thunk['function'] = outexpressions[thunk['node'].uid][0]
thunks.append(thunk)
if not use_unique_input_names:
# check for name collisions
for thunk in thunks:
if len(set(thunk['input_sources'])) != (len(thunk['input_sources'])):
raise RuntimeError("""
Name Collision in inputs detected!
This graph can only be compiled as callable if you use unique_input_names.
set use_unique_input_names to True, and give the inputs as "UID_NAME"
where uid is the uid of the node getting this input, and name is the input name of this node""")
def compiled(thetas=None, **kwargs):
""" Compiled callable for this subgraph """
all_outputs = [] # outputs for use within this thunk
final_outputs = [] # final, external dangling outputs
for idx, thunk in enumerate(thunks):
funcargs = []
# get the inputs: Either from the kwargs, or from the already existing outputs
for source, pidx, item in thunk['input_sources']:
if source == 'kwargs':
funcargs.append(kwargs[item])
elif source == 'path':
funcargs.append(all_outputs[pidx][item])
if thunk['implementation'] == 'python':
params = thunk['node'].clone_parameters()
out = thunk['function'](*funcargs, netapi=self.netapi, node=thunk['node'], parameters=params)
if len(thunk['node'].outputs) <= 1:
out = [out]
else:
if type(out) != tuple:
raise RuntimeError("""Output mismatch!
Node %s returned only one output instead of %d.""" % (str(thunk['node']), len(thunk['node'].outputs)))
elif len(out) != len(thunk['node'].outputs):
raise RuntimeError("""Output mismatch!
Node %s returned %d outputs instead of %d.""" % (str(thunk['node']), len(out), len(thunk['node'].outputs)))
else:
if thetas:
funcargs += thetas
out = thunk['function'](*funcargs)
if thunk['list_outputs']:
# if we have list_outputs, we need to nest the output of this thunk again
# to recreate the nested structure from a flat list of outputs
new_out = []
out_iter = iter(out)
try:
for out_index in range(len(out)):
for offset, length in thunk['list_outputs']:
if offset == out_index:
sublist = []
for i in range(length):
sublist.append(next(out_iter))
new_out.append(sublist)
else:
new_out.append(next(out_iter))
except StopIteration:
# iterator finished, we handled all items.
pass
out = new_out
if out:
all_outputs.append(out)
for idx in thunk['dangling_outputs']:
if requested_outputs is None or thunk['outputs'][idx] in requested_outputs:
final_outputs.append(out[idx])
return final_outputs
compiled.__doc__ = """Compiled subgraph of nodes %s
Inputs: %s
Outputs: %s
""" % (str(subgraph), str([("%s of %s" % x[::-1]) for x in dangling_inputs]), str([("%s of %s" % x[::-1]) for x in dangling_outputs]))
return compiled, dangling_inputs, dangling_outputs
def extract(
cls,
exc_type: Type[BaseException],
exc_value: BaseException,
traceback: Optional[TracebackType],
show_locals: bool = False,
locals_max_length: int = LOCALS_MAX_LENGTH,
locals_max_string: int = LOCALS_MAX_STRING,
) -> Trace:
"""Extract traceback information.
Args:
exc_type (Type[BaseException]): Exception type.
exc_value (BaseException): Exception value.
traceback (TracebackType): Python Traceback object.
show_locals (bool, optional): Enable display of local variables. Defaults to False.
locals_max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
Defaults to 10.
locals_max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to 80.
Returns:
Trace: A Trace instance which you can use to construct a `Traceback`.
"""
stacks: List[Stack] = []
while True:
stack = Stack(exc_type=str(exc_type.__name__),
exc_value=str(exc_value))
if isinstance(exc_value, SyntaxError):
stack.syntax_error = _SyntaxError(
offset=exc_value.offset or 0,
filename=exc_value.filename or "?",
lineno=exc_value.lineno or 0,
line=exc_value.text or "",
msg=exc_value.msg,
)
stacks.append(stack)
append = stack.frames.append
for frame_summary, line_no in walk_tb(traceback):
filename = frame_summary.f_code.co_filename
filename = os.path.abspath(filename) if filename else "?"
frame = Frame(
filename=filename,
lineno=line_no,
name=frame_summary.f_code.co_name,
locals={
key: pretty.traverse(
value,
max_length=locals_max_length,
max_string=locals_max_string,
)
for key, value in frame_summary.f_locals.items()
} if show_locals else None,
)
append(frame)
cause = exc_value.__context__
if cause and cause.__traceback__:
exc_type = cause.__class__
exc_value = cause
traceback = cause.__traceback__
if traceback:
continue
# No cover, code is reached but coverage doesn't recognize it.
break # pragma: no cover
trace = Trace(stacks=stacks)
return trace
def add_tracebackhide_to_hidden_frames(tb):
for f, lineno in traceback.walk_tb(tb):
if not include_frame(f):
f.f_locals["__tracebackhide__"] = True
