def test_create_source_map_multiline_call(self):
test_fn = basic_definitions.function_with_multiline_call
source_map = self._create_source_map(test_fn)
module_path = tf_inspect.getsourcefile(test_fn)
# Origin line numbers below should match those in basic_definitions.py
fn_start = inspect.getsourcelines(test_fn)[1]
call_loc = origin_info.LineLocation('test_filename', 3)
self.assertIn(call_loc, source_map)
self.assertEqual(source_map[call_loc].loc.lineno, fn_start + 2)
self.assertEqual(source_map[call_loc].loc.filename, module_path)
self.assertEqual(source_map[call_loc].function_name,
'function_with_multiline_call')
self.assertEqual(source_map[call_loc].source_code_line,
' return range(')
second_arg_loc = origin_info.LineLocation('test_filename', 5)
self.assertIn(second_arg_loc, source_map)
self.assertEqual(source_map[second_arg_loc].loc.lineno, fn_start + 4)
self.assertEqual(source_map[second_arg_loc].loc.filename, module_path)
self.assertEqual(source_map[second_arg_loc].function_name,
'function_with_multiline_call')
self.assertEqual(source_map[second_arg_loc].source_code_line,
' x + 1,')
def test_create_source_map_multiple_nodes(self):
source = """
from __future__ import print_function
def test_fn(x):
return x + 1
"""
source = textwrap.dedent(source)
nodes = parser.parse_str(source, single_node=False)
fake_import_origin = origin_info.OriginInfo(
loc=origin_info.Location('fake_filename', 3, 7),
function_name='fake_function_name',
source_code_line='fake source line',
comment=None)
anno.setanno(nodes[0], anno.Basic.ORIGIN, fake_import_origin)
fake_function_origin = origin_info.OriginInfo(
loc=origin_info.Location('fake_filename', 3, 7),
function_name='fake_function_name',
source_code_line='fake source line',
comment=None)
anno.setanno(nodes[1], anno.Basic.ORIGIN, fake_function_origin)
source_map = origin_info.create_source_map(nodes, source,
'test_filename')
loc = origin_info.LineLocation('test_filename', 2)
self.assertIn(loc, source_map)
self.assertIs(source_map[loc], fake_import_origin)
loc = origin_info.LineLocation('test_filename', 3)
self.assertIn(loc, source_map)
self.assertIs(source_map[loc], fake_function_origin)
def test_get_message_converted_code(self):
callsite_tb = [
('/path/one.py', 11, 'test_fn_1', 'test code 1'),
('/path/two.py', 171, 'test_fn_2', 'test code 2'),
('/path/three.py', 171, 'test_fn_3', 'test code 3'),
]
cause_message = 'Test message'
em = error_utils.ErrorMetadataBase(
callsite_tb=callsite_tb,
cause_metadata=None,
cause_message=cause_message,
source_map={
origin_info.LineLocation(filename='/path/two.py', lineno=171):
origin_info.OriginInfo(
loc=origin_info.LineLocation(
filename='/path/other_two.py', lineno=13),
function_name='converted_fn',
source_code_line='converted test code',
comment=None)
},
converter_filename=None)
result = em.get_message()
self.assertRegex(
result,
re.compile((r'converted_fn \*.*'
r'"/path/three.py", line 171, in test_fn_3.*'
r'Test message'), re.DOTALL))
self.assertNotRegex(result, re.compile('test_fn_1'))
def map(self, filename, lineno, name):
loc = origin_info.LineLocation(filename=filename, lineno=lineno)
if loc not in self._source_map:
return filename, lineno, name
origin = self._source_map[loc]
return origin.loc.filename, origin.loc.lineno, origin.function_name
def get_effective_source_map(self):
effective_source_map = self._effective_source_map
if effective_source_map is None:
if self.parent is not None:
parent_map = self.parent.get_effective_source_map()
else:
parent_map = {}
effective_source_map = {}
for loc, origin in self._source_map.items():
effective_source_map[(loc.filename,
loc.lineno)] = (origin.loc.filename,
origin.loc.lineno,
origin.function_name)
for key, value in parent_map.items():
filename, lineno, _ = value
value_loc = origin_info.LineLocation(filename=filename,
lineno=lineno)
if value_loc in self._source_map:
origin = self._source_map[value_loc]
effective_source_map[key] = (origin.loc.filename,
origin.loc.lineno,
origin.function_name)
else:
effective_source_map[key] = value
self._effective_source_map = effective_source_map
return effective_source_map
def fake_origin(self, function, line_offset):
_, lineno = tf_inspect.getsourcelines(function)
filename = tf_inspect.getsourcefile(function)
lineno += line_offset
loc = origin_info.LineLocation(filename, lineno)
origin = origin_info.OriginInfo(loc, 'test_function_name', 'test_code',
'test_comment')
return loc, origin
def test_source_map(self):
def test_fn(x):
if x > 0:
x += 1
return x
node, source = parser.parse_entity(test_fn)
fn_node = node.body[0]
origin_info.resolve(fn_node, source)
# Insert a traced line.
new_node = parser.parse_str('x = abs(x)').body[0]
anno.copyanno(fn_node.body[0], new_node, anno.Basic.ORIGIN)
fn_node.body.insert(0, new_node)
# Insert an untraced line.
fn_node.body.insert(0, parser.parse_str('x = 0').body[0])
modified_source = compiler.ast_to_source(fn_node)
source_map = origin_info.source_map(fn_node, modified_source,
'test_filename', [0])
loc = origin_info.LineLocation('test_filename', 1)
origin = source_map[loc]
self.assertEqual(origin.source_code_line, 'def test_fn(x):')
self.assertEqual(origin.loc.lineno, 1)
# The untraced line, inserted second.
loc = origin_info.LineLocation('test_filename', 2)
self.assertFalse(loc in source_map)
# The traced line, inserted first.
loc = origin_info.LineLocation('test_filename', 3)
origin = source_map[loc]
self.assertEqual(origin.source_code_line, ' if x > 0:')
self.assertEqual(origin.loc.lineno, 2)
loc = origin_info.LineLocation('test_filename', 4)
origin = source_map[loc]
self.assertEqual(origin.source_code_line, ' if x > 0:')
self.assertEqual(origin.loc.lineno, 2)
def _cut_traceback_loops(source_map, original_traceback):
"""Check for cases where we leave a user method and re-enter it.
This is done by looking at the function names when the filenames are from any
files the user code is in. If we find a case where we return to a user method
after leaving it then we cut out the frames in between because we assume this
means these in between frames are from internal AutoGraph code that shouldn't
be included.
An example of this is:
File "file1.py", line 57, in my_func
...
File "control_flow_ops.py", line 231, in cond
...
File "control_flow_ops.py", line 1039, in inner_cond
...
File "file1.py", line 68, in my_func
...
Where we would remove the control_flow_ops.py frames because we re-enter
my_func in file1.py.
The source map keys are (file_path, line_number) so get the set of all user
file_paths.
Args:
source_map: Dict[origin_info.LineLocation, origin_info.OriginInfo], mapping
locations to their origin
original_traceback: List[Tuple[Text, Text, Text, Text]], consistent with
traceback.extract_tb.
Returns:
List[Tuple[Text, Text, Text, Text]], the traceback with any loops removed.
"""
all_user_files = set(loc.filename for loc in source_map)
cleaned_traceback = []
last_user_frame_index = None
last_user_user_file_path = None
# TODO(mdan): Simplify this logic.
for fi, frame in enumerate(original_traceback):
frame_file_path, lineno, _, _ = frame
src_map_key = origin_info.LineLocation(frame_file_path, lineno)
if frame_file_path in all_user_files:
if src_map_key in source_map:
if (last_user_frame_index is not None
and last_user_user_file_path == frame_file_path):
cleaned_traceback = cleaned_traceback[:
last_user_frame_index]
last_user_frame_index = fi
last_user_user_file_path = frame_file_path
cleaned_traceback.append(frame)
return cleaned_traceback
def test_create_source_map_identity(self):
test_fn = basic_definitions.simple_function
source_map = self._create_source_map(test_fn)
module_path = tf_inspect.getsourcefile(test_fn)
# Origin line numbers below should match those in basic_definitions.py
definition_loc = origin_info.LineLocation('test_filename', 1)
self.assertIn(definition_loc, source_map)
self.assertEqual(source_map[definition_loc].loc.lineno, 23)
self.assertEqual(source_map[definition_loc].loc.filename, module_path)
self.assertEqual(source_map[definition_loc].function_name,
'simple_function')
def extract_origin_info(converted_f):
"""Attempts to use converted_f's source map to get error origin info."""
source_map = converted_f.ag_source_map
original_traceback = traceback.extract_tb(sys.exc_info()[2])
# Can go through all frames and check which ones have origin info in order to
# filter for only the locations relevant to converted_f.
#
# Return the first occurrence of the reversed traceback in the source map in
# order to return the innermost frame for this function. We want to do this
# because when have a tf.cond we will have multiple matches and we want to
# return the last one in this function, because any after that will be in
# the next function/frame in the stacktrace.
for frame in reversed(original_traceback):
converted_loc = origin_info.LineLocation(filename=frame[0],
lineno=frame[1])
if converted_loc in source_map:
return source_map[converted_loc]
return None
def test_create_source_map(self):
def test_fn(x):
return x + 1
node, _, _ = parser.parse_entity(test_fn)
fake_origin = origin_info.OriginInfo(
loc=origin_info.Location('fake_filename', 3, 7),
function_name='fake_function_name',
source_code_line='fake source line',
comment=None)
anno.setanno(node.body[0], anno.Basic.ORIGIN, fake_origin)
converted_code = compiler.ast_to_source(node)
source_map = origin_info.create_source_map(node, converted_code,
'test_filename', [0])
loc = origin_info.LineLocation('test_filename', 2)
self.assertIn(loc, source_map)
self.assertIs(source_map[loc], fake_origin)
def _rewrite_tb(source_map, tb):
"""Rewrites code references in a traceback.
Args:
source_map: Dict[origin_info.LineLocation, origin_info.OriginInfo], mapping
locations to their origin
tb: List[Tuple[Text, Text, Text, Text]], consistent with
traceback.extract_tb.
Returns:
List[Tuple[Text, Text, Text, Text]], the rewritten traceback
"""
new_tb = []
for frame in tb:
filename, lineno, _, _ = frame
loc = origin_info.LineLocation(filename, lineno)
origin = source_map.get(loc)
if origin is not None:
new_tb.append(origin.as_frame())
else:
new_tb.append(frame)
return new_tb
def test_create_source_map(self):
source = """
def test_fn(x):
return x + 1
"""
source = textwrap.dedent(source)
node = parser.parse(source)
fake_origin = origin_info.OriginInfo(
loc=origin_info.Location('fake_filename', 3, 7),
function_name='fake_function_name',
source_code_line='fake source line',
comment=None)
anno.setanno(node, anno.Basic.ORIGIN, fake_origin)
source_map = origin_info.create_source_map(node, source, 'test_filename')
loc = origin_info.LineLocation('test_filename', 2)
self.assertIn(loc, source_map)
self.assertIs(source_map[loc], fake_origin)
def _stack_trace_inside_mapped_code(tb, source_map, converter_filename):
"""Summarizes inner traceback frames up to the call to a given function.
This functions locates the innermost (i.e. most recent) frame that corresponds
to code that can be mapped by source_map originated from, and returns a
translated stack trace ending at that frame. If no such frame is found, the
entire stack trace is summarized.
For example, the following code:
def f():
for i in tf.range(1):
z = y + i # z only defined here
Would generate this traceback:
<converted code>
ag__.for_stmt(...)
<for_stmt>
return _known_len_tf_for_stmt(iter_, extra_test, body, init_state)
<_known_len_tf_for_stmt>
_disallow_undefs_into_loop(*init_state)
<_disallow_undefs_into_loop>
raise ...
Which is then processed into:
<f>
for i in tf.range(1):
<for_stmt>
return _known_len_tf_for_stmt(iter_, extra_test, body, init_state)
<_known_len_tf_for_stmt>
_disallow_undefs_into_loop(*init_state)
<_disallow_undefs_into_loop>
raise ...
Args:
tb: traceback.FrameSummary, The traceback corresponding to an error.
Typically, the output of traceback.Summary.extract(capture_locals=True).
source_map: Dict[LineLocation, OriginInfo], a source map as created by
origin_info.create_source_map.
converter_filename: str, the file path of the converted module. Call frames
corresponding to this module are elided and their preceding frames are
marked as allowlisted. Note that frames enclosing converted code are
dropped using a different mechanism.
Returns:
List[FrameInfo]
"""
result_frames = []
for filename, line_number, function_name, text in reversed(tb):
loc = origin_info.LineLocation(filename=filename, lineno=line_number)
if loc in source_map:
origin = source_map[loc]
fi = FrameInfo(filename=origin.loc.filename,
lineno=origin.loc.lineno,
function_name=origin.function_name,
code=origin.source_code_line,
is_converted=True,
is_allowlisted=False)
result_frames.append(fi)
break
if filename == converter_filename:
if result_frames:
prev = result_frames[-1]
assert not prev.is_converted # See the if above.
fi = FrameInfo(filename=prev.filename,
lineno=prev.lineno,
function_name=prev.function_name,
code=prev.code,
is_converted=False,
is_allowlisted=True)
result_frames[-1] = fi
continue
fi = FrameInfo(filename=filename,
lineno=line_number,
function_name=function_name,
code=text,
is_converted=False,
is_allowlisted=False)
result_frames.append(fi)
return tuple(result_frames)
def _stack_trace_inside_mapped_code(tb, source_map):
"""Summarizes inner traceback frames up to the call to a given function.
This functions locates the innermost (i.e. most recent) frame that corresponds
to code that can be mapped by source_map originated from, and returns a
translated stack trace ending at that frame. If no such frame is found, the
entire stack trace is summarized.
For example, the following code:
def f():
for i in tf.range(1):
z = y + i # z only defined here
Would generate this traceback:
<converted code>
ag__.for_stmt(...)
<for_stmt>
return _known_len_tf_for_stmt(iter_, extra_test, body, init_state)
<_known_len_tf_for_stmt>
_disallow_undefs_into_loop(*init_state)
<_disallow_undefs_into_loop>
raise ...
Which is then processed into:
<f>
for i in tf.range(1):
<for_stmt>
return _known_len_tf_for_stmt(iter_, extra_test, body, init_state)
<_known_len_tf_for_stmt>
_disallow_undefs_into_loop(*init_state)
<_disallow_undefs_into_loop>
raise ...
Args:
tb: List[Tuple], the traceback corresponding to an error; typically,
the output of traceback.extract_tb.
source_map: Dict[LineLocation, OriginInfo], a source map as created by
origin_info.create_source_map.
Returns:
List[FrameInfo]
"""
result_frames = []
for filename, line_number, function_name, text in reversed(tb):
loc = origin_info.LineLocation(filename=filename, lineno=line_number)
if loc in source_map:
origin = source_map[loc]
origin_frame_info = FrameInfo(filename=origin.loc.filename,
lineno=origin.loc.lineno,
function_name=origin.function_name,
code=origin.source_code_line,
converted=True)
result_frames.append(origin_frame_info)
break
fi = FrameInfo(filename=filename,
lineno=line_number,
function_name=function_name,
code=text,
converted=False)
result_frames.append(fi)
return tuple(result_frames)
