def fake_origin(self, function, line_offset): _, lineno = tf_inspect.getsourcelines(function) filename = tf_inspect.getsourcefile(function) lineno += line_offset loc = origin_info.LineLocation(filename, lineno) origin = origin_info.OriginInfo(loc, 'test_function_name', 'test_code') return loc, origin
def fake_origin(self, function, line_offset):
_, lineno = tf_inspect.getsourcelines(function)
filename = tf_inspect.getsourcefile(function)
lineno += line_offset
loc = origin_info.LineLocation(filename, lineno)
origin = origin_info.OriginInfo(loc, 'test_function_name', 'test_code')
return loc, origin
def resolve(nodes, source, function=None):
"""Adds an origin information to all nodes inside the body of function.
Args:
nodes: Union[ast.AST, Iterable[ast.AST, ...]]
source: Text, the source code string for the function whose body nodes will
be annotated.
function: Callable, the function that will have all nodes inside of it
annotation with an OriginInfo annotation with key anno.Basic.ORIGIN. If
it is None then only the line numbers and column offset will be set in the
annotation, with the rest of the information being None.
Returns:
A tuple of the AST node for function and a String containing its source
code.
"""
if not isinstance(nodes, (list, tuple)):
nodes = (nodes, )
if function:
_, function_lineno = tf_inspect.getsourcelines(function)
function_filepath = tf_inspect.getsourcefile(function)
else:
function_lineno = None
function_filepath = None
# TODO(mdan): Pull this to a separate utility.
code_reader = six.StringIO(source)
comment_map = {}
for token in tokenize.generate_tokens(code_reader.readline):
tok_type, tok_string, loc, _, _ = token
srow, _ = loc
if tok_type == tokenize.COMMENT:
comment_map[srow] = tok_string.strip()[1:].strip()
source_lines = source.split('\n')
for node in nodes:
for n in gast.walk(node):
if not hasattr(n, 'lineno'):
continue
lineno_in_body = n.lineno
source_code_line = source_lines[lineno_in_body - 1]
if function:
source_lineno = function_lineno + lineno_in_body
function_name = function.__name__
else:
source_lineno = lineno_in_body
function_name = None
location = Location(function_filepath, source_lineno, n.col_offset)
origin = OriginInfo(location, function_name, source_code_line,
comment_map.get(source_lineno))
anno.setanno(n, anno.Basic.ORIGIN, origin)
def resolve(nodes, source, function=None):
"""Adds an origin information to all nodes inside the body of function.
Args:
nodes: Union[ast.AST, Iterable[ast.AST, ...]]
source: Text, the source code string for the function whose body nodes will
be annotated.
function: Callable, the function that will have all nodes inside of it
annotation with an OriginInfo annotation with key anno.Basic.ORIGIN. If
it is None then only the line numbers and column offset will be set in the
annotation, with the rest of the information being None.
Returns:
A tuple of the AST node for function and a String containing its source
code.
"""
if not isinstance(nodes, (list, tuple)):
nodes = (nodes,)
if function:
_, function_lineno = tf_inspect.getsourcelines(function)
function_filepath = tf_inspect.getsourcefile(function)
else:
function_lineno = None
function_filepath = None
# TODO(mdan): Pull this to a separate utility.
code_reader = six.StringIO(source)
comment_map = {}
for token in tokenize.generate_tokens(code_reader.readline):
tok_type, tok_string, loc, _, _ = token
srow, _ = loc
if tok_type == tokenize.COMMENT:
comment_map[srow] = tok_string.strip()[1:].strip()
source_lines = source.split('\n')
for node in nodes:
for n in gast.walk(node):
if not hasattr(n, 'lineno'):
continue
lineno_in_body = n.lineno
source_code_line = source_lines[lineno_in_body - 1]
if function:
source_lineno = function_lineno + lineno_in_body
function_name = function.__name__
else:
source_lineno = lineno_in_body
function_name = None
location = Location(function_filepath, source_lineno, n.col_offset)
origin = OriginInfo(location, function_name,
source_code_line, comment_map.get(source_lineno))
anno.setanno(n, anno.Basic.ORIGIN, origin)
def resolve_entity(node, source, entity): """Like resolve, but extracts the context informartion from an entity.""" lines, lineno = tf_inspect.getsourcelines(entity) filepath = tf_inspect.getsourcefile(entity) # Poor man's attempt at guessing the column offset: count the leading # whitespace. This might not work well with tabs. definition_line = lines[0] col_offset = len(definition_line) - len(definition_line.lstrip()) resolve(node, source, filepath, lineno, col_offset)
def test_create_source_map_no_origin_info(self):
test_fn = basic_definitions.simple_function
node, _ = parser.parse_entity(test_fn,
inspect_utils.getfutureimports(test_fn))
# No origin information should result in an empty map.
test_fn_lines, _ = tf_inspect.getsourcelines(test_fn)
source_map = origin_info.create_source_map(node, '\n'.join(test_fn_lines),
test_fn)
self.assertEmpty(source_map)
def testSetFilenameAndLineFromCallerUsesCallersStack(self):
t_obj = traceable_stack.TraceableObject(17)
# Do not separate placeholder from the set_filename_and_line_from_caller()
# call one line below it as it is used to calculate the latter's line
# number.
placeholder = lambda x: x
result = t_obj.set_filename_and_line_from_caller()
expected_lineno = inspect.getsourcelines(placeholder)[1] + 1
self.assertEqual(expected_lineno, t_obj.lineno)
self.assertEqual(_THIS_FILENAME, t_obj.filename)
self.assertEqual(t_obj.SUCCESS, result)
def testSetFilenameAndLineFromCallerUsesCallersStack(self):
t_obj = traceable_stack.TraceableObject(17)
# Do not separate placeholder from the set_filename_and_line_from_caller()
# call one line below it as it is used to calculate the latter's line
# number.
placeholder = lambda x: x
result = t_obj.set_filename_and_line_from_caller()
expected_lineno = inspect.getsourcelines(placeholder)[1] + 1
self.assertEqual(expected_lineno, t_obj.lineno)
self.assertEqual(_THIS_FILENAME, t_obj.filename)
self.assertEqual(t_obj.SUCCESS, result)
def resolve(node, source, function=None):
"""Adds an origin information to node and its subnodes.
This allows us to map the original source code line numbers to generated
source code.
Args:
node: gast.AST node. Should be a gast.FunctionDef. This is the node we
annotate with origin information.
source: Text, the source code. Should satisfy relationship
`node in iter_tree(gast.parse(source))`; otherwise the lineno will be
unreliable.
function: The original function. If it is None then only the line numbers
and column offset will be set in the annotation, with the rest of the
information being None.
"""
if function:
_, function_lineno = tf_inspect.getsourcelines(function)
function_filepath = tf_inspect.getsourcefile(function)
else:
function_lineno = None
function_filepath = None
# TODO(mdan): Pull this to a separate utility.
code_reader = six.StringIO(source)
comment_map = {}
for token in tokenize.generate_tokens(code_reader.readline):
tok_type, tok_string, loc, _, _ = token
srow, _ = loc
if tok_type == tokenize.COMMENT:
comment_map[srow] = tok_string.strip()[1:].strip()
source_lines = source.split('\n')
for n in gast.walk(node):
if not hasattr(n, 'lineno'):
continue
within_body_offset = n.lineno - node.lineno
source_code_line = source_lines[n.lineno - 1]
if function:
source_lineno = function_lineno + within_body_offset
function_name = function.__name__
else:
source_lineno = n.lineno
function_name = None
location = Location(function_filepath, source_lineno, n.col_offset)
origin = OriginInfo(location, function_name,
source_code_line, comment_map.get(source_lineno))
anno.setanno(n, anno.Basic.ORIGIN, origin)
def resolve(node, source, function=None):
"""Adds an origin information to node and its subnodes.
This allows us to map the original source code line numbers to generated
source code.
Args:
node: gast.AST node. Should be a gast.FunctionDef. This is the node we
annotate with origin information.
source: Text, the source code. Should satisfy relationship
`node in iter_tree(gast.parse(source))`; otherwise the lineno will be
unreliable.
function: The original function. If it is None then only the line numbers
and column offset will be set in the annotation, with the rest of the
information being None.
"""
if function:
_, function_lineno = tf_inspect.getsourcelines(function)
function_filepath = tf_inspect.getsourcefile(function)
else:
function_lineno = None
function_filepath = None
# TODO(mdan): Pull this to a separate utility.
code_reader = six.StringIO(source)
comment_map = {}
for token in tokenize.generate_tokens(code_reader.readline):
tok_type, tok_string, loc, _, _ = token
srow, _ = loc
if tok_type == tokenize.COMMENT:
comment_map[srow] = tok_string.strip()[1:].strip()
source_lines = source.split('\n')
for n in gast.walk(node):
if not hasattr(n, 'lineno'):
continue
within_body_offset = n.lineno - node.lineno
source_code_line = source_lines[n.lineno - 1]
if function:
source_lineno = function_lineno + within_body_offset
function_name = function.__name__
else:
source_lineno = n.lineno
function_name = None
location = Location(function_filepath, source_lineno, n.col_offset)
origin = OriginInfo(location, function_name, source_code_line,
comment_map.get(source_lineno))
anno.setanno(n, anno.Basic.ORIGIN, origin)
def testPushObjSetsFilenameAndLineInfoForCaller(self):
t_stack = traceable_stack.TraceableStack()
# We expect that the line number recorded for the 1-object will come from
# the call to t_stack.push_obj(1). Do not separate the next two lines!
placeholder_1 = lambda x: x
t_stack.push_obj(1)
# We expect that the line number recorded for the 2-object will come from
# the call to call_push_obj() and _not_ the call to t_stack.push_obj().
def call_push_obj(obj):
t_stack.push_obj(obj, offset=1)
# Do not separate the next two lines!
placeholder_2 = lambda x: x
call_push_obj(2)
expected_lineno_1 = inspect.getsourcelines(placeholder_1)[1] + 1
expected_lineno_2 = inspect.getsourcelines(placeholder_2)[1] + 1
t_obj_2, t_obj_1 = t_stack.peek_traceable_objs()
self.assertEqual(expected_lineno_2, t_obj_2.lineno)
self.assertEqual(expected_lineno_1, t_obj_1.lineno)
def testPushObjSetsFilenameAndLineInfoForCaller(self):
t_stack = traceable_stack.TraceableStack()
# We expect that the line number recorded for the 1-object will come from
# the call to t_stack.push_obj(1). Do not separate the next two lines!
placeholder_1 = lambda x: x
t_stack.push_obj(1)
# We expect that the line number recorded for the 2-object will come from
# the call to call_push_obj() and _not_ the call to t_stack.push_obj().
def call_push_obj(obj):
t_stack.push_obj(obj, offset=1)
# Do not separate the next two lines!
placeholder_2 = lambda x: x
call_push_obj(2)
expected_lineno_1 = inspect.getsourcelines(placeholder_1)[1] + 1
expected_lineno_2 = inspect.getsourcelines(placeholder_2)[1] + 1
t_obj_2, t_obj_1 = t_stack.peek_traceable_objs()
self.assertEqual(expected_lineno_2, t_obj_2.lineno)
self.assertEqual(expected_lineno_1, t_obj_1.lineno)
def test_rewriting_error(self):
_, zero_div_lineno = tf_inspect.getsourcelines(zero_div)
src_map = {
errors.CodeLocation(file_path=__file__,
line_number=zero_div_lineno + 1):
None
}
with self.assertRaisesRegexp(tf_errors.InvalidArgumentError,
'Integer division by zero'):
z = zero_div_caller()
zero_div_caller.ag_source_map = src_map
with errors.improved_errors(zero_div_caller):
with self.test_session() as sess:
sess.run(z)
def test_rewriting_error(self):
_, zero_div_lineno = tf_inspect.getsourcelines(zero_div)
src_map = {
errors.CodeLocation(
file_path=__file__, line_number=zero_div_lineno + 1):
None
}
with self.assertRaisesRegexp(tf_errors.InvalidArgumentError,
'Integer division by zero'):
z = zero_div_caller()
zero_div_caller.ag_source_map = src_map
with errors.improved_errors(zero_div_caller):
with self.test_session() as sess:
sess.run(z)
def testSetFilenameAndLineFromCallerRespectsOffset(self):
def call_set_filename_and_line_from_caller(t_obj):
# We expect to retrieve the line number from _our_ caller.
return t_obj.set_filename_and_line_from_caller(offset=1)
t_obj = traceable_stack.TraceableObject(None)
# Do not separate placeholder from the
# call_set_filename_and_line_from_caller() call one line below it as it is
# used to calculate the latter's line number.
placeholder = lambda x: x
result = call_set_filename_and_line_from_caller(t_obj)
expected_lineno = inspect.getsourcelines(placeholder)[1] + 1
self.assertEqual(expected_lineno, t_obj.lineno)
self.assertEqual(t_obj.SUCCESS, result)
def testSetFilenameAndLineFromCallerRespectsOffset(self):
def call_set_filename_and_line_from_caller(t_obj):
# We expect to retrieve the line number from _our_ caller.
return t_obj.set_filename_and_line_from_caller(offset=1)
t_obj = traceable_stack.TraceableObject(None)
# Do not separate placeholder from the
# call_set_filename_and_line_from_caller() call one line below it as it is
# used to calculate the latter's line number.
placeholder = lambda x: x
result = call_set_filename_and_line_from_caller(t_obj)
expected_lineno = inspect.getsourcelines(placeholder)[1] + 1
self.assertEqual(expected_lineno, t_obj.lineno)
self.assertEqual(t_obj.SUCCESS, result)
def resolve(nodes, source, function=None):
"""Adds an origin information to all nodes inside the body of function.
Args:
nodes: Union[ast.AST, Iterable[ast.AST, ...]]
source: Text, the source code string for the function whose body nodes will
be annotated.
function: Callable, the function that will have all nodes inside of it
annotation with an OriginInfo annotation with key anno.Basic.ORIGIN. If
it is None then only the line numbers and column offset will be set in the
annotation, with the rest of the information being None.
Returns:
A tuple of the AST node for function and a String containing its source
code.
"""
if not isinstance(nodes, (list, tuple)):
nodes = (nodes,)
if function:
_, function_lineno = tf_inspect.getsourcelines(function)
function_filepath = tf_inspect.getsourcefile(function)
else:
function_lineno = None
function_filepath = None
source_lines = source.split('\n')
for node in nodes:
for n in gast.walk(node):
if not hasattr(n, 'lineno'):
continue
lineno_in_body = n.lineno
source_code_line = source_lines[lineno_in_body - 1]
if function:
source_lineno = function_lineno + lineno_in_body
function_name = function.__name__
else:
source_lineno = lineno_in_body
function_name = None
location = Location(function_filepath, source_lineno, n.col_offset)
origin = OriginInfo(location, function_name, source_code_line)
anno.setanno(n, anno.Basic.ORIGIN, origin)
def test_error_replacement(self):
_, zero_div_lineno = tf_inspect.getsourcelines(zero_div)
src_map = {
errors.CodeLocation(file_path=__file__,
line_number=zero_div_lineno + 1):
self._fake_origin
}
with self.assertRaises(errors.TfRuntimeError) as cm:
z = zero_div_caller()
zero_div_caller.ag_source_map = src_map
with errors.improved_errors(zero_div_caller):
with self.test_session() as sess:
sess.run(z)
expected = cm.exception
current_traceback = expected.custom_traceback
for frame in current_traceback:
self.assertNotEqual('zero_div', frame[2])
self.assertTrue(
any(self._fake_origin.as_frame() == frame
for frame in current_traceback))
def test_error_replacement(self):
_, zero_div_lineno = tf_inspect.getsourcelines(zero_div)
src_map = {
errors.CodeLocation(
file_path=__file__, line_number=zero_div_lineno + 1):
self._fake_origin
}
with self.assertRaises(errors.TfRuntimeError) as cm:
z = zero_div_caller()
zero_div_caller.ag_source_map = src_map
with errors.improved_errors(zero_div_caller):
with self.test_session() as sess:
sess.run(z)
expected = cm.exception
current_traceback = expected.custom_traceback
for frame in current_traceback:
self.assertNotEqual('zero_div', frame[2])
self.assertTrue(
any(self._fake_origin.as_frame() == frame
for frame in current_traceback))
def resolve(node, source, function=None):
"""Adds an origin information to all nodes inside the body of function.
Args:
node: The AST node for the function whose body nodes will be annotated.
source: Text, the source code string for the function whose body nodes will
be annotated.
function: Callable, the function that will have all nodes inside of it
annotation with an OriginInfo annotation with key anno.Basic.ORIGIN. If
it is None then only the line numbers and column offset will be set in the
annotation, with the rest of the information being None.
Returns:
A tuple of the AST node for function and a String containing its source
code.
"""
if function:
_, function_lineno = tf_inspect.getsourcelines(function)
function_filepath = tf_inspect.getsourcefile(function)
else:
function_lineno = None
function_filepath = None
source_lines = source.split('\n')
for n in gast.walk(node):
if hasattr(n, 'lineno'):
# n.lineno is relative to the start of the enclosing function, so need to
# offset it by the line of the function.
source_code_line = source_lines[n.lineno - 1]
if function:
source_lineno = n.lineno + function_lineno - 1
function_name = function.__name__
else:
source_lineno = n.lineno
function_name = None
anno.setanno(
n, anno.Basic.ORIGIN,
OriginInfo(function_filepath, function_name, source_lineno,
n.col_offset, source_code_line))
def resolve(node, source, function=None):
"""Adds an origin information to all nodes inside the body of function.
Args:
node: The AST node for the function whose body nodes will be annotated.
source: Text, the source code string for the function whose body nodes will
be annotated.
function: Callable, the function that will have all nodes inside of it
annotation with an OriginInfo annotation with key anno.Basic.ORIGIN. If
it is None then only the line numbers and column offset will be set in the
annotation, with the rest of the information being None.
Returns:
A tuple of the AST node for function and a String containing its source
code.
"""
if function:
_, function_lineno = tf_inspect.getsourcelines(function)
function_filepath = tf_inspect.getsourcefile(function)
else:
function_lineno = None
function_filepath = None
source_lines = source.split('\n')
for n in gast.walk(node):
if hasattr(n, 'lineno'):
# n.lineno is relative to the start of the enclosing function, so need to
# offset it by the line of the function.
source_code_line = source_lines[n.lineno - 1]
if function:
source_lineno = n.lineno + function_lineno - 1
function_name = function.__name__
else:
source_lineno = n.lineno
function_name = None
anno.setanno(
n, anno.Basic.ORIGIN,
OriginInfo(function_filepath, function_name, source_lineno,
n.col_offset, source_code_line))
def testGetSourceLines(self):
expected = inspect.getsourcelines(
test_decorated_function_with_defaults.decorated_target)
self.assertEqual(
expected,
tf_inspect.getsourcelines(test_decorated_function_with_defaults))
def testGetSourceLines(self):
expected = inspect.getsourcelines(
test_decorated_function_with_defaults.decorated_target)
self.assertEqual(
expected,
tf_inspect.getsourcelines(test_decorated_function_with_defaults))
