def fake_origin(self, function, line_offset): _, lineno = tf_inspect.getsourcelines(function) filename = tf_inspect.getsourcefile(function) lineno += line_offset loc = origin_info.LineLocation(filename, lineno) origin = origin_info.OriginInfo(loc, 'test_function_name', 'test_code') return loc, origin
def resolve(nodes, source, function=None): """Adds an origin information to all nodes inside the body of function. Args: nodes: Union[ast.AST, Iterable[ast.AST, ...]] source: Text, the source code string for the function whose body nodes will be annotated. function: Callable, the function that will have all nodes inside of it annotation with an OriginInfo annotation with key anno.Basic.ORIGIN. If it is None then only the line numbers and column offset will be set in the annotation, with the rest of the information being None. Returns: A tuple of the AST node for function and a String containing its source code. """ if not isinstance(nodes, (list, tuple)): nodes = (nodes, ) if function: _, function_lineno = tf_inspect.getsourcelines(function) function_filepath = tf_inspect.getsourcefile(function) else: function_lineno = None function_filepath = None # TODO(mdan): Pull this to a separate utility. code_reader = six.StringIO(source) comment_map = {} for token in tokenize.generate_tokens(code_reader.readline): tok_type, tok_string, loc, _, _ = token srow, _ = loc if tok_type == tokenize.COMMENT: comment_map[srow] = tok_string.strip()[1:].strip() source_lines = source.split('\n') for node in nodes: for n in gast.walk(node): if not hasattr(n, 'lineno'): continue lineno_in_body = n.lineno source_code_line = source_lines[lineno_in_body - 1] if function: source_lineno = function_lineno + lineno_in_body function_name = function.__name__ else: source_lineno = lineno_in_body function_name = None location = Location(function_filepath, source_lineno, n.col_offset) origin = OriginInfo(location, function_name, source_code_line, comment_map.get(source_lineno)) anno.setanno(n, anno.Basic.ORIGIN, origin)
def resolve(nodes, source, function=None): """Adds an origin information to all nodes inside the body of function. Args: nodes: Union[ast.AST, Iterable[ast.AST, ...]] source: Text, the source code string for the function whose body nodes will be annotated. function: Callable, the function that will have all nodes inside of it annotation with an OriginInfo annotation with key anno.Basic.ORIGIN. If it is None then only the line numbers and column offset will be set in the annotation, with the rest of the information being None. Returns: A tuple of the AST node for function and a String containing its source code. """ if not isinstance(nodes, (list, tuple)): nodes = (nodes,) if function: _, function_lineno = tf_inspect.getsourcelines(function) function_filepath = tf_inspect.getsourcefile(function) else: function_lineno = None function_filepath = None # TODO(mdan): Pull this to a separate utility. code_reader = six.StringIO(source) comment_map = {} for token in tokenize.generate_tokens(code_reader.readline): tok_type, tok_string, loc, _, _ = token srow, _ = loc if tok_type == tokenize.COMMENT: comment_map[srow] = tok_string.strip()[1:].strip() source_lines = source.split('\n') for node in nodes: for n in gast.walk(node): if not hasattr(n, 'lineno'): continue lineno_in_body = n.lineno source_code_line = source_lines[lineno_in_body - 1] if function: source_lineno = function_lineno + lineno_in_body function_name = function.__name__ else: source_lineno = lineno_in_body function_name = None location = Location(function_filepath, source_lineno, n.col_offset) origin = OriginInfo(location, function_name, source_code_line, comment_map.get(source_lineno)) anno.setanno(n, anno.Basic.ORIGIN, origin)
def resolve_entity(node, source, entity): """Like resolve, but extracts the context informartion from an entity.""" lines, lineno = tf_inspect.getsourcelines(entity) filepath = tf_inspect.getsourcefile(entity) # Poor man's attempt at guessing the column offset: count the leading # whitespace. This might not work well with tabs. definition_line = lines[0] col_offset = len(definition_line) - len(definition_line.lstrip()) resolve(node, source, filepath, lineno, col_offset)
def test_create_source_map_no_origin_info(self): test_fn = basic_definitions.simple_function node, _ = parser.parse_entity(test_fn, inspect_utils.getfutureimports(test_fn)) # No origin information should result in an empty map. test_fn_lines, _ = tf_inspect.getsourcelines(test_fn) source_map = origin_info.create_source_map(node, '\n'.join(test_fn_lines), test_fn) self.assertEmpty(source_map)
def testSetFilenameAndLineFromCallerUsesCallersStack(self): t_obj = traceable_stack.TraceableObject(17) # Do not separate placeholder from the set_filename_and_line_from_caller() # call one line below it as it is used to calculate the latter's line # number. placeholder = lambda x: x result = t_obj.set_filename_and_line_from_caller() expected_lineno = inspect.getsourcelines(placeholder)[1] + 1 self.assertEqual(expected_lineno, t_obj.lineno) self.assertEqual(_THIS_FILENAME, t_obj.filename) self.assertEqual(t_obj.SUCCESS, result)
def resolve(node, source, function=None): """Adds an origin information to node and its subnodes. This allows us to map the original source code line numbers to generated source code. Args: node: gast.AST node. Should be a gast.FunctionDef. This is the node we annotate with origin information. source: Text, the source code. Should satisfy relationship `node in iter_tree(gast.parse(source))`; otherwise the lineno will be unreliable. function: The original function. If it is None then only the line numbers and column offset will be set in the annotation, with the rest of the information being None. """ if function: _, function_lineno = tf_inspect.getsourcelines(function) function_filepath = tf_inspect.getsourcefile(function) else: function_lineno = None function_filepath = None # TODO(mdan): Pull this to a separate utility. code_reader = six.StringIO(source) comment_map = {} for token in tokenize.generate_tokens(code_reader.readline): tok_type, tok_string, loc, _, _ = token srow, _ = loc if tok_type == tokenize.COMMENT: comment_map[srow] = tok_string.strip()[1:].strip() source_lines = source.split('\n') for n in gast.walk(node): if not hasattr(n, 'lineno'): continue within_body_offset = n.lineno - node.lineno source_code_line = source_lines[n.lineno - 1] if function: source_lineno = function_lineno + within_body_offset function_name = function.__name__ else: source_lineno = n.lineno function_name = None location = Location(function_filepath, source_lineno, n.col_offset) origin = OriginInfo(location, function_name, source_code_line, comment_map.get(source_lineno)) anno.setanno(n, anno.Basic.ORIGIN, origin)
def testPushObjSetsFilenameAndLineInfoForCaller(self): t_stack = traceable_stack.TraceableStack() # We expect that the line number recorded for the 1-object will come from # the call to t_stack.push_obj(1). Do not separate the next two lines! placeholder_1 = lambda x: x t_stack.push_obj(1) # We expect that the line number recorded for the 2-object will come from # the call to call_push_obj() and _not_ the call to t_stack.push_obj(). def call_push_obj(obj): t_stack.push_obj(obj, offset=1) # Do not separate the next two lines! placeholder_2 = lambda x: x call_push_obj(2) expected_lineno_1 = inspect.getsourcelines(placeholder_1)[1] + 1 expected_lineno_2 = inspect.getsourcelines(placeholder_2)[1] + 1 t_obj_2, t_obj_1 = t_stack.peek_traceable_objs() self.assertEqual(expected_lineno_2, t_obj_2.lineno) self.assertEqual(expected_lineno_1, t_obj_1.lineno)
def test_rewriting_error(self): _, zero_div_lineno = tf_inspect.getsourcelines(zero_div) src_map = { errors.CodeLocation(file_path=__file__, line_number=zero_div_lineno + 1): None } with self.assertRaisesRegexp(tf_errors.InvalidArgumentError, 'Integer division by zero'): z = zero_div_caller() zero_div_caller.ag_source_map = src_map with errors.improved_errors(zero_div_caller): with self.test_session() as sess: sess.run(z)
def test_rewriting_error(self): _, zero_div_lineno = tf_inspect.getsourcelines(zero_div) src_map = { errors.CodeLocation( file_path=__file__, line_number=zero_div_lineno + 1): None } with self.assertRaisesRegexp(tf_errors.InvalidArgumentError, 'Integer division by zero'): z = zero_div_caller() zero_div_caller.ag_source_map = src_map with errors.improved_errors(zero_div_caller): with self.test_session() as sess: sess.run(z)
def testSetFilenameAndLineFromCallerRespectsOffset(self): def call_set_filename_and_line_from_caller(t_obj): # We expect to retrieve the line number from _our_ caller. return t_obj.set_filename_and_line_from_caller(offset=1) t_obj = traceable_stack.TraceableObject(None) # Do not separate placeholder from the # call_set_filename_and_line_from_caller() call one line below it as it is # used to calculate the latter's line number. placeholder = lambda x: x result = call_set_filename_and_line_from_caller(t_obj) expected_lineno = inspect.getsourcelines(placeholder)[1] + 1 self.assertEqual(expected_lineno, t_obj.lineno) self.assertEqual(t_obj.SUCCESS, result)
def resolve(nodes, source, function=None): """Adds an origin information to all nodes inside the body of function. Args: nodes: Union[ast.AST, Iterable[ast.AST, ...]] source: Text, the source code string for the function whose body nodes will be annotated. function: Callable, the function that will have all nodes inside of it annotation with an OriginInfo annotation with key anno.Basic.ORIGIN. If it is None then only the line numbers and column offset will be set in the annotation, with the rest of the information being None. Returns: A tuple of the AST node for function and a String containing its source code. """ if not isinstance(nodes, (list, tuple)): nodes = (nodes,) if function: _, function_lineno = tf_inspect.getsourcelines(function) function_filepath = tf_inspect.getsourcefile(function) else: function_lineno = None function_filepath = None source_lines = source.split('\n') for node in nodes: for n in gast.walk(node): if not hasattr(n, 'lineno'): continue lineno_in_body = n.lineno source_code_line = source_lines[lineno_in_body - 1] if function: source_lineno = function_lineno + lineno_in_body function_name = function.__name__ else: source_lineno = lineno_in_body function_name = None location = Location(function_filepath, source_lineno, n.col_offset) origin = OriginInfo(location, function_name, source_code_line) anno.setanno(n, anno.Basic.ORIGIN, origin)
def test_error_replacement(self): _, zero_div_lineno = tf_inspect.getsourcelines(zero_div) src_map = { errors.CodeLocation(file_path=__file__, line_number=zero_div_lineno + 1): self._fake_origin } with self.assertRaises(errors.TfRuntimeError) as cm: z = zero_div_caller() zero_div_caller.ag_source_map = src_map with errors.improved_errors(zero_div_caller): with self.test_session() as sess: sess.run(z) expected = cm.exception current_traceback = expected.custom_traceback for frame in current_traceback: self.assertNotEqual('zero_div', frame[2]) self.assertTrue( any(self._fake_origin.as_frame() == frame for frame in current_traceback))
def test_error_replacement(self): _, zero_div_lineno = tf_inspect.getsourcelines(zero_div) src_map = { errors.CodeLocation( file_path=__file__, line_number=zero_div_lineno + 1): self._fake_origin } with self.assertRaises(errors.TfRuntimeError) as cm: z = zero_div_caller() zero_div_caller.ag_source_map = src_map with errors.improved_errors(zero_div_caller): with self.test_session() as sess: sess.run(z) expected = cm.exception current_traceback = expected.custom_traceback for frame in current_traceback: self.assertNotEqual('zero_div', frame[2]) self.assertTrue( any(self._fake_origin.as_frame() == frame for frame in current_traceback))
def resolve(node, source, function=None): """Adds an origin information to all nodes inside the body of function. Args: node: The AST node for the function whose body nodes will be annotated. source: Text, the source code string for the function whose body nodes will be annotated. function: Callable, the function that will have all nodes inside of it annotation with an OriginInfo annotation with key anno.Basic.ORIGIN. If it is None then only the line numbers and column offset will be set in the annotation, with the rest of the information being None. Returns: A tuple of the AST node for function and a String containing its source code. """ if function: _, function_lineno = tf_inspect.getsourcelines(function) function_filepath = tf_inspect.getsourcefile(function) else: function_lineno = None function_filepath = None source_lines = source.split('\n') for n in gast.walk(node): if hasattr(n, 'lineno'): # n.lineno is relative to the start of the enclosing function, so need to # offset it by the line of the function. source_code_line = source_lines[n.lineno - 1] if function: source_lineno = n.lineno + function_lineno - 1 function_name = function.__name__ else: source_lineno = n.lineno function_name = None anno.setanno( n, anno.Basic.ORIGIN, OriginInfo(function_filepath, function_name, source_lineno, n.col_offset, source_code_line))
def testGetSourceLines(self): expected = inspect.getsourcelines( test_decorated_function_with_defaults.decorated_target) self.assertEqual( expected, tf_inspect.getsourcelines(test_decorated_function_with_defaults))