def parse_entity(entity, future_features):
"""Returns the AST and source code of given entity.
Args:
entity: Any, Python function/method/class
future_features: Iterable[Text], future features to use (e.g.
'print_statement'). See
https://docs.python.org/2/reference/simple_stmts.html#future
Returns:
gast.AST, Text: the parsed AST node; the source code that was parsed to
generate the AST (including any prefixes that this function may have added).
"""
if inspect_utils.islambda(entity):
return _parse_lambda(entity)
try:
original_source = inspect_utils.getimmediatesource(entity)
except (IOError, OSError) as e:
raise ValueError(
'Unable to locate the source code of {}. Note that functions defined'
' in certain environments, like the interactive Python shell, do not'
' expose their source code. If that is the case, you should define'
' them in a .py source file. If you are certain the code is'
' graph-compatible, wrap the call using'
' @tf.autograph.experimental.do_not_convert. Original error: {}'.format(
entity, e))
source = dedent_block(original_source)
future_statements = tuple(
'from __future__ import {}'.format(name) for name in future_features)
source = '\n'.join(future_statements + (source,))
return parse(source, preamble_len=len(future_features)), source
def test_getimmediatesource_normal_decorator_different_module(self):
expected = textwrap.dedent("""
def standalone_wrapper(*args, **kwargs):
return f(*args, **kwargs)
""")
@decorators.standalone_decorator
def test_fn(a):
"""Test docstring."""
return [a]
self.assertSourceIdentical(inspect_utils.getimmediatesource(test_fn),
expected)
def test_getimmediatesource_normal_decorator_different_module(self):
expected = textwrap.dedent("""
def standalone_wrapper(*args, **kwargs):
return f(*args, **kwargs)
""")
@decorators.standalone_decorator
def test_fn(a):
"""Test docstring."""
return [a]
self.assertSourceIdentical(
inspect_utils.getimmediatesource(test_fn), expected)
def test_getimmediatesource_functools_wrapper_different_module(self):
expected = textwrap.dedent("""
@functools.wraps(f)
def wrapper(*args, **kwargs):
return f(*args, **kwargs)
""")
@decorators.wrapping_decorator
def test_fn(a):
"""Test docstring."""
return [a]
self.assertSourceIdentical(inspect_utils.getimmediatesource(test_fn),
expected)
def test_getimmediatesource_functools_wrapper_different_module(self):
expected = textwrap.dedent("""
@functools.wraps(f)
def wrapper(*args, **kwargs):
return f(*args, **kwargs)
""")
@decorators.wrapping_decorator
def test_fn(a):
"""Test docstring."""
return [a]
self.assertSourceIdentical(
inspect_utils.getimmediatesource(test_fn), expected)
def test_getimmediatesource_noop_decorator(self):
def test_decorator(f):
return f
expected = '''
@test_decorator
def test_fn(a):
"""Test docstring."""
return [a]
'''
@test_decorator
def test_fn(a):
"""Test docstring."""
return [a]
self.assertSourceIdentical(inspect_utils.getimmediatesource(test_fn),
expected)
def test_getimmediatesource_basic(self):
def test_decorator(f):
def f_wrapper(*args, **kwargs):
return f(*args, **kwargs)
return f_wrapper
expected = """
def f_wrapper(*args, **kwargs):
return f(*args, **kwargs)
"""
@test_decorator
def test_fn(a):
"""Test docstring."""
return [a]
self.assertSourceIdentical(inspect_utils.getimmediatesource(test_fn),
expected)
def test_getimmediatesource_noop_decorator(self):
def test_decorator(f):
return f
expected = '''
@test_decorator
def test_fn(a):
"""Test docstring."""
return [a]
'''
@test_decorator
def test_fn(a):
"""Test docstring."""
return [a]
self.assertSourceIdentical(
inspect_utils.getimmediatesource(test_fn), expected)
def test_getimmediatesource_basic(self):
def test_decorator(f):
def f_wrapper(*args, **kwargs):
return f(*args, **kwargs)
return f_wrapper
expected = """
def f_wrapper(*args, **kwargs):
return f(*args, **kwargs)
"""
@test_decorator
def test_fn(a):
"""Test docstring."""
return [a]
self.assertSourceIdentical(
inspect_utils.getimmediatesource(test_fn), expected)
def parse_entity(entity, future_features):
"""Returns the AST and source code of given entity.
Args:
entity: Any, Python function/method/class
future_features: Iterable[Text], future features to use (e.g.
'print_statement'). See
https://docs.python.org/2/reference/simple_stmts.html#future
Returns:
gast.AST, Text: the parsed AST node; the source code that was parsed to
generate the AST (including any prefixes that this function may have added).
"""
try:
original_source = inspect_utils.getimmediatesource(entity)
except (IOError, OSError) as e:
raise ValueError(
'Unable to locate the source code of {}. Note that functions defined'
' in certain environments, like the interactive Python shell do not'
' expose their source code. If that is the case, you should to define'
' them in a .py source file. If you are certain the code is'
' graph-compatible, wrap the call using'
' @tf.autograph.do_not_convert. Original error: {}'.format(
entity, e))
def raise_parse_failure(comment):
raise ValueError(
'Failed to parse source code of {}, which Python reported as:\n{}\n'
'{}'.format(entity, original_source, comment))
# Comments and multiline strings can appear at arbitrary indentation levels,
# causing textwrap.dedent to not correctly dedent source code.
# TODO(b/115884650): Automatic handling of comments/multiline strings.
source = textwrap.dedent(original_source)
future_statements = tuple('from __future__ import {}'.format(name)
for name in future_features)
source = '\n'.join(future_statements + (source, ))
try:
return parse_str(source, preamble_len=len(future_features)), source
except IndentationError:
# The text below lists the causes of this error known to us. There may
# be more.
raise_parse_failure(
'This may be caused by multiline strings or comments not indented at'
' the same level as the code.')
except SyntaxError as e:
if not tf_inspect.isfunction(entity) or entity.__name__ != '<lambda>':
raise
# Certain entities, like lambdas, only hold the raw code lines which defined
# them, which may include surrounding tokens and may be syntactically
# invalid out of context. For example:
#
# l = (
# lambda x: x,)[0]
#
# will have the dedented source "lambda x: x,)[0]"
# Here we make an attempt to stip away the garbage by looking at the
# information in the syntax error.
lines = source.split('\n')
lineno, offset = e.lineno, e.offset # 1-based
# Give up if there's nothing we can chip away.
if len(lines) == lineno and len(lines[-1]) == offset:
raise_parse_failure(
'If this is a lambda function, the error may be avoided by creating'
' the lambda in a standalone statement.')
# Drop all lines following the error location
# TODO(mdan): What's with the pylint errors?
lines = lines[:lineno] # pylint:disable=invalid-slice-index
# Drop all characters following the error location
lines[-1] = lines[-1][:offset - 1] # pylint:disable=invalid-slice-index
source = '\n'.join(lines)
try:
return parse_str(source, preamble_len=len(future_features)), source
except SyntaxError as e:
raise_parse_failure(
'If this is a lambda function, the error may be avoided by creating'
' the lambda in a standalone statement.')
def parse_entity(entity, future_features):
"""Returns the AST and source code of given entity.
Args:
entity: Any, Python function/method/class
future_features: Iterable[Text], future features to use (e.g.
'print_statement'). See
https://docs.python.org/2/reference/simple_stmts.html#future
Returns:
gast.AST, Text: the parsed AST node; the source code that was parsed to
generate the AST (including any prefixes that this function may have added).
"""
try:
source = inspect_utils.getimmediatesource(entity)
except (IOError, OSError) as e:
raise ValueError(
'Unable to locate the source code of {}. Note that functions defined'
' in certain environments, like the interactive Python shell do not'
' expose their source code. If that is the case, you should to define'
' them in a .py source file. If you are certain the code is'
' graph-compatible, wrap the call using'
' @tf.autograph.do_not_convert. Original error: {}'.format(entity, e))
def raise_parse_failure(comment):
raise ValueError(
'Failed to parse source code of {}, which Python reported as:\n{}\n'
'{}'.format(entity, source, comment))
# Comments and multiline strings can appear at arbitrary indentation levels,
# causing textwrap.dedent to not correctly dedent source code.
# TODO(b/115884650): Automatic handling of comments/multiline strings.
source = textwrap.dedent(source)
future_statements = tuple(
'from __future__ import {}'.format(name) for name in future_features)
source = '\n'.join(future_statements + (source,))
try:
return parse_str(source, preamble_len=len(future_features)), source
except IndentationError:
# The text below lists the causes of this error known to us. There may
# be more.
raise_parse_failure(
'This may be caused by multiline strings or comments not indented at'
' the same level as the code.')
except SyntaxError as e:
if not tf_inspect.isfunction(entity) or entity.__name__ != '<lambda>':
raise
# Certain entities, like lambdas, only hold the raw code lines which defined
# them, which may include surrounding tokens and may be syntactically
# invalid out of context. For example:
#
# l = (
# lambda x: x,)[0]
#
# will have the dedented source "lambda x: x,)[0]"
# Here we make an attempt to stip away the garbage by looking at the
# information in the syntax error.
lines = source.split('\n')
lineno, offset = e.lineno, e.offset # 1-based
# Give up if there's nothing we can chip away.
if len(lines) == lineno and len(lines[-1]) == offset:
raise_parse_failure(
'If this is a lambda function, the error may be avoided by creating'
' the lambda in a standalone statement.')
# Drop all lines following the error location
# TODO(mdan): What's with the pylint errors?
lines = lines[:lineno] # pylint:disable=invalid-slice-index
# Drop all characters following the error location
lines[-1] = lines[-1][:offset - 1] # pylint:disable=invalid-slice-index
source = '\n'.join(lines)
try:
return parse_str(source, preamble_len=len(future_features)), source
except SyntaxError as e:
raise_parse_failure(
'If this is a lambda function, the error may be avoided by creating'
' the lambda in a standalone statement. Tried to strip down the'
' source to:\n{}\nBut that did not work.'.format(source))
