def compiled(self, node, *symbols):
source = None
self.dynamic_calls = []
def converted_call(*args):
"""Mock version of api.converted_call."""
self.dynamic_calls.append(args)
return 7
try:
result, source = compiler.ast_to_object(node)
result.tf = self.make_fake_mod('fake_tf', *symbols)
fake_ag = self.make_fake_mod('fake_ag', converted_call)
fake_ag.__dict__.update(operators.__dict__)
fake_ag.__dict__['utils'] = utils
fake_ag.__dict__['rewrite_graph_construction_error'] = (
errors.rewrite_graph_construction_error)
result.__dict__['ag__'] = fake_ag
yield result
except Exception: # pylint:disable=broad-except
if source is None:
print('Offending AST:\n%s' %
pretty_printer.fmt(node, color=False))
else:
print('Offending compiled code:\n%s' % source)
raise
def visit(self, node):
source_code = self.context.source_code
source_file = self.context.source_file
did_enter_function = False
try:
if isinstance(node, (gast.FunctionDef, gast.ClassDef, gast.Lambda)):
self._enclosing_entities.append(node)
did_enter_function = True
if source_code and hasattr(node, 'lineno'):
self._lineno = node.lineno
self._col_offset = node.col_offset
if anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
return node
return super(Base, self).visit(node)
except (ValueError, AttributeError, KeyError, NotImplementedError,
AssertionError) as e:
msg = '%s: %s\nOffending source:\n%s\n\nOccurred at node:\n%s' % (
e.__class__.__name__, str(e), try_ast_to_source(node),
pretty_printer.fmt(node, color=False))
if source_code:
line = source_code.splitlines()[self._lineno - 1]
else:
line = '<no source available>'
six.reraise(AutographParseError,
AutographParseError(
msg,
(source_file, self._lineno, self._col_offset + 1, line)),
sys.exc_info()[2])
finally:
if did_enter_function:
self._enclosing_entities.pop()
def compiled(self, node, *symbols):
source = None
self.dynamic_calls = []
def converted_call(*args):
"""Mock version of api.converted_call."""
self.dynamic_calls.append(args)
return 7
try:
result, source = compiler.ast_to_object(node)
result.tf = self.make_fake_mod('fake_tf', *symbols)
result.autograph_utils = utils
result.autograph_api = self.make_fake_mod('fake_api',
converted_call)
result.__dict__['__ops'] = operators
yield result
except Exception: # pylint:disable=broad-except
if source is None:
print('Offending AST:\n%s' %
pretty_printer.fmt(node, color=False))
else:
print('Offending compiled code:\n%s' % source)
raise
def compiled(self, node, namespace, *symbols):
source = None
self.dynamic_calls = []
def converted_call(*args):
"""Mock version of api.converted_call."""
self.dynamic_calls.append(args)
return 7
try:
result, source = compiler.ast_to_object(node)
result.tf = self.make_fake_mod('fake_tf', *symbols)
fake_ag = self.make_fake_mod('fake_ag', converted_call)
fake_ag.__dict__.update(operators.__dict__)
fake_ag.__dict__['utils'] = utils
fake_ag.__dict__['rewrite_graph_construction_error'] = (
errors.rewrite_graph_construction_error)
result.__dict__['ag__'] = fake_ag
for k, v in namespace.items():
result.__dict__[k] = v
yield result
except Exception: # pylint:disable=broad-except
if source is None:
print('Offending AST:\n%s' % pretty_printer.fmt(node, color=False))
else:
print('Offending compiled code:\n%s' % source)
raise
def visit(self, node):
source_code = self.context.source_code
source_file = self.context.source_file
try:
if source_code and hasattr(node, 'lineno'):
self._lineno = node.lineno
self._col_offset = node.col_offset
if anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
return node
return super(Base, self).visit(node)
except (ValueError, AttributeError, KeyError, NotImplementedError,
AssertionError) as e:
msg = '%s: %s\nOffending source:\n%s\n\nOccurred at node:\n%s' % (
e.__class__.__name__, str(e), try_ast_to_source(node),
pretty_printer.fmt(node, color=False))
if source_code:
line = source_code.splitlines()[self._lineno - 1]
else:
line = '<no source available>'
six.reraise(
AutographParseError,
AutographParseError(
msg,
(source_file, self._lineno, self._col_offset + 1, line)),
sys.exc_info()[2])
def visit(self, node):
source_code = self.context.source_code
source_file = self.context.source_file
did_enter_function = False
local_scope_size_at_entry = len(self._local_scope_state)
try:
if isinstance(node,
(gast.FunctionDef, gast.ClassDef, gast.Lambda)):
did_enter_function = True
if did_enter_function:
self._enclosing_entities.append(node)
if source_code and hasattr(node, 'lineno'):
self._lineno = node.lineno
self._col_offset = node.col_offset
if not anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
result = super(Base, self).visit(node)
# On exception, the local scope integrity is not guaranteed.
if did_enter_function:
self._enclosing_entities.pop()
if local_scope_size_at_entry != len(self._local_scope_state):
raise AssertionError(
'Inconsistent local scope stack. Before entering node %s, the'
' stack had length %d, after exit it has length %d. This'
' indicates enter_local_scope and exit_local_scope are not'
' well paired.' % (node, local_scope_size_at_entry,
len(self._local_scope_state)))
return result
except (ValueError, AttributeError, KeyError,
NotImplementedError) as e:
msg = '%s: %s\nOffending source:\n%s\n\nOccurred at node:\n%s' % (
e.__class__.__name__, str(e), try_ast_to_source(node),
pretty_printer.fmt(node, color=False))
if source_code:
line = source_code.splitlines()[self._lineno - 1]
else:
line = '<no source available>'
# TODO(mdan): Avoid the printing of the original exception.
# In other words, we need to find how to suppress the "During handling
# of the above exception, another exception occurred" message.
six.reraise(
AutographParseError,
AutographParseError(
msg,
(source_file, self._lineno, self._col_offset + 1, line)),
sys.exc_info()[2])
def visit(self, node):
source_code = self.entity_info.source_code
source_file = self.entity_info.source_file
did_enter_function = False
local_scope_size_at_entry = len(self._local_scope_state)
try:
if isinstance(node, (gast.FunctionDef, gast.ClassDef, gast.Lambda)):
did_enter_function = True
if did_enter_function:
self._enclosing_entities.append(node)
if source_code and hasattr(node, 'lineno'):
self._lineno = node.lineno
self._col_offset = node.col_offset
if not anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
result = super(Base, self).visit(node)
# On exception, the local scope integrity is not guaranteed.
if did_enter_function:
self._enclosing_entities.pop()
if local_scope_size_at_entry != len(self._local_scope_state):
raise AssertionError(
'Inconsistent local scope stack. Before entering node %s, the'
' stack had length %d, after exit it has length %d. This'
' indicates enter_local_scope and exit_local_scope are not'
' well paired.' % (
node,
local_scope_size_at_entry,
len(self._local_scope_state)
))
return result
except (ValueError, AttributeError, KeyError, NotImplementedError) as e:
msg = '%s: %s\nOffending source:\n%s\n\nOccurred at node:\n%s' % (
e.__class__.__name__, str(e), self._get_source(node),
pretty_printer.fmt(node, color=False))
if source_code:
line = source_code.splitlines()[self._lineno - 1]
else:
line = '<no source available>'
# TODO(mdan): Avoid the printing of the original exception.
# In other words, we need to find how to suppress the "During handling
# of the above exception, another exception occurred" message.
six.reraise(AutographParseError,
AutographParseError(
msg,
(source_file, self._lineno, self._col_offset + 1, line)),
sys.exc_info()[2])
def visit(self, node):
source_code = self.context.source_code
source_file = self.context.source_file
did_enter_function = False
local_scope_state_size = len(self._local_scope_state)
try:
if isinstance(node,
(gast.FunctionDef, gast.ClassDef, gast.Lambda)):
self._enclosing_entities.append(node)
did_enter_function = True
if source_code and hasattr(node, 'lineno'):
self._lineno = node.lineno
self._col_offset = node.col_offset
if anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
return node
return super(Base, self).visit(node)
except (ValueError, AttributeError, KeyError, NotImplementedError,
AssertionError) as e:
msg = '%s: %s\nOffending source:\n%s\n\nOccurred at node:\n%s' % (
e.__class__.__name__, str(e), try_ast_to_source(node),
pretty_printer.fmt(node, color=False))
if source_code:
line = source_code.splitlines()[self._lineno - 1]
else:
line = '<no source available>'
six.reraise(
AutographParseError,
AutographParseError(
msg,
(source_file, self._lineno, self._col_offset + 1, line)),
sys.exc_info()[2])
finally:
if did_enter_function:
self._enclosing_entities.pop()
if local_scope_state_size != len(self._local_scope_state):
raise AssertionError(
'Inconsistent local scope stack. Before entering node %s, the'
' stack had length %d, after exit it has length %d. This'
' indicates enter_local_scope and exit_local_scope are not'
' well paired.')
def visit_FunctionDef(self, node):
self.generic_visit(node)
kept_decorators = []
for dec in node.decorator_list:
if isinstance(dec, gast.Call):
dec_func = dec.func
else:
dec_func = dec
# Special cases.
# TODO (mdan): Is there any way we can treat these more generically? id:956
# https://github.com/imdone/tensorflow/issues/957
# We may want to forego using decorators altogether if we can't
# properly support them.
if isinstance(dec_func,
gast.Name) and dec_func.id in ('classmethod', ):
# Assumption: decorators are only visible in the AST when converting
# a function inline (via another decorator).
# In that case, the converted function is no longer part of the
# original object that it was declared into.
# This is currently verified by tests.
continue
if not anno.hasanno(dec_func, 'live_val'):
raise ValueError('Could not resolve decorator: %s' %
pretty_printer.fmt(dec_func))
dec_value = anno.getanno(dec_func, 'live_val')
if dec_value not in self.remove_decorators:
kept_decorators.append((dec, dec_value))
for _, dec_value in kept_decorators:
if dec_value.__module__ == '__main__':
raise ValueError(
'decorator "%s" was not allowed because it is declared '
'in the module "%s". To fix this, declare it in a separate '
'module that we can import it from.' %
(dec_value, dec_value.__module__))
else:
self.additional_dependencies.add(dec_value)
node.decorator_list = [dec for dec, _ in kept_decorators]
return node
def visit(self, node):
source_code = self.context.source_code
source_file = self.context.source_file
did_enter_function = False
local_scope_state_size = len(self._local_scope_state)
try:
if isinstance(node, (gast.FunctionDef, gast.ClassDef, gast.Lambda)):
self._enclosing_entities.append(node)
did_enter_function = True
if source_code and hasattr(node, 'lineno'):
self._lineno = node.lineno
self._col_offset = node.col_offset
if anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
return node
return super(Base, self).visit(node)
except (ValueError, AttributeError, KeyError, NotImplementedError,
AssertionError) as e:
msg = '%s: %s\nOffending source:\n%s\n\nOccurred at node:\n%s' % (
e.__class__.__name__, str(e), try_ast_to_source(node),
pretty_printer.fmt(node, color=False))
if source_code:
line = source_code.splitlines()[self._lineno - 1]
else:
line = '<no source available>'
six.reraise(AutographParseError,
AutographParseError(
msg,
(source_file, self._lineno, self._col_offset + 1, line)),
sys.exc_info()[2])
finally:
if did_enter_function:
self._enclosing_entities.pop()
if local_scope_state_size != len(self._local_scope_state):
raise AssertionError(
'Inconsistent local scope stack. Before entering node %s, the'
' stack had length %d, after exit it has length %d. This'
' indicates enter_local_scope and exit_local_scope are not'
' well paired.')
def test_format(self):
node = ast.FunctionDef(
name='f',
args=ast.arguments(
args=[ast.Name(id='a', ctx=ast.Param())],
vararg=None,
kwarg=None,
defaults=[]),
body=[
ast.Return(
ast.BinOp(
op=ast.Add(),
left=ast.Name(id='a', ctx=ast.Load()),
right=ast.Num(1)))
],
decorator_list=[],
returns=None)
# Just checking for functionality, the color control characters make it
# difficult to inspect the result.
self.assertIsNotNone(pretty_printer.fmt(node))
def visit_FunctionDef(self, node):
self.generic_visit(node)
kept_decorators = []
for dec in node.decorator_list:
if isinstance(dec, gast.Call):
dec_func = dec.func
else:
dec_func = dec
# Special cases.
# TODO(mdan): Is there any way we can treat these more generically?
# We may want to forego using decorators altogether if we can't
# properly support them.
if isinstance(dec_func, gast.Name) and dec_func.id in ('classmethod',):
# Assumption: decorators are only visible in the AST when converting
# a function inline (via another decorator).
# In that case, the converted function is no longer part of the
# original object that it was declared into.
# This is currently verified by tests.
continue
if not anno.hasanno(dec_func, 'live_val'):
raise ValueError(
'Could not resolve decorator: %s' % pretty_printer.fmt(dec_func))
dec_value = anno.getanno(dec_func, 'live_val')
if dec_value not in self.remove_decorators:
kept_decorators.append((dec, dec_value))
for _, dec_value in kept_decorators:
if dec_value.__module__ == '__main__':
raise ValueError(
'decorator "%s" was not allowed because it is declared '
'in the module "%s". To fix this, declare it in a separate '
'module that we can import it from.' % (dec_value,
dec_value.__module__))
else:
self.additional_dependencies.add(dec_value)
node.decorator_list = [dec for dec, _ in kept_decorators]
return node
def compiled(self, node, *symbols):
source = None
self.dynamic_calls = []
def converted_call(*args):
"""Mock version of api.converted_call."""
self.dynamic_calls.append(args)
return 7
try:
result, source = compiler.ast_to_object(node)
result.tf = self.make_fake_mod('fake_tf', *symbols)
result.autograph_utils = utils
result.autograph_api = self.make_fake_mod('fake_api', converted_call)
result.__dict__['__ops'] = operators
yield result
except Exception: # pylint:disable=broad-except
if source is None:
print('Offending AST:\n%s' % pretty_printer.fmt(node, color=False))
else:
print('Offending compiled code:\n%s' % source)
raise
def debug_print(self, node):
"""Helper method useful for debugging."""
if __debug__:
print(pretty_printer.fmt(node))
return node
def visit(self, node):
if not isinstance(node, gast.AST):
# This is not that uncommon a mistake: various node bodies are lists, for
# example, posing a land mine for transformers that need to recursively
# call `visit`. The error needs to be raised before the exception handler
# below is installed, because said handler will mess up if `node` is not,
# in fact, a node.
msg = (
'invalid value for "node": expected "ast.AST", got "{}"; to'
' visit lists of nodes, use "visit_block" instead').format(type(node))
raise ValueError(msg)
source_code = self.entity_info.source_code
source_file = self.entity_info.source_file
did_enter_function = False
local_scope_size_at_entry = len(self._local_scope_state)
processing_expr_node = False
try:
if isinstance(node, (gast.FunctionDef, gast.ClassDef, gast.Lambda)):
did_enter_function = True
elif isinstance(node, gast.Expr):
processing_expr_node = True
if did_enter_function:
self._enclosing_entities.append(node)
if source_code and hasattr(node, 'lineno'):
self._lineno = node.lineno
self._col_offset = node.col_offset
if processing_expr_node:
entry_expr_value = node.value
if not anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
result = super(Base, self).visit(node)
# Adjust for consistency: replacing the value of an Expr with
# an Assign node removes the need for the Expr node.
if processing_expr_node:
if isinstance(result, gast.Expr) and result.value != entry_expr_value:
# When the replacement is a list, it is assumed that the list came
# from a template that contained a number of statements, which
# themselves are standalone and don't require an enclosing Expr.
if isinstance(result.value,
(list, tuple, gast.Assign, gast.AugAssign)):
result = result.value
# On exception, the local scope integrity is not guaranteed.
if did_enter_function:
self._enclosing_entities.pop()
if local_scope_size_at_entry != len(self._local_scope_state):
raise AssertionError(
'Inconsistent local scope stack. Before entering node %s, the'
' stack had length %d, after exit it has length %d. This'
' indicates enter_local_scope and exit_local_scope are not'
' well paired.' % (
node,
local_scope_size_at_entry,
len(self._local_scope_state)
))
return result
except (ValueError, AttributeError, KeyError, NotImplementedError) as e:
msg = '%s: %s\nOffending source:\n%s\n\nOccurred at node:\n%s' % (
e.__class__.__name__, str(e), self._get_source(node),
pretty_printer.fmt(node, color=False))
if source_code:
line = source_code.splitlines()[self._lineno - 1]
else:
line = '<no source available>'
# TODO(mdan): Avoid the printing of the original exception.
# In other words, we need to find how to suppress the "During handling
# of the above exception, another exception occurred" message.
six.reraise(AutographParseError,
AutographParseError(
msg,
(source_file, self._lineno, self._col_offset + 1, line)),
sys.exc_info()[2])
def visit(self, node):
if not isinstance(node, gast.AST):
# This is not that uncommon a mistake: various node bodies are lists, for
# example, posing a land mine for transformers that need to recursively
# call `visit`. The error needs to be raised before the exception handler
# below is installed, because said handler will mess up if `node` is not,
# in fact, a node.
msg = ('invalid value for "node": expected "ast.AST", got "{}"; to'
' visit lists of nodes, use "visit_block" instead').format(
type(node))
raise ValueError(msg)
source_code = self.entity_info.source_code
source_file = self.entity_info.source_file
did_enter_function = False
local_scope_size_at_entry = len(self._local_scope_state)
processing_expr_node = False
try:
if isinstance(node,
(gast.FunctionDef, gast.ClassDef, gast.Lambda)):
did_enter_function = True
elif isinstance(node, gast.Expr):
processing_expr_node = True
if did_enter_function:
self._enclosing_entities.append(node)
if source_code and hasattr(node, 'lineno'):
self._lineno = node.lineno
self._col_offset = node.col_offset
if processing_expr_node:
entry_expr_value = node.value
if not anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
result = super(Base, self).visit(node)
# Adjust for consistency: replacing the value of an Expr with
# an Assign node removes the need for the Expr node.
if processing_expr_node:
if isinstance(result,
gast.Expr) and result.value != entry_expr_value:
# When the replacement is a list, it is assumed that the list came
# from a template that contained a number of statements, which
# themselves are standalone and don't require an enclosing Expr.
if isinstance(result.value,
(list, tuple, gast.Assign, gast.AugAssign)):
result = result.value
# On exception, the local scope integrity is not guaranteed.
if did_enter_function:
self._enclosing_entities.pop()
if local_scope_size_at_entry != len(self._local_scope_state):
raise AssertionError(
'Inconsistent local scope stack. Before entering node %s, the'
' stack had length %d, after exit it has length %d. This'
' indicates enter_local_scope and exit_local_scope are not'
' well paired.' % (node, local_scope_size_at_entry,
len(self._local_scope_state)))
return result
except (ValueError, AttributeError, KeyError,
NotImplementedError) as e:
msg = '%s: %s\nOffending source:\n%s\n\nOccurred at node:\n%s' % (
e.__class__.__name__, str(e), self._get_source(node),
pretty_printer.fmt(node, color=False))
if source_code:
line = source_code.splitlines()[self._lineno - 1]
else:
line = '<no source available>'
# TODO(mdan): Avoid the printing of the original exception.
# In other words, we need to find how to suppress the "During handling
# of the above exception, another exception occurred" message.
six.reraise(
AutographParseError,
AutographParseError(
msg,
(source_file, self._lineno, self._col_offset + 1, line)),
sys.exc_info()[2])
