def test_function_decorator(self):
def function_decorator():
def decorator(f):
return lambda a: f(a) + 1
return decorator
# The Python parser does capture decorators into the AST.
# However, the interpreter desugars them on load, and refering to the
# decorated function at runtime usually loses any trace of the decorator.
# Below is an example when that doesn't happen.
def static_wrapper():
@function_decorator()
def test_fn(a): # pylint:disable=unused-variable
return a
node = self.parse_and_analyze(static_wrapper,
{'function_decorator': function_decorator})
node = node.body[0].body[0]
node = decorators.transform(node, remove_decorators=())
# Since the decorator is not removed, we need to include its source
# code. We cannot do it after the fact because decorators are executed
# on load.
result, _ = compiler.ast_to_object(
node,
source_prefix=textwrap.dedent(tf_inspect.getsource(function_decorator)))
self.assertEqual(2, result.test_fn(1))
node = decorators.transform(node, remove_decorators=(function_decorator,))
with self.compiled(node) as result:
self.assertEqual(1, result.test_fn(1))
def test_simple_decorator(self):
def simple_decorator(f):
return lambda a: f(a) + 1
# The Python parser does capture decorators into the AST.
# However, the interpreter desugars them upon load, and refering to the
# decorated function at runtime usually loses any trace of the decorator.
# Below is an example when that doesn't happen.
def static_wrapper():
@simple_decorator
def test_fn(a): # pylint:disable=unused-variable
return a
node = self.parse_and_analyze(static_wrapper,
{'simple_decorator': simple_decorator})
node = node.body[0].body[0]
node = decorators.transform(node, remove_decorators=())
result = compiler.ast_to_object(
node,
source_prefix=textwrap.dedent(tf_inspect.getsource(simple_decorator)))
self.assertEqual(2, result.test_fn(1))
node = decorators.transform(node, remove_decorators=(simple_decorator,))
result = compiler.ast_to_object(node)
self.assertEqual(1, result.test_fn(1))
def test_simple_decorator(self):
def simple_decorator(f):
return lambda a: f(a) + 1
# The Python parser does capture decorators into the AST.
# However, the interpreter desugars them upon load, and refering to the
# decorated function at runtime usually loses any trace of the decorator.
# Below is an example when that doesn't happen.
def static_wrapper():
@simple_decorator
def test_fn(a): # pylint:disable=unused-variable
return a
node = self.parse_and_analyze(static_wrapper,
{'simple_decorator': simple_decorator})
node = node.body[0].body[0]
node = decorators.transform(node, remove_decorators=())
# Since the decorator is not removed, we need to include its source
# code. We cannot do it after the fact because decorators are executed
# on load.
result, _ = compiler.ast_to_object(
node,
source_prefix=textwrap.dedent(
tf_inspect.getsource(simple_decorator)))
self.assertEqual(2, result.test_fn(1))
node = decorators.transform(node,
remove_decorators=(simple_decorator, ))
with self.compiled(node) as result:
self.assertEqual(1, result.test_fn(1))
def test_function_decorator(self):
def function_decorator():
def decorator(f):
return lambda a: f(a) + 1
return decorator
# The Python parser does capture decorators into the AST.
# However, the interpreter desugars them on load, and refering to the
# decorated function at runtime usually loses any trace of the decorator.
# Below is an example when that doesn't happen.
def static_wrapper():
@function_decorator()
def test_fn(a): # pylint:disable=unused-variable
return a
node = self.parse_and_analyze(
static_wrapper, {'function_decorator': function_decorator})
node = node.body[0].body[0]
node = decorators.transform(node, remove_decorators=())
result = compiler.ast_to_object(
node,
source_prefix=textwrap.dedent(
tf_inspect.getsource(function_decorator)))
self.assertEqual(2, result.test_fn(1))
node = decorators.transform(node,
remove_decorators=(function_decorator, ))
result = compiler.ast_to_object(node)
self.assertEqual(1, result.test_fn(1))
def node_to_graph(node, ctx, nocompile_decorators):
"""Convert Python code to equivalent TF graph mode code.
Args:
node: A Python AST node representing the code to convert.
ctx: An EntityContext object.
nocompile_decorators: A tuple containing decorators to be stripped from
functions during conversion.
Returns:
A tuple (node, deps):
* node: A Python ast node, representing the converted code.
* deps: A set of strings, the fully qualified names of entity
dependencies that this node has.
"""
# TODO(mdan): Verify arguments for correctness.
# TODO(mdan): Factor out common elements.
# These include:
# * code move between blocks
# * visiting blocks in transformers
# Certain steps, especially canonicalization, insert new symbols into the
# tree, which must be accounted. Although less efficient, it is most robust
# to re-run the analysis.
node = _static_analysis_pass(node, ctx)
# Past this point, line numbers are no longer accurate so we ignore the
# source.
# TODO(mdan): Is it feasible to reconstruct intermediate source code?
ctx.source_code = None
node = decorators.transform(node, nocompile_decorators)
node = break_statements.transform(node, ctx)
node = asserts.transform(node, ctx)
# Note: sequencing continue canonicalization before for loop one avoids
# dealing with the extra loop increment operation that the for
# canonicalization creates.
node = continue_statements.transform(node, ctx)
ctx.namespace['len'] = len
node = _static_analysis_pass(node, ctx)
node = for_loops.transform(node, ctx)
# for_loops may insert new global references.
node = builtin_functions.transform(node, ctx)
# TODO(mdan): Kept for CL consistency. Remove.
# builtin_functions may insert new global references.
ctx.namespace['print'] = print
node = _static_analysis_pass(node, ctx)
node = call_trees.transform(node, ctx, config.DEFAULT_UNCOMPILED_MODULES,
nocompile_decorators)
node = control_flow.transform(node, ctx)
# control_flow may create new symbols and change scopes.
node = _static_analysis_pass(node, ctx)
node = logical_expressions.transform(node)
node = side_effect_guards.transform(node, ctx)
return node
def node_to_graph(node, ctx, nocompile_decorators):
"""Convert Python code to equivalent TF graph mode code.
Args:
node: A Python AST node representing the code to convert.
ctx: An EntityContext object.
nocompile_decorators: A tuple containing decorators to be stripped from
functions during conversion.
Returns:
A tuple (node, deps):
* node: A Python ast node, representing the converted code.
* deps: A set of strings, the fully qualified names of entity
dependencies that this node has.
"""
# TODO(mdan): Verify arguments for correctness.
# TODO(mdan): Factor out common elements.
# These include:
# * code move between blocks
# * visiting blocks in transformers
# Certain steps, especially canonicalization, insert new symbols into the
# tree, which must be accounted. Although less efficient, it is most robust
# to re-run the analysis.
node = _static_analysis_pass(node, ctx)
# Past this point, line numbers are no longer accurate so we ignore the
# source.
# TODO(mdan): Is it feasible to reconstruct intermediate source code?
ctx.source_code = None
node = decorators.transform(node, nocompile_decorators)
node = break_canonicalization.transform(node, ctx)
node = asserts.transform(node, ctx)
# Note: sequencing continue canonicalization before for loop one avoids
# dealing with the extra loop increment operation that the for
# canonicalization creates.
node = continue_canonicalization.transform(node, ctx)
ctx.namespace['len'] = len
node = _static_analysis_pass(node, ctx)
node = for_canonicalization.transform(node, ctx)
# for_canonicalization may insert new global references.
node = builtin_functions.transform(node, ctx)
# builtin_functions may insert new global references.
ctx.namespace['print'] = print
node = _static_analysis_pass(node, ctx)
node = call_trees.transform(node, ctx, config.DEFAULT_UNCOMPILED_MODULES,
nocompile_decorators)
node = control_flow.transform(node, ctx)
# control_flow may create new symbols and change scopes.
node = _static_analysis_pass(node, ctx)
node = logical_expressions.transform(node)
node = side_effect_guards.transform(node, ctx)
return node
def _remover_wrapper(self, f, remove_decorators):
namespace = {
'self_removing_decorator': self_removing_decorator,
'simple_decorator': simple_decorator
}
node = self.parse_and_analyze(f, namespace)
node, _ = decorators.transform(node, remove_decorators=remove_decorators)
result, _ = compiler.ast_to_object(node)
return getattr(result, f.__name__)
def node_to_graph(node, ctx, nocompile_decorators):
"""Convert Python code to equivalent TF graph mode code.
Args:
node: A Python AST node representing the code to convert.
ctx: An EntityContext object.
nocompile_decorators: A tuple containing decorators to be stripped from
functions during conversion.
Returns:
A tuple (node, deps):
* node: A Python ast node, representing the converted code.
* deps: A set of strings, the fully qualified names of entity
dependencies that this node has.
"""
# TODO(mdan): Verify arguments for correctness.
# TODO(mdan): Factor out common elements.
# These include:
# * keeping track of symbols that have been created
# * marking nodes (e.g. py_func wrappers) to suppress further processing
# * code move between blocks
# * insertion of new global references
# * visiting blocks in transformers
# Certain steps, especially canonicalization, insert new symbols into the
# tree, which must be accounted. Although less efficient, it is most robust
# to re-run the analysis.
node = _static_analysis_pass(node, ctx)
node = decorators.transform(node, nocompile_decorators)
node = break_canonicalization.transform(node, ctx.namer)
# Note: sequencing continue canonicalization before for loop one avoids
# dealing with the extra loop increment operation that the for
# canonicalization creates.
node = continue_canonicalization.transform(node, ctx.namer)
ctx.namespace['len'] = len
node = _static_analysis_pass(node, ctx)
node = for_canonicalization.transform(node, ctx.namer)
# for_canonicalization may insert new global references.
node = builtin_functions.transform(node)
# builtin_functions may insert new global references.
ctx.namespace['print'] = print
node = _static_analysis_pass(node, ctx)
node = print_functions.transform(node)
node = call_trees.transform(node, ctx.namer, ctx.namespace,
config.DEFAULT_UNCOMPILED_MODULES,
nocompile_decorators)
node = control_flow.transform(node, ctx.namer)
node = logical_expressions.transform(node)
node = side_effect_guards.transform(node, ctx.namer)
return node
def test_noop(self):
def test_fn(a):
return a
node = self.parse_and_analyze(test_fn, {})
node, deps = decorators.transform(node, remove_decorators=())
result, _ = compiler.ast_to_object(node)
self.assertFalse(deps)
self.assertEqual(1, result.test_fn(1))
def _remover_wrapper(self, f, remove_decorators):
namespace = {
'self_removing_decorator': self_removing_decorator,
'simple_decorator': simple_decorator
}
node = self.parse_and_analyze(f, namespace)
node, _ = decorators.transform(node,
remove_decorators=remove_decorators)
result, _ = compiler.ast_to_object(node)
return getattr(result, f.__name__)
def test_noop(self):
def test_fn(a):
return a
node = self.parse_and_analyze(test_fn, {})
node, deps = decorators.transform(node, remove_decorators=())
result, _ = compiler.ast_to_object(node)
self.assertFalse(deps)
self.assertEqual(1, result.test_fn(1))