def test_function_decorator(self): def function_decorator(): def decorator(f): return lambda a: f(a) + 1 return decorator # The Python parser does capture decorators into the AST. # However, the interpreter desugars them on load, and refering to the # decorated function at runtime usually loses any trace of the decorator. # Below is an example when that doesn't happen. def static_wrapper(): @function_decorator() def test_fn(a): # pylint:disable=unused-variable return a node = self.parse_and_analyze(static_wrapper, {'function_decorator': function_decorator}) node = node.body[0].body[0] node = decorators.transform(node, remove_decorators=()) # Since the decorator is not removed, we need to include its source # code. We cannot do it after the fact because decorators are executed # on load. result, _ = compiler.ast_to_object( node, source_prefix=textwrap.dedent(tf_inspect.getsource(function_decorator))) self.assertEqual(2, result.test_fn(1)) node = decorators.transform(node, remove_decorators=(function_decorator,)) with self.compiled(node) as result: self.assertEqual(1, result.test_fn(1))
def test_simple_decorator(self): def simple_decorator(f): return lambda a: f(a) + 1 # The Python parser does capture decorators into the AST. # However, the interpreter desugars them upon load, and refering to the # decorated function at runtime usually loses any trace of the decorator. # Below is an example when that doesn't happen. def static_wrapper(): @simple_decorator def test_fn(a): # pylint:disable=unused-variable return a node = self.parse_and_analyze(static_wrapper, {'simple_decorator': simple_decorator}) node = node.body[0].body[0] node = decorators.transform(node, remove_decorators=()) result = compiler.ast_to_object( node, source_prefix=textwrap.dedent(tf_inspect.getsource(simple_decorator))) self.assertEqual(2, result.test_fn(1)) node = decorators.transform(node, remove_decorators=(simple_decorator,)) result = compiler.ast_to_object(node) self.assertEqual(1, result.test_fn(1))
def test_simple_decorator(self): def simple_decorator(f): return lambda a: f(a) + 1 # The Python parser does capture decorators into the AST. # However, the interpreter desugars them upon load, and refering to the # decorated function at runtime usually loses any trace of the decorator. # Below is an example when that doesn't happen. def static_wrapper(): @simple_decorator def test_fn(a): # pylint:disable=unused-variable return a node = self.parse_and_analyze(static_wrapper, {'simple_decorator': simple_decorator}) node = node.body[0].body[0] node = decorators.transform(node, remove_decorators=()) # Since the decorator is not removed, we need to include its source # code. We cannot do it after the fact because decorators are executed # on load. result, _ = compiler.ast_to_object( node, source_prefix=textwrap.dedent( tf_inspect.getsource(simple_decorator))) self.assertEqual(2, result.test_fn(1)) node = decorators.transform(node, remove_decorators=(simple_decorator, )) with self.compiled(node) as result: self.assertEqual(1, result.test_fn(1))
def test_function_decorator(self): def function_decorator(): def decorator(f): return lambda a: f(a) + 1 return decorator # The Python parser does capture decorators into the AST. # However, the interpreter desugars them on load, and refering to the # decorated function at runtime usually loses any trace of the decorator. # Below is an example when that doesn't happen. def static_wrapper(): @function_decorator() def test_fn(a): # pylint:disable=unused-variable return a node = self.parse_and_analyze( static_wrapper, {'function_decorator': function_decorator}) node = node.body[0].body[0] node = decorators.transform(node, remove_decorators=()) result = compiler.ast_to_object( node, source_prefix=textwrap.dedent( tf_inspect.getsource(function_decorator))) self.assertEqual(2, result.test_fn(1)) node = decorators.transform(node, remove_decorators=(function_decorator, )) result = compiler.ast_to_object(node) self.assertEqual(1, result.test_fn(1))
def node_to_graph(node, ctx, nocompile_decorators): """Convert Python code to equivalent TF graph mode code. Args: node: A Python AST node representing the code to convert. ctx: An EntityContext object. nocompile_decorators: A tuple containing decorators to be stripped from functions during conversion. Returns: A tuple (node, deps): * node: A Python ast node, representing the converted code. * deps: A set of strings, the fully qualified names of entity dependencies that this node has. """ # TODO(mdan): Verify arguments for correctness. # TODO(mdan): Factor out common elements. # These include: # * code move between blocks # * visiting blocks in transformers # Certain steps, especially canonicalization, insert new symbols into the # tree, which must be accounted. Although less efficient, it is most robust # to re-run the analysis. node = _static_analysis_pass(node, ctx) # Past this point, line numbers are no longer accurate so we ignore the # source. # TODO(mdan): Is it feasible to reconstruct intermediate source code? ctx.source_code = None node = decorators.transform(node, nocompile_decorators) node = break_statements.transform(node, ctx) node = asserts.transform(node, ctx) # Note: sequencing continue canonicalization before for loop one avoids # dealing with the extra loop increment operation that the for # canonicalization creates. node = continue_statements.transform(node, ctx) ctx.namespace['len'] = len node = _static_analysis_pass(node, ctx) node = for_loops.transform(node, ctx) # for_loops may insert new global references. node = builtin_functions.transform(node, ctx) # TODO(mdan): Kept for CL consistency. Remove. # builtin_functions may insert new global references. ctx.namespace['print'] = print node = _static_analysis_pass(node, ctx) node = call_trees.transform(node, ctx, config.DEFAULT_UNCOMPILED_MODULES, nocompile_decorators) node = control_flow.transform(node, ctx) # control_flow may create new symbols and change scopes. node = _static_analysis_pass(node, ctx) node = logical_expressions.transform(node) node = side_effect_guards.transform(node, ctx) return node
def node_to_graph(node, ctx, nocompile_decorators): """Convert Python code to equivalent TF graph mode code. Args: node: A Python AST node representing the code to convert. ctx: An EntityContext object. nocompile_decorators: A tuple containing decorators to be stripped from functions during conversion. Returns: A tuple (node, deps): * node: A Python ast node, representing the converted code. * deps: A set of strings, the fully qualified names of entity dependencies that this node has. """ # TODO(mdan): Verify arguments for correctness. # TODO(mdan): Factor out common elements. # These include: # * code move between blocks # * visiting blocks in transformers # Certain steps, especially canonicalization, insert new symbols into the # tree, which must be accounted. Although less efficient, it is most robust # to re-run the analysis. node = _static_analysis_pass(node, ctx) # Past this point, line numbers are no longer accurate so we ignore the # source. # TODO(mdan): Is it feasible to reconstruct intermediate source code? ctx.source_code = None node = decorators.transform(node, nocompile_decorators) node = break_canonicalization.transform(node, ctx) node = asserts.transform(node, ctx) # Note: sequencing continue canonicalization before for loop one avoids # dealing with the extra loop increment operation that the for # canonicalization creates. node = continue_canonicalization.transform(node, ctx) ctx.namespace['len'] = len node = _static_analysis_pass(node, ctx) node = for_canonicalization.transform(node, ctx) # for_canonicalization may insert new global references. node = builtin_functions.transform(node, ctx) # builtin_functions may insert new global references. ctx.namespace['print'] = print node = _static_analysis_pass(node, ctx) node = call_trees.transform(node, ctx, config.DEFAULT_UNCOMPILED_MODULES, nocompile_decorators) node = control_flow.transform(node, ctx) # control_flow may create new symbols and change scopes. node = _static_analysis_pass(node, ctx) node = logical_expressions.transform(node) node = side_effect_guards.transform(node, ctx) return node
def _remover_wrapper(self, f, remove_decorators): namespace = { 'self_removing_decorator': self_removing_decorator, 'simple_decorator': simple_decorator } node = self.parse_and_analyze(f, namespace) node, _ = decorators.transform(node, remove_decorators=remove_decorators) result, _ = compiler.ast_to_object(node) return getattr(result, f.__name__)
def node_to_graph(node, ctx, nocompile_decorators): """Convert Python code to equivalent TF graph mode code. Args: node: A Python AST node representing the code to convert. ctx: An EntityContext object. nocompile_decorators: A tuple containing decorators to be stripped from functions during conversion. Returns: A tuple (node, deps): * node: A Python ast node, representing the converted code. * deps: A set of strings, the fully qualified names of entity dependencies that this node has. """ # TODO(mdan): Verify arguments for correctness. # TODO(mdan): Factor out common elements. # These include: # * keeping track of symbols that have been created # * marking nodes (e.g. py_func wrappers) to suppress further processing # * code move between blocks # * insertion of new global references # * visiting blocks in transformers # Certain steps, especially canonicalization, insert new symbols into the # tree, which must be accounted. Although less efficient, it is most robust # to re-run the analysis. node = _static_analysis_pass(node, ctx) node = decorators.transform(node, nocompile_decorators) node = break_canonicalization.transform(node, ctx.namer) # Note: sequencing continue canonicalization before for loop one avoids # dealing with the extra loop increment operation that the for # canonicalization creates. node = continue_canonicalization.transform(node, ctx.namer) ctx.namespace['len'] = len node = _static_analysis_pass(node, ctx) node = for_canonicalization.transform(node, ctx.namer) # for_canonicalization may insert new global references. node = builtin_functions.transform(node) # builtin_functions may insert new global references. ctx.namespace['print'] = print node = _static_analysis_pass(node, ctx) node = print_functions.transform(node) node = call_trees.transform(node, ctx.namer, ctx.namespace, config.DEFAULT_UNCOMPILED_MODULES, nocompile_decorators) node = control_flow.transform(node, ctx.namer) node = logical_expressions.transform(node) node = side_effect_guards.transform(node, ctx.namer) return node
def test_noop(self): def test_fn(a): return a node = self.parse_and_analyze(test_fn, {}) node, deps = decorators.transform(node, remove_decorators=()) result, _ = compiler.ast_to_object(node) self.assertFalse(deps) self.assertEqual(1, result.test_fn(1))
def _remover_wrapper(self, f, remove_decorators): namespace = { 'self_removing_decorator': self_removing_decorator, 'simple_decorator': simple_decorator } node = self.parse_and_analyze(f, namespace) node, _ = decorators.transform(node, remove_decorators=remove_decorators) result, _ = compiler.ast_to_object(node) return getattr(result, f.__name__)
def test_noop(self): def test_fn(a): return a node = self.parse_and_analyze(test_fn, {}) node, deps = decorators.transform(node, remove_decorators=()) result, _ = compiler.ast_to_object(node) self.assertFalse(deps) self.assertEqual(1, result.test_fn(1))