def test_py_func_wrap_known_function(self):
def test_fn():
return np.random.binomial(2, 0.5)
node = self.parse_and_analyze(test_fn, {'np': np})
node = call_trees.transform(node, self.ctx, (), ())
with self.compiled(node, dtypes.int64) as result:
result.np = np
with self.test_session() as sess:
self.assertTrue(isinstance(result.test_fn(), ops.Tensor))
self.assertIn(sess.run(result.test_fn()), (0, 1, 2))
def test_dynamic_function(self):
def test_fn_1():
raise ValueError('This should be masked by the mock.')
def test_fn_2(f):
return f() + 3
node = self.parse_and_analyze(test_fn_2, {})
node = call_trees.transform(node, self.ctx, (), ())
with self.compiled(node) as result:
# 10 = 7 (from the mock) + 3 (from test_fn_2)
self.assertEquals(10, result.test_fn_2(test_fn_1))
def test_py_func_wrap_known_function(self):
def test_fn():
return np.random.binomial(2, 0.5)
node = self.parse_and_analyze(test_fn, {'np': np})
node = call_trees.transform(node, self.ctx, (), ())
with self.compiled(node, dtypes.int64) as result:
result.np = np
with self.test_session() as sess:
self.assertTrue(isinstance(result.test_fn(), ops.Tensor))
self.assertIn(sess.run(result.test_fn()), (0, 1, 2))
def test_dynamic_function(self):
def test_fn_1():
raise ValueError('This should be masked by the mock.')
def test_fn_2(f):
return f() + 3
node = self.parse_and_analyze(test_fn_2, {})
node = call_trees.transform(node, self.ctx, (), ())
with self.compiled(node) as result:
# 10 = 7 (from the mock) + 3 (from test_fn_2)
self.assertEquals(10, result.test_fn_2(test_fn_1))
def test_simple_methods(self):
class TestClass(object):
def test_fn_1(self, a):
return a + 1
def test_fn_2(self, a):
return self.test_fn_1(a) + 1
node = self.parse_and_analyze(
TestClass.test_fn_2, {'TestClass': TestClass},
namer=converter_test_base.FakeNoRenameNamer(),
arg_types={'self': (TestClass.__name__, TestClass)})
node = call_trees.transform(node, self.ctx, (), ())
with self.compiled(node) as result:
tc = TestClass()
self.assertEquals(3, result.test_fn_2(tc, 1))
def test_uncompiled_modules(self):
def test_fn(a):
a = math_ops.multiply(a, constant_op.constant(2))
a = math_ops.add(a, constant_op.constant(1))
return a
ns = {'math_ops': math_ops, 'constant_op': constant_op}
node, ctx = self.prepare(test_fn,
ns,
arg_types=set(((math_ops.__name__, ),
(constant_op.__name__, ))))
node = call_trees.transform(node, ctx)
with self.compiled(node, ns) as result:
with self.test_session() as sess:
result_tensor = result.test_fn(constant_op.constant(1))
self.assertEquals(sess.run(result_tensor), 3)
def test_basic(self):
def test_fn_1(_):
raise ValueError(
'This should not be called in the compiled version.')
def renamed_test_fn_1(a):
return a + 1
def test_fn_2(a):
return test_fn_1(a) + 1
node = self.parse_and_analyze(test_fn_2, {'test_fn_1': test_fn_1})
node = call_trees.transform(node, self.ctx, (), ())
with self.compiled(node) as result:
# Only test_fn_2 is transformed, so we'll insert renamed_test_fn_1
# manually.
result.renamed_test_fn_1 = renamed_test_fn_1
self.assertEquals(3, result.test_fn_2(1))
def test_basic(self):
def test_fn_1(_):
raise ValueError(
'This should not be called in the compiled version.')
def other_test_fn_1(a):
return a + 1
def test_fn_2(a):
return test_fn_1(a) + 1
ns = {'test_fn_1': test_fn_1}
node, ctx = self.prepare(test_fn_2, ns)
node = call_trees.transform(node, ctx)
with self.compiled(node, ns) as result:
new_name, _ = ctx.namer.compiled_function_name(('test_fn_1', ))
setattr(result, new_name, other_test_fn_1)
self.assertEquals(result.test_fn_2(1), 3)
def test_basic_method(self):
class TestClass(object):
def test_fn_1(self, a):
return a + 1
def test_fn_2(self, a):
return self.test_fn_1(a) + 1
ns = {'TestClass': TestClass}
node, ctx = self.prepare(
TestClass.test_fn_2,
ns,
namer=converter_testing.FakeNoRenameNamer(),
arg_types={'self': (TestClass.__name__, TestClass)})
node = call_trees.transform(node, ctx)
with self.compiled(node, ns) as result:
tc = TestClass()
self.assertEquals(3, result.test_fn_2(tc, 1))
def test_simple_methods(self):
class TestClass(object):
def test_fn_1(self, a):
return a + 1
def test_fn_2(self, a):
return self.test_fn_1(a) + 1
node = self.parse_and_analyze(
TestClass.test_fn_2, {'TestClass': TestClass},
namer=converter_test_base.FakeNoRenameNamer(),
arg_types={'self': (TestClass.__name__, TestClass)})
node = call_trees.transform(node, self.ctx, (), ())
with self.compiled(node) as result:
tc = TestClass()
self.assertEquals(3, result.test_fn_2(tc, 1))
def test_basic(self):
def test_fn_1(_):
raise ValueError('This should not be called in the compiled version.')
def renamed_test_fn_1(a):
return a + 1
def test_fn_2(a):
return test_fn_1(a) + 1
node = self.parse_and_analyze(test_fn_2, {'test_fn_1': test_fn_1})
node = call_trees.transform(node, self.ctx, (), ())
with self.compiled(node) as result:
# Only test_fn_2 is transformed, so we'll insert renamed_test_fn_1
# manually.
result.renamed_test_fn_1 = renamed_test_fn_1
self.assertEquals(3, result.test_fn_2(1))
def test_py_func_wrap_no_retval(self):
def test_fn(a):
setattr(a, 'foo', 'bar')
node = self.parse_and_analyze(test_fn, {'setattr': setattr})
node = call_trees.transform(node, self.ctx, (), ())
with self.compiled(node) as result:
with self.test_session() as sess:
# The function has no return value, so we do some tricks to grab the
# generated py_func node and ensure its effect only happens at graph
# execution.
class Dummy(object):
pass
a = Dummy()
result.test_fn(a)
self.assertFalse(hasattr(a, 'foo'))
sess.run(sess.graph.get_operations()[0])
self.assertEquals('bar', a.foo)
def test_py_func_wrap_no_retval(self):
def test_fn(a):
setattr(a, 'foo', 'bar')
node = self.parse_and_analyze(test_fn, {'setattr': setattr})
node = call_trees.transform(node, self.ctx, (), ())
with self.compiled(node) as result:
with self.test_session() as sess:
# The function has no return value, so we do some tricks to grab the
# generated py_func node and ensure its effect only happens at graph
# execution.
class Dummy(object):
pass
a = Dummy()
result.test_fn(a)
self.assertFalse(hasattr(a, 'foo'))
sess.run(sess.graph.get_operations()[0])
self.assertEquals('bar', a.foo)
def test_uncompiled_modules(self):
def test_fn(a):
a = math_ops.multiply(a, constant_op.constant(2))
a = math_ops.add(a, constant_op.constant(1))
return a
node = self.parse_and_analyze(test_fn, {
'math_ops': math_ops,
'constant_op': constant_op
})
node = call_trees.transform(
node, self.ctx,
set(((math_ops.__name__, ), (constant_op.__name__, ))), ())
with self.compiled(node) as result:
result.math_ops = math_ops
result.constant_op = constant_op
with self.test_session() as sess:
# Not renamed, because the converter doesn't rename the definition
# itself (the caller is responsible for that).
result_tensor = result.test_fn(constant_op.constant(1))
self.assertEquals(3, sess.run(result_tensor))
def test_uncompiled_modules(self):
def test_fn(a):
a = math_ops.multiply(a, constant_op.constant(2))
a = math_ops.add(a, constant_op.constant(1))
return a
node = self.parse_and_analyze(test_fn, {
'math_ops': math_ops,
'constant_op': constant_op
})
node = call_trees.transform(node, self.ctx,
set(((math_ops.__name__,),
(constant_op.__name__,))), ())
with self.compiled(node) as result:
result.math_ops = math_ops
result.constant_op = constant_op
with self.test_session() as sess:
# Not renamed, because the converter doesn't rename the definition
# itself (the caller is responsible for that).
result_tensor = result.test_fn(constant_op.constant(1))
self.assertEquals(3, sess.run(result_tensor))
def node_to_graph(node, ctx, nocompile_decorators):
"""Convert Python code to equivalent TF graph mode code.
Args:
node: A Python AST node representing the code to convert.
ctx: An EntityContext object.
nocompile_decorators: A tuple containing decorators to be stripped from
functions during conversion.
Returns:
A tuple (node, deps):
* node: A Python ast node, representing the converted code.
* deps: A set of strings, the fully qualified names of entity
dependencies that this node has.
"""
# TODO(mdan): Verify arguments for correctness.
# TODO(mdan): Factor out common elements.
# These include:
# * code move between blocks
# * visiting blocks in transformers
# Certain steps, especially canonicalization, insert new symbols into the
# tree, which must be accounted. Although less efficient, it is most robust
# to re-run the analysis.
node = _static_analysis_pass(node, ctx)
# TODO(mdan): Clean this up.
# Some intermediate analyses are not required, and some comments got orphaned.
# Past this point, line numbers are no longer accurate so we ignore the
# source.
# TODO(mdan): Is it feasible to reconstruct intermediate source code?
ctx.source_code = None
node = ifexp.transform(node, ctx)
node, deps = decorators.transform(node, nocompile_decorators)
node = break_statements.transform(node, ctx)
node = asserts.transform(node, ctx)
# Note: sequencing continue canonicalization before for loop one avoids
# dealing with the extra loop increment operation that the for
# canonicalization creates.
node = continue_statements.transform(node, ctx)
ctx.namespace['len'] = len
node = _static_analysis_pass(node, ctx)
node = single_return.transform(node, ctx)
node = _static_analysis_pass(node, ctx)
node = lists.transform(node, ctx)
node = builtin_functions.transform(node, ctx)
node = _static_analysis_pass(node, ctx)
node = call_trees.transform(node, ctx, config.DEFAULT_UNCOMPILED_MODULES,
nocompile_decorators)
node = control_flow.transform(node, ctx)
# control_flow may create new symbols and change scopes.
node = _static_analysis_pass(node, ctx)
node = logical_expressions.transform(node, ctx)
node = side_effect_guards.transform(node, ctx)
node = name_scopes.transform(node, ctx)
return node, deps
def node_to_graph(node, ctx, nocompile_decorators):
"""Convert Python code to equivalent TF graph mode code.
Args:
node: A Python AST node representing the code to convert.
ctx: An EntityContext object.
nocompile_decorators: A tuple containing decorators to be stripped from
functions during conversion.
Returns:
A tuple (node, deps):
* node: A Python ast node, representing the converted code.
* deps: A set of strings, the fully qualified names of entity
dependencies that this node has.
"""
# TODO(mdan): Verify arguments for correctness.
# TODO(mdan): Factor out common elements.
# These include:
# * code move between blocks
# * visiting blocks in transformers
# Certain steps, especially canonicalization, insert new symbols into the
# tree, which must be accounted. Although less efficient, it is most robust
# to re-run the analysis.
node = _static_analysis_pass(node, ctx)
# TODO(mdan): Clean this up.
# Some intermediate analyses are not required, and some comments got orphaned.
# Past this point, line numbers are no longer accurate so we ignore the
# source.
# TODO(mdan): Is it feasible to reconstruct intermediate source code?
ctx.source_code = None
node = ifexp.transform(node, ctx)
node, deps = decorators.transform(node, nocompile_decorators)
node = break_statements.transform(node, ctx)
node = asserts.transform(node, ctx)
# Note: sequencing continue canonicalization before for loop one avoids
# dealing with the extra loop increment operation that the for
# canonicalization creates.
node = continue_statements.transform(node, ctx)
ctx.namespace['len'] = len
node = _static_analysis_pass(node, ctx)
node = single_return.transform(node, ctx)
node = _static_analysis_pass(node, ctx)
node = lists.transform(node, ctx)
node = builtin_functions.transform(node, ctx)
node = _static_analysis_pass(node, ctx)
node = call_trees.transform(node, ctx, config.DEFAULT_UNCOMPILED_MODULES,
nocompile_decorators)
node = control_flow.transform(node, ctx)
# control_flow may create new symbols and change scopes.
node = _static_analysis_pass(node, ctx)
node = logical_expressions.transform(node, ctx)
node = side_effect_guards.transform(node, ctx)
node = name_scopes.transform(node, ctx)
return node, deps
