def test_delete_kwarg(self):
# expected
expected = '''
def test2(x, y):
data4 = paddle.tensor.add(x, y, name=None, out=None, alpha=1)
'''
# captured input
input_test = '''
def test2(x, y):
data4 = paddle.fluid.layers.elementwise_add(x, y, axis=-1, act=None, name=None)
'''
# get captured
root = gast.parse(input_test)
root_expected = gast.parse(expected)
print("*******test_2_source_code*******")
# print(inspect.getsource(test1))
root = transformer(root, modify_dict)
print("*******test_delete_kwarg*******")
print(astor.to_source(gast.gast_to_ast(root)))
captured = astor.to_source(gast.gast_to_ast(root))
expected = astor.to_source(gast.gast_to_ast(root_expected))
# testting
self.assertEqual(expected, captured)
def test_rename_kwarg(self):
# expected
expected = '''
def test3(x, y):
data4 = paddle.tensor.argmax(input=x, axis=2, out=None, dtype=None, keepdims=False, name=None)
'''
# captured input
input_test = '''
def test3(x, y):
data4 = paddle.fluid.layers.argmax(x=x, axis=2)
'''
# get captured
root = gast.parse(input_test)
root_expected = gast.parse(expected)
print("*******test_3_source_code*******")
# print(inspect.getsource(test1))
root = transformer(root, modify_dict)
print("*******test_rename_kwarg*******")
print(astor.to_source(gast.gast_to_ast(root)))
captured = astor.to_source(gast.gast_to_ast(root))
expected = astor.to_source(gast.gast_to_ast(root_expected))
# testting
self.assertEqual(expected, captured)
def test_add_kwarg(self):
# expected
expected = '''
def test1():
data4 = paddle.tensor.zeros(x=x, shape=[3, 2], dtype='float32', out=None, device=None)
'''
# captured input
input_test = '''
def test1():
data4 = paddle.fluid.layers.zeros(x=x, shape=[3, 2], dtype='float32', force_cpu=True)
'''
# get captured
# root = gast.parse(inspect.getsource(input_test))
root = gast.parse(input_test)
root_expected = gast.parse(expected)
print("*******test_1_source_code*******")
# print(inspect.getsource(test1))
root = transformer(root, modify_dict)
print("*******test_add_kwarg*******")
print(astor.to_source(gast.gast_to_ast(root)))
captured = astor.to_source(gast.gast_to_ast(root))
expected = astor.to_source(gast.gast_to_ast(root_expected))
# testting
self.assertEqual(expected, captured)
def _to_array_write_node(self, node):
assert isinstance(node, gast.Call)
array = astor.to_source(gast.gast_to_ast(node.func.value))
x = astor.to_source(gast.gast_to_ast(node.args[0]))
i = "fluid.layers.array_length({})".format(array)
func_code = "fluid.layers.array_write(x={}, i={}, array={})".format(
x, i, array)
return gast.parse(func_code).body[0].value
def test_replace_full_name_transformer(self, import_map=import_dict):
# expected
expected = '''
import paddle
import paddle.fluid as fluid
import numpy as np
from paddle.fluid.dygraph.nn import Conv2D, Pool2D, Linear
class LeNet(fluid.dygraph.Layer):
def __init__(self, name_scope, num_classes=1):
super(LeNet, self).__init__(name_scope)
self.conv1 = paddle.fluid.dygraph.nn.Conv2D(num_channels=1, num_filters=6, filter_size=5, act='sigmoid')
self.pool1 = paddle.fluid.dygraph.nn.Pool2D(pool_size=2, pool_stride=2, pool_type='max')
self.conv2 = paddle.fluid.dygraph.nn.Conv2D(num_channels=6, num_filters=16, filter_size=5, act='sigmoid')
self.pool2 = paddle.fluid.dygraph.nn.Pool2D(pool_size=2, pool_stride=2, pool_type='max')
self.conv3 = paddle.fluid.dygraph.nn.Conv2D(num_channels=16, num_filters=120, filter_size=4, act='sigmoid')
self.fc1 = paddle.fluid.dygraph.nn.Linear(input_dim=120, output_dim=64, act='sigmoid')
self.fc2 = paddle.fluid.dygraph.nn.Linear(input_dim=64, output_dim=num_classes)
def forward(self, x):
x = self.conv1(x)
x = self.pool1(x)
x = self.conv2(x)
x = self.pool2(x)
x = self.conv3(x)
x = paddle.fluid.layers.reshape(x, [x.shape[0], -1])
x = self.fc1(x)
x = self.fc2(x)
return x
'''
# captured input
input_test = gast.parse(inspect.getsource(dygraph_lenet))
print("*******original code*******")
original_code = astor.to_source(gast.gast_to_ast(input_test))
original_code_formated = FormatCode(original_code)
print(original_code_formated[0])
# get captured
root = gast.parse(input_test)
root_expected = gast.parse(expected)
print("*******start_transfer*******")
root = transfomer_replace_test(root, import_map)
print("*******code_for_captured_formated*******")
# print(astor.to_source(gast.gast_to_ast(root)))
captured = astor.to_source(gast.gast_to_ast(root))
captured_formated = FormatCode(captured)
print(captured_formated[0])
expected = astor.to_source(gast.gast_to_ast(root_expected))
expected_formated = FormatCode(expected)
print("*******code_for_expected_formated*******")
print(expected_formated[0])
# testting
self.assertEqual(expected_formated, captured_formated)
def test_fusion_defDes(self):
# expected
expected = '''
def test1(x):
data1 = paddle.nn.data(x=x)
data2 = paddle.tensor.reshape(x=data1, shape=[1, 2, 3])
data4 = paddle.tensor.zeros(shape=[3, 2], dtype='float32', out=None, device=None)
return abc
def test2(x):
data1 = paddle.tensor.tanh(input=x, out=None)
data2 = paddle.tensor.zeros(shape=[3, 2], dtype='float32', out=None, device=None)
def test3(x):
data1 = paddle.nn.Sigmoid(x=x)
data2 = paddle.tensor.sum(x, axis=0)
def test4(x):
data1 = paddle.tensor.max(input, dim=None, keep_dim=False, out=None, name=None)
data2 = paddle.tensor.stack(x, dim=0, out=None)
'''
# captured input
input_test = '''
def test1(x):
data1 = paddle.fluid.layers.data(x=x)
data2 = paddle.fluid.layers.reshape(x=data1, shape=[1, 2, 3])
data3 = paddle.fluid.layers.zeros(shape=[3, 2], dtype='float32')
data4 = paddle.fluid.layers.zeros(shape=[3, 2], dtype='float32', force_cpu=True)
return abc
def test2(x):
data1 = paddle.fluid.layers.tanh(x=x)
data2 = paddle.fluid.layers.zeros(shape=[3, 2], dtype='float32')
def test3(x):
data1 = paddle.fluid.layers.sigmoid(x=x)
data2 = paddle.fluid.layers.reduce_sum(x, dim=0)
def test4(x):
data1 = paddle.fluid.layers.reduce_max(x=x)
data2 = fluid.layers.stack(x, dim=0)
'''
# get captured
root = gast.parse(input_test)
root_expected = gast.parse(expected)
print("*******test_4_source_code*******")
# print(inspect.getsource(test1))
root = transformer(root, modify_dict)
print("*******test_fusion_function_kwarg*******")
print(astor.to_source(gast.gast_to_ast(root)))
captured = astor.to_source(gast.gast_to_ast(root))
expected = astor.to_source(gast.gast_to_ast(root_expected))
# testting
self.assertEqual(expected, captured)
def test_single_model_file(self):
parser = argparse.ArgumentParser("Paddle API upgrade")
parser.add_argument("--modify_dict",
type=str,
default="../dict/modify.dict")
parser.add_argument("--input", type=str, default='class_cnn.py')
parser.add_argument("--output",
type=str,
default='class_cnn_update.py')
args = parser.parse_args()
# read captured
with open('class_cnn.py', 'r') as fp:
data_captured = fp.read()
with open('class_cnn_expected.py', 'r') as fp:
data_expected = fp.read()
# expected
expected = data_expected
# captured input
input_test = data_captured
# # get captured
root = gast.parse(input_test)
root_expected = gast.parse(expected)
print("*******test_5_source_code*******")
root = transformer(root, modify_dict)
print("*******test_model_single_file*******")
print(astor.to_source(gast.gast_to_ast(root)))
captured = astor.to_source(gast.gast_to_ast(root))
captured_formated = FormatCode(captured)
with open('class_cnn_update.py', 'w') as fp:
fp.write(captured_formated[0])
expected = astor.to_source(gast.gast_to_ast(root_expected))
data_expected_formated = FormatCode(captured)
# read captured
with open('class_cnn_update.py', 'r') as fp:
data_captured = fp.read()
captured = astor.to_source(gast.gast_to_ast(root))
data_captured_formated = FormatCode(captured)
# testting
self.assertEqual(data_expected_formated, data_captured_formated)
def testCompile(self):
srcs = glob.glob(os.path.join(gast.__path__[0], '*.py'))
for src_py in srcs:
with open(src_py) as f:
content = f.read()
gnode = gast.parse(content)
compile(gast.gast_to_ast(gnode), src_py, 'exec')
def compile_file(source, globals_=None):
"""Compile by saving to file and importing that.
Compiling the AST/source code this way ensures that the source code is
readable by e.g. `pdb` or `inspect`.
Args:
source: The code to compile, either as a string or as an AST.
globals_: A dictionary of variables that should be available as globals in
the compiled module. They will be monkey patched after importing the
module.
Returns:
A module object containing the compiled source code.
"""
if isinstance(source, gast.AST):
source = astor.to_source(gast.gast_to_ast(source))
# Write source to temporary file
tempdir = tempfile.mkdtemp()
uuid = str(uuid4().hex[:4])
tmpname = os.path.join(tempdir, 'matchbox_%s.py' % uuid)
with open(tmpname, 'w') as f:
f.write(source)
# Load the temporary file as a module
module_name = 'matchbox_%s' % uuid
if six.PY3:
spec = util.spec_from_file_location(module_name, tmpname)
m = util.module_from_spec(spec)
spec.loader.exec_module(m)
else:
m = imp.load_source(module_name, tmpname)
# Update the modules namespace
if globals_:
m.__dict__.update(globals_)
return m
def test_TryFinally(self):
code = 'try:pass\nfinally:pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
norm = ("Module(body=[Try(body=[Pass()], handlers=[], orelse=[], "
"finalbody=[Pass()])], type_ignores=[])")
self.assertEqual(gast.dump(tree), norm)
def ast_to_source(node, indentation=' '):
"""Return the source code of given AST.
Args:
node: The code to compile, as an AST object.
indentation: The string to use for indentation.
Returns:
code: The source code generated from the AST object
source_mapping: A mapping between the user and AutoGraph generated code.
"""
original_node = node
if isinstance(node, gast.AST):
node = gast.gast_to_ast(node)
generator = astor.codegen.SourceGenerator(indentation, False,
astor.string_repr.pretty_string)
generator.visit(node)
generator.result.append('\n')
# In some versions of Python, literals may appear as actual values. This
# ensures everything is string.
code = map(str, generator.result)
code = astor.source_repr.pretty_source(code).lstrip()
source_mapping = _build_source_map(original_node, code)
return code, source_mapping
def prepare(self, node):
assert isinstance(node, ast.Module)
self.env = {
'builtins': __import__('builtins'),
}
for module_name in MODULES:
# __dispatch__ is the only fake top-level module
if module_name != '__dispatch__':
import_name = module_name
alias_module_name = mangle(module_name)
self.env[alias_module_name] = __import__(import_name)
# handle functions conflicting with c++ keywords
for fun in MODULES[module_name]:
if fun in ("__theitemgetter__", "pythran"):
# these ones do not exist in Python
continue
# we need to parse the whole code to be able to apply user-defined pure
# function but import are resolved before so we remove them to avoid
# ImportError (for operator_ for example)
dummy_module = ast.Module([s for s in node.body
if not isinstance(s, ast.Import)],
[])
eval(compile(ast.gast_to_ast(dummy_module),
'<constant_folding>', 'exec'),
self.env)
super(ConstantFolding, self).prepare(node)
def ast_to_source(node, indentation=' '):
"""Return the source code of given AST.
Args:
node: The code to compile, as an AST object.
indentation: The string to use for indentation.
Returns:
code: The source code generated from the AST object
source_mapping: A mapping between the user and AutoGraph generated code.
"""
if not isinstance(node, (list, tuple)):
node = (node, )
generator = astor.codegen.SourceGenerator(indentation, False,
astor.string_repr.pretty_string)
for n in node:
if isinstance(n, gast.AST):
n = gast.gast_to_ast(n)
generator.visit(n)
generator.result.append('\n')
# In some versions of Python, literals may appear as actual values. This
# ensures everything is string.
code = map(str, generator.result)
code = astor.source_repr.pretty_source(code).lstrip()
return code
def generic_visit(self, node):
if isinstance(node, ast.expr) and node in self.constant_expressions:
fake_node = ast.Expression(node)
code = compile(ast.gast_to_ast(fake_node),
'<constant folding>', 'eval')
try:
value = eval(code, self.env)
new_node = to_ast(value)
if not ASTMatcher(node).match(new_node):
self.update = True
return new_node
except DamnTooLongPattern as e:
print("W: ", e, " Assume no update happened.")
except ConversionError as e:
print('error in constant folding: ', e)
raise
except ToNotEval:
pass
except AttributeError as e:
# this may miss a few optimization
logger.info('During constant folding, bailing out due to: ' +
e.args[0])
except NameError as e:
# FIXME dispatched function are not processed by constant
# folding
if "__dispatch__" in e.args[0]:
return Transformation.generic_visit(self, node)
# this may miss a few optimization
logger.info('During constant folding, bailing out due to: ' +
e.args[0])
except Exception as e:
raise PythranSyntaxError(str(e), node)
return Transformation.generic_visit(self, node)
def visit_For(self, node):
# if the user added some OpenMP directive, trust him and no unroll
if metadata.get(node, OMPDirective):
return node # don't visit children because of collapse
# first unroll children if needed or possible
self.generic_visit(node)
# a break or continue in the loop prevents unrolling too
has_break = any(self.gather(HasBreak, n)
for n in node.body)
has_cont = any(self.gather(HasContinue, n)
for n in node.body)
if has_break or has_cont:
return node
# do not unroll too much to prevent code growth
node_count = self.gather(NodeCount, node)
def unroll(elt, body):
return [ast.Assign([deepcopy(node.target)], elt)] + body
def dc(body, i, n):
if i == n - 1:
return body
else:
return deepcopy(body)
def getrange(n):
return getattr(getattr(n, 'func', None), 'attr', None)
if isinstance(node.iter, (ast.Tuple, ast.List)):
elts_count = len(node.iter.elts)
total_count = node_count * elts_count
issmall = total_count < LoopFullUnrolling.MAX_NODE_COUNT
if issmall:
self.update = True
return sum([unroll(elt, dc(node.body, i, elts_count))
for i, elt in enumerate(node.iter.elts)], [])
code = compile(ast.gast_to_ast(ast.Expression(node.iter)),
'<loop unrolling>', 'eval')
try:
values = list(eval(code, {'builtins': __import__('builtins')}))
except Exception:
return node
values_count = len(values)
total_count = node_count * values_count
issmall = total_count < LoopFullUnrolling.MAX_NODE_COUNT
if issmall:
try:
new_node = sum([unroll(to_ast(elt),
dc(node.body, i, values_count))
for i, elt in enumerate(values)], [])
self.update = True
return new_node
except Exception:
return node
return node
def unparse(node, indentation=None, include_encoding_marker=True):
"""Returns the source code of given AST.
Args:
node: The code to compile, as an AST object.
indentation: Unused, deprecated. The returning code will always be indented
at 4 spaces.
include_encoding_marker: Bool, whether to include a comment on the first
line to explicitly specify UTF-8 encoding.
Returns:
code: The source code generated from the AST object
source_mapping: A mapping between the user and AutoGraph generated code.
"""
del indentation # astunparse doesn't allow configuring it.
if not isinstance(node, (list, tuple)):
node = (node,)
codes = []
if include_encoding_marker:
codes.append('# coding=utf-8')
for n in node:
if isinstance(n, gast.AST):
ast_n = gast.gast_to_ast(n)
else:
ast_n = n
if astunparse is ast:
ast.fix_missing_locations(ast_n) # Only ast needs to call this.
codes.append(astunparse.unparse(ast_n).strip())
return '\n'.join(codes)
def prepare(self, node):
assert isinstance(node, ast.Module)
self.env = {
'builtins': __import__('builtins'),
}
for module_name in MODULES:
# __dispatch__ is the only fake top-level module
if module_name != '__dispatch__':
alias_module_name = mangle(module_name)
try:
self.env[alias_module_name] = __import__(module_name)
except ImportError:
pass
# we need to parse the whole code to be able to apply user-defined pure
# function but import are resolved before so we remove them to avoid
# ImportError (for operator_ for example)
dummy_module = ast.Module([s for s in node.body
if not isinstance(s, ast.Import)],
[])
eval(compile(ast.gast_to_ast(dummy_module),
'<constant_folding>', 'exec'),
self.env)
super(ConstantFolding, self).prepare(node)
def is_api_in_module(node, module_prefix):
assert isinstance(node,
gast.Call), "Input non-Call node for is_dygraph_api"
# Python can have gast.Call as function, for example: covert_call(func)(x)
# We only check the most outside function
func_node = node.func
while isinstance(func_node, gast.Call):
func_node = func_node.func
func_str = astor.to_source(gast.gast_to_ast(func_node)).strip()
try:
# TODO(liym27):
# Consider a better to import modules like:
# source_file = inspect.getfile(dyfunc)
# import_statements = ImportVisitor(source_file).transform()
# import_str = "".join(import_statements)
import paddle
import paddle.fluid as fluid
import paddle.fluid.dygraph as dygraph
import paddle.fluid.layers as layers
from paddle.fluid.dygraph import to_variable
from paddle import to_tensor
return eval("_is_api_in_module_helper({}, '{}')".format(
func_str, module_prefix))
except NameError:
return False
def test_Bytes(self):
code = 'b"0012"'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
norm = ("Module(body=[Expr(value=Constant(value=b'0012', "
"kind=None))], type_ignores=[])")
self.assertEqual(gast.dump(tree), norm)
def ast_to_source(node, indentation=' '):
"""Return the source code of given AST.
Args:
node: The code to compile, as an AST object.
indentation: The string to use for indentation.
Returns:
code: The source code generated from the AST object
source_mapping: A mapping between the user and AutoGraph generated code.
"""
if not isinstance(node, (list, tuple)):
node = (node,)
generator = astor.codegen.SourceGenerator(indentation, False,
astor.string_repr.pretty_string)
for n in node:
if isinstance(n, gast.AST):
n = gast.gast_to_ast(n)
generator.visit(n)
generator.result.append('\n')
# In some versions of Python, literals may appear as actual values. This
# ensures everything is string.
code = ''.join(map(str, generator.result))
# Strip leading blank lines.
code_lines = code.split('\n')
trimmed_code_lines = []
for l in code_lines:
if l.rstrip() or trimmed_code_lines:
trimmed_code_lines.append(l)
code = '\n'.join(trimmed_code_lines)
return code
def make_ast(
target_ast_node_count,
rng = None,
distribution = (
python_numbers_control_flow.DATAFLOW_FNS_DISTRIBUTION)
):
"""Generates an AST for this task.
Args:
target_ast_node_count: How many nodes to put in the AST.
rng: Random state to use.
distribution: Sampling distribution to use when building the AST. May also
be a callable that produces a distribution given a random state.
Returns:
AST of a generated program.
"""
def root_build(body):
"""Given a list of statements, puts them into a function in a module."""
return gast.Module(
body=[
gast.FunctionDef(
name="random_function",
args=_make_arguments(
python_numbers_control_flow.make_name("a"),
python_numbers_control_flow.make_name("b")),
body=body,
decorator_list=[],
returns=None,
type_comment=None)
],
type_ignores=[])
root_template = python_numbers_control_flow.ASTWithHoles(
cost=5,
holes=[
top_down_refinement.Hole(
python_numbers_control_flow.ASTHoleType.STMTS_NONEMPTY,
python_numbers_control_flow.ASTHoleMetadata(("a", "b"), True,
False, 0))
],
build=root_build)
if rng is None:
rng = np.random.RandomState()
if callable(distribution):
distribution = distribution(rng)
tree = top_down_refinement.top_down_construct(
root_object=root_template,
target_cost=target_ast_node_count,
refinement_distribution=distribution,
rng=rng)
# Re-parse the tree so that it is valid. This is required for program graph
# analysis to work.
return gast.parse(astunparse.unparse(gast.gast_to_ast(tree)))
def test_NamedExpr(self):
code = '(x := 1) '
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
norm = ("Module(body=[Expr(value=NamedExpr(target=Name(id='x',"
" ctx=Store(), annotation=None, type_comment=None), "
"value=Constant(value=1, kind=None)))], type_ignores="
"[])")
self.assertEqual(gast.dump(tree), norm)
def test_TryExcept(self):
code = 'try:pass\nexcept e:pass\nelse:pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
norm = ("Module(body=[Try(body=[Pass()], handlers=[ExceptHandler("
"type=Name(id='e', ctx=Load(), annotation=None, "
"type_comment=None), name=None, body=[Pass()])]"
", orelse=[Pass()], finalbody=[])], type_ignores=[])")
self.assertEqual(gast.dump(tree), norm)
def _delete_keywords_from(node):
assert isinstance(node, gast.Call)
func_src = astor.to_source(gast.gast_to_ast(node.func))
import paddle.fluid as fluid
full_args = eval("inspect.getargspec({})".format(func_src))
full_args_name = full_args[0]
node.keywords = [k for k in node.keywords if k.arg in full_args_name]
return
def to_source(node, indentation=' ' * 4):
"""Return source code of a given AST."""
if isinstance(node, gast.AST):
node = gast.gast_to_ast(node)
generator = SourceWithCommentGenerator(indentation, False,
astor.string_repr.pretty_string)
generator.visit(node)
generator.result.append('\n')
return astor.source_repr.pretty_source(generator.result).lstrip()
def ast_to_source(node, indentation):
"""Return the source code of given AST."""
if isinstance(node, gast.AST):
node = gast.gast_to_ast(node)
generator = astor.codegen.SourceGenerator(indentation, False,
astor.string_repr.pretty_string)
generator.visit(node)
generator.result.append('\n')
return astor.source_repr.pretty_source(generator.result).lstrip()
def ast_to_source(node, indentation):
"""Return the source code of given AST."""
if isinstance(node, gast.AST):
node = gast.gast_to_ast(node)
generator = astor.codegen.SourceGenerator(indentation, False,
astor.string_repr.pretty_string)
generator.visit(node)
generator.result.append('\n')
return astor.source_repr.pretty_source(generator.result).lstrip()
def test_Raise(self):
codes = ('raise Exception',
'raise "Exception"',
'raise Exception, "err"',
'raise Exception("err")',
'raise E, V, T',)
for code in codes:
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
def code_gast_ast(source):
"""
Transform source_code into gast.Node and modify it,
then back to ast.Node.
"""
source = textwrap.dedent(source)
root = gast.parse(source)
new_root = GastNodeTransformer(root).apply()
ast_root = gast.gast_to_ast(new_root)
return ast.dump(ast_root)
def test_Raise(self):
codes = ('raise Exception',
'raise "Exception"',
'raise Exception, "err"',
'raise Exception("err")',
'raise E, V, T',)
for code in codes:
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_With(self):
code = 'with open("any"): pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
norm = ("Module(body=[With(items=[withitem(context_expr=Call(func="
"Name(id='open', ctx=Load(), annotation=None, "
"type_comment=None), args=[Constant(value='any', "
"kind=None)], keywords=[]), optional_vars=None)], body=["
"Pass()], type_comment=None)], type_ignores=[])")
self.assertEqual(gast.dump(tree), norm)
def test_TypeIgnore(self):
code = 'def foo(): pass # type: ignore[excuse]'
tree = gast.parse(code, type_comments=True)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
norm = ("Module(body=[FunctionDef(name='foo', args=arguments("
"args=[], posonlyargs=[], vararg=None, kwonlyargs=[], "
"kw_defaults=[], kwarg=None, defaults=[]), body=["
"Pass()], decorator_list=[], returns=None, "
"type_comment=None)], type_ignores="
"[TypeIgnore(lineno=1, tag='[excuse]')])")
self.assertEqual(gast.dump(tree), norm)
def test_KeywordOnlyArgument(self):
code = 'def foo(*, x=1): pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
norm = ("Module(body=[FunctionDef(name='foo', args=arguments(args="
"[], posonlyargs=[], vararg=None, kwonlyargs=[Name"
"(id='x', ctx=Param(), annotation=None, type_comment=None"
")], kw_defaults=[Constant(value=1, kind=None)], kwarg="
"None, defaults=[]), body=[Pass()], decorator_list=[], "
"returns=None, type_comment=None)], type_ignores=[])")
self.assertEqual(gast.dump(tree), norm)
def test_keyword_argument(self):
code = 'def foo(**a): pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
norm = ("Module(body=[FunctionDef(name='foo', args=arguments(args=[], "
"posonlyargs=[], vararg=None, kwonlyargs=[], kw_defaults=[], "
"kwarg=Name(id='a', ctx=Param(), annotation=None, "
"type_comment=None), defaults=[]), body=[Pass()], "
"decorator_list=[], returns=None, type_comment=None)], "
"type_ignores=[])")
self.assertEqual(gast.dump(tree), norm)
def ast_to_source(node, indentation):
"""Return the source code of given AST."""
if isinstance(node, gast.AST):
node = gast.gast_to_ast(node)
generator = astor.codegen.SourceGenerator(indentation, False,
astor.string_repr.pretty_string)
generator.visit(node)
generator.result.append('\n')
# In some versions of Python, literals may appear as actual values. This
# ensures everything is string.
code = map(str, generator.result)
return astor.source_repr.pretty_source(code).lstrip()
def visit_For(self, node):
# if the user added some OpenMP directive, trust him and no unroll
if metadata.get(node, OMPDirective):
return node # don't visit children because of collapse
# first unroll children if needed or possible
self.generic_visit(node)
# a break or continue in the loop prevents unrolling too
has_break = any(self.gather(HasBreak, n)
for n in node.body)
has_cont = any(self.gather(HasContinue, n)
for n in node.body)
if has_break or has_cont:
return node
# do not unroll too much to prevent code growth
node_count = self.gather(NodeCount, node)
def unroll(elt):
return ([ast.Assign([deepcopy(node.target)], elt)] +
deepcopy(node.body))
def getrange(n):
return getattr(getattr(n, 'func', None), 'attr', None)
if isinstance(node.iter, (ast.Tuple, ast.List)):
total_count = node_count * len(node.iter.elts)
issmall = total_count < LoopFullUnrolling.MAX_NODE_COUNT
if issmall:
self.update = True
return sum([unroll(elt) for elt in node.iter.elts], [])
code = compile(ast.gast_to_ast(ast.Expression(node.iter)),
'<loop unrolling>', 'eval')
try:
values = list(eval(code,
{'__builtin__': __import__('__builtin__')}))
except Exception:
return node
total_count = node_count * len(values)
issmall = total_count < LoopFullUnrolling.MAX_NODE_COUNT
if issmall:
try:
new_node = sum([unroll(to_ast(elt)) for elt in values], [])
self.update = True
return new_node
except Exception:
return node
return node
def generic_visit(self, node):
if node in self.constant_expressions:
try:
fake_node = ast.Expression(
node.value if isinstance(node, ast.Index) else node)
code = compile(ast.gast_to_ast(fake_node),
'<constant folding>', 'eval')
value = eval(code, self.env)
new_node = to_ast(value)
if(isinstance(node, ast.Index) and
not isinstance(new_node, ast.Index)):
new_node = ast.Index(new_node)
try:
if not ASTMatcher(node).search(new_node):
self.update = True
return new_node
except DamnTooLongPattern as e:
print("W: ", e, " Assume no update happened.")
return Transformation.generic_visit(self, node)
except ConversionError as e:
print('error in constant folding: ', e)
raise
except ToNotEval:
return Transformation.generic_visit(self, node)
except AttributeError as e:
# FIXME union_ function is not handle by constant folding
if "union_" in e.args[0]:
return Transformation.generic_visit(self, node)
elif "pythran" in e.args[0]:
# FIXME: Can be fix giving a Python implementation for
# these functions.
return Transformation.generic_visit(self, node)
raise
except NameError as e:
# FIXME dispatched function are not processed by constant
# folding
if "__dispatch__" in e.args[0]:
return Transformation.generic_visit(self, node)
raise
except Exception as e:
raise PythranSyntaxError(str(e), node)
else:
return Transformation.generic_visit(self, node)
def ast_to_source(node, indentation=' '):
"""Return the source code of given AST.
Args:
node: The code to compile, as an AST object.
indentation: The string to use for indentation.
Returns:
code: The source code generated from the AST object
source_mapping: A mapping between the user and AutoGraph generated code.
"""
if not isinstance(node, (list, tuple)):
node = (node,)
generator = astor.codegen.SourceGenerator(indentation, False,
astor.string_repr.pretty_string)
for n in node:
if isinstance(n, gast.AST):
n = gast.gast_to_ast(n)
generator.visit(n)
generator.result.append('\n')
# In some versions of Python, literals may appear as actual values. This
# ensures everything is string.
code = ''.join(map(str, generator.result))
# Strip leading blank lines.
code_lines = code.split('\n')
trimmed_code_lines = []
for l in code_lines:
if l.rstrip() or trimmed_code_lines:
trimmed_code_lines.append(l)
code = '\n'.join(trimmed_code_lines)
# Work around the reference cycle generated by astor.
# See https://github.com/berkerpeksag/astor/blob/55dd323f7d8d696610c703c0296763c567685c31/astor/code_gen.py#L162 # pylint:disable=line-too-long
# Reference cycles are quite disliked by TensorFlow's tests.
if hasattr(generator, 'write'):
generator.write = None
del generator
return code
def prepare(self, node):
assert isinstance(node, ast.Module)
self.env = {
'__builtin__': __import__('__builtin__'),
}
for module_name in MODULES:
# __dispatch__ is the only fake top-level module
if module_name != '__dispatch__':
import_name = module_name
# handle module name conflicting with c++ keywords
if(module_name.endswith("_") and
module_name[:-1] in cxx_keywords):
import_name = module_name[:-1]
alias_module_name = mangle(module_name)
self.env[alias_module_name] = __import__(import_name)
# handle functions conflicting with c++ keywords
for fun in MODULES[module_name]:
if fun in ("__theitemgetter__", "pythran"):
# these ones do not exist in Python
continue
# Set attributs pointing to another for C++ keyword
# case of __builtin__.int_ that point on __builtin__.int
if not hasattr(self.env[alias_module_name], fun):
setattr(self.env[alias_module_name], fun,
getattr(self.env[alias_module_name],
fun.strip("_")))
# we need to parse the whole code to be able to apply user-defined pure
# function but import are resolved before so we remove them to avoid
# ImportError (for operator_ for example)
dummy_module = ast.Module([s for s in node.body
if not isinstance(s, ast.Import)])
eval(compile(ast.gast_to_ast(dummy_module),
'<constant_folding>', 'exec'),
self.env)
super(ConstantFolding, self).prepare(node)
def testCompile(self):
for src_py in self.srcs:
with open(src_py) as f:
content = f.read()
gnode = gast.parse(content)
compile(gast.gast_to_ast(gnode), src_py, 'exec')
def test_With(self):
code = 'with open("any"): pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_TryFinally(self):
code = 'try:pass\nfinally:pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_ArgAnnotation(self):
code = 'def foo(x:int): pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_KeywordOnlyArgument(self):
code = 'def foo(*, x=1): pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_TryExceptNamed(self):
code = 'try:pass\nexcept e as f:pass\nelse:pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_Call(self):
code = 'foo(x, y=1, *args, **kwargs)'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_FormattedValue(self):
code = 'e = 1; f"{e}"'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_JoinedStr(self):
code = 'e = 1; f"e = {e}"'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_keyword_argument(self):
code = 'def foo(**a): pass'
tree = gast.parse(code)
compile(gast.gast_to_ast(tree), '<test>', 'exec')
gast.dump(tree, include_attributes=True)
def test_unparse(self):
for src_py in self.srcs:
with open(src_py) as f:
content = f.read()
gnode = gast.parse(content)
astunparse.unparse(gast.gast_to_ast(gnode))
