def test_delete_kwarg(self): # expected expected = ''' def test2(x, y): data4 = paddle.tensor.add(x, y, name=None, out=None, alpha=1) ''' # captured input input_test = ''' def test2(x, y): data4 = paddle.fluid.layers.elementwise_add(x, y, axis=-1, act=None, name=None) ''' # get captured root = gast.parse(input_test) root_expected = gast.parse(expected) print("*******test_2_source_code*******") # print(inspect.getsource(test1)) root = transformer(root, modify_dict) print("*******test_delete_kwarg*******") print(astor.to_source(gast.gast_to_ast(root))) captured = astor.to_source(gast.gast_to_ast(root)) expected = astor.to_source(gast.gast_to_ast(root_expected)) # testting self.assertEqual(expected, captured)
def test_rename_kwarg(self): # expected expected = ''' def test3(x, y): data4 = paddle.tensor.argmax(input=x, axis=2, out=None, dtype=None, keepdims=False, name=None) ''' # captured input input_test = ''' def test3(x, y): data4 = paddle.fluid.layers.argmax(x=x, axis=2) ''' # get captured root = gast.parse(input_test) root_expected = gast.parse(expected) print("*******test_3_source_code*******") # print(inspect.getsource(test1)) root = transformer(root, modify_dict) print("*******test_rename_kwarg*******") print(astor.to_source(gast.gast_to_ast(root))) captured = astor.to_source(gast.gast_to_ast(root)) expected = astor.to_source(gast.gast_to_ast(root_expected)) # testting self.assertEqual(expected, captured)
def test_add_kwarg(self): # expected expected = ''' def test1(): data4 = paddle.tensor.zeros(x=x, shape=[3, 2], dtype='float32', out=None, device=None) ''' # captured input input_test = ''' def test1(): data4 = paddle.fluid.layers.zeros(x=x, shape=[3, 2], dtype='float32', force_cpu=True) ''' # get captured # root = gast.parse(inspect.getsource(input_test)) root = gast.parse(input_test) root_expected = gast.parse(expected) print("*******test_1_source_code*******") # print(inspect.getsource(test1)) root = transformer(root, modify_dict) print("*******test_add_kwarg*******") print(astor.to_source(gast.gast_to_ast(root))) captured = astor.to_source(gast.gast_to_ast(root)) expected = astor.to_source(gast.gast_to_ast(root_expected)) # testting self.assertEqual(expected, captured)
def _to_array_write_node(self, node): assert isinstance(node, gast.Call) array = astor.to_source(gast.gast_to_ast(node.func.value)) x = astor.to_source(gast.gast_to_ast(node.args[0])) i = "fluid.layers.array_length({})".format(array) func_code = "fluid.layers.array_write(x={}, i={}, array={})".format( x, i, array) return gast.parse(func_code).body[0].value
def test_replace_full_name_transformer(self, import_map=import_dict): # expected expected = ''' import paddle import paddle.fluid as fluid import numpy as np from paddle.fluid.dygraph.nn import Conv2D, Pool2D, Linear class LeNet(fluid.dygraph.Layer): def __init__(self, name_scope, num_classes=1): super(LeNet, self).__init__(name_scope) self.conv1 = paddle.fluid.dygraph.nn.Conv2D(num_channels=1, num_filters=6, filter_size=5, act='sigmoid') self.pool1 = paddle.fluid.dygraph.nn.Pool2D(pool_size=2, pool_stride=2, pool_type='max') self.conv2 = paddle.fluid.dygraph.nn.Conv2D(num_channels=6, num_filters=16, filter_size=5, act='sigmoid') self.pool2 = paddle.fluid.dygraph.nn.Pool2D(pool_size=2, pool_stride=2, pool_type='max') self.conv3 = paddle.fluid.dygraph.nn.Conv2D(num_channels=16, num_filters=120, filter_size=4, act='sigmoid') self.fc1 = paddle.fluid.dygraph.nn.Linear(input_dim=120, output_dim=64, act='sigmoid') self.fc2 = paddle.fluid.dygraph.nn.Linear(input_dim=64, output_dim=num_classes) def forward(self, x): x = self.conv1(x) x = self.pool1(x) x = self.conv2(x) x = self.pool2(x) x = self.conv3(x) x = paddle.fluid.layers.reshape(x, [x.shape[0], -1]) x = self.fc1(x) x = self.fc2(x) return x ''' # captured input input_test = gast.parse(inspect.getsource(dygraph_lenet)) print("*******original code*******") original_code = astor.to_source(gast.gast_to_ast(input_test)) original_code_formated = FormatCode(original_code) print(original_code_formated[0]) # get captured root = gast.parse(input_test) root_expected = gast.parse(expected) print("*******start_transfer*******") root = transfomer_replace_test(root, import_map) print("*******code_for_captured_formated*******") # print(astor.to_source(gast.gast_to_ast(root))) captured = astor.to_source(gast.gast_to_ast(root)) captured_formated = FormatCode(captured) print(captured_formated[0]) expected = astor.to_source(gast.gast_to_ast(root_expected)) expected_formated = FormatCode(expected) print("*******code_for_expected_formated*******") print(expected_formated[0]) # testting self.assertEqual(expected_formated, captured_formated)
def test_fusion_defDes(self): # expected expected = ''' def test1(x): data1 = paddle.nn.data(x=x) data2 = paddle.tensor.reshape(x=data1, shape=[1, 2, 3]) data4 = paddle.tensor.zeros(shape=[3, 2], dtype='float32', out=None, device=None) return abc def test2(x): data1 = paddle.tensor.tanh(input=x, out=None) data2 = paddle.tensor.zeros(shape=[3, 2], dtype='float32', out=None, device=None) def test3(x): data1 = paddle.nn.Sigmoid(x=x) data2 = paddle.tensor.sum(x, axis=0) def test4(x): data1 = paddle.tensor.max(input, dim=None, keep_dim=False, out=None, name=None) data2 = paddle.tensor.stack(x, dim=0, out=None) ''' # captured input input_test = ''' def test1(x): data1 = paddle.fluid.layers.data(x=x) data2 = paddle.fluid.layers.reshape(x=data1, shape=[1, 2, 3]) data3 = paddle.fluid.layers.zeros(shape=[3, 2], dtype='float32') data4 = paddle.fluid.layers.zeros(shape=[3, 2], dtype='float32', force_cpu=True) return abc def test2(x): data1 = paddle.fluid.layers.tanh(x=x) data2 = paddle.fluid.layers.zeros(shape=[3, 2], dtype='float32') def test3(x): data1 = paddle.fluid.layers.sigmoid(x=x) data2 = paddle.fluid.layers.reduce_sum(x, dim=0) def test4(x): data1 = paddle.fluid.layers.reduce_max(x=x) data2 = fluid.layers.stack(x, dim=0) ''' # get captured root = gast.parse(input_test) root_expected = gast.parse(expected) print("*******test_4_source_code*******") # print(inspect.getsource(test1)) root = transformer(root, modify_dict) print("*******test_fusion_function_kwarg*******") print(astor.to_source(gast.gast_to_ast(root))) captured = astor.to_source(gast.gast_to_ast(root)) expected = astor.to_source(gast.gast_to_ast(root_expected)) # testting self.assertEqual(expected, captured)
def test_single_model_file(self): parser = argparse.ArgumentParser("Paddle API upgrade") parser.add_argument("--modify_dict", type=str, default="../dict/modify.dict") parser.add_argument("--input", type=str, default='class_cnn.py') parser.add_argument("--output", type=str, default='class_cnn_update.py') args = parser.parse_args() # read captured with open('class_cnn.py', 'r') as fp: data_captured = fp.read() with open('class_cnn_expected.py', 'r') as fp: data_expected = fp.read() # expected expected = data_expected # captured input input_test = data_captured # # get captured root = gast.parse(input_test) root_expected = gast.parse(expected) print("*******test_5_source_code*******") root = transformer(root, modify_dict) print("*******test_model_single_file*******") print(astor.to_source(gast.gast_to_ast(root))) captured = astor.to_source(gast.gast_to_ast(root)) captured_formated = FormatCode(captured) with open('class_cnn_update.py', 'w') as fp: fp.write(captured_formated[0]) expected = astor.to_source(gast.gast_to_ast(root_expected)) data_expected_formated = FormatCode(captured) # read captured with open('class_cnn_update.py', 'r') as fp: data_captured = fp.read() captured = astor.to_source(gast.gast_to_ast(root)) data_captured_formated = FormatCode(captured) # testting self.assertEqual(data_expected_formated, data_captured_formated)
def testCompile(self): srcs = glob.glob(os.path.join(gast.__path__[0], '*.py')) for src_py in srcs: with open(src_py) as f: content = f.read() gnode = gast.parse(content) compile(gast.gast_to_ast(gnode), src_py, 'exec')
def compile_file(source, globals_=None): """Compile by saving to file and importing that. Compiling the AST/source code this way ensures that the source code is readable by e.g. `pdb` or `inspect`. Args: source: The code to compile, either as a string or as an AST. globals_: A dictionary of variables that should be available as globals in the compiled module. They will be monkey patched after importing the module. Returns: A module object containing the compiled source code. """ if isinstance(source, gast.AST): source = astor.to_source(gast.gast_to_ast(source)) # Write source to temporary file tempdir = tempfile.mkdtemp() uuid = str(uuid4().hex[:4]) tmpname = os.path.join(tempdir, 'matchbox_%s.py' % uuid) with open(tmpname, 'w') as f: f.write(source) # Load the temporary file as a module module_name = 'matchbox_%s' % uuid if six.PY3: spec = util.spec_from_file_location(module_name, tmpname) m = util.module_from_spec(spec) spec.loader.exec_module(m) else: m = imp.load_source(module_name, tmpname) # Update the modules namespace if globals_: m.__dict__.update(globals_) return m
def test_TryFinally(self): code = 'try:pass\nfinally:pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec') norm = ("Module(body=[Try(body=[Pass()], handlers=[], orelse=[], " "finalbody=[Pass()])], type_ignores=[])") self.assertEqual(gast.dump(tree), norm)
def ast_to_source(node, indentation=' '): """Return the source code of given AST. Args: node: The code to compile, as an AST object. indentation: The string to use for indentation. Returns: code: The source code generated from the AST object source_mapping: A mapping between the user and AutoGraph generated code. """ original_node = node if isinstance(node, gast.AST): node = gast.gast_to_ast(node) generator = astor.codegen.SourceGenerator(indentation, False, astor.string_repr.pretty_string) generator.visit(node) generator.result.append('\n') # In some versions of Python, literals may appear as actual values. This # ensures everything is string. code = map(str, generator.result) code = astor.source_repr.pretty_source(code).lstrip() source_mapping = _build_source_map(original_node, code) return code, source_mapping
def prepare(self, node): assert isinstance(node, ast.Module) self.env = { 'builtins': __import__('builtins'), } for module_name in MODULES: # __dispatch__ is the only fake top-level module if module_name != '__dispatch__': import_name = module_name alias_module_name = mangle(module_name) self.env[alias_module_name] = __import__(import_name) # handle functions conflicting with c++ keywords for fun in MODULES[module_name]: if fun in ("__theitemgetter__", "pythran"): # these ones do not exist in Python continue # we need to parse the whole code to be able to apply user-defined pure # function but import are resolved before so we remove them to avoid # ImportError (for operator_ for example) dummy_module = ast.Module([s for s in node.body if not isinstance(s, ast.Import)], []) eval(compile(ast.gast_to_ast(dummy_module), '<constant_folding>', 'exec'), self.env) super(ConstantFolding, self).prepare(node)
def ast_to_source(node, indentation=' '): """Return the source code of given AST. Args: node: The code to compile, as an AST object. indentation: The string to use for indentation. Returns: code: The source code generated from the AST object source_mapping: A mapping between the user and AutoGraph generated code. """ if not isinstance(node, (list, tuple)): node = (node, ) generator = astor.codegen.SourceGenerator(indentation, False, astor.string_repr.pretty_string) for n in node: if isinstance(n, gast.AST): n = gast.gast_to_ast(n) generator.visit(n) generator.result.append('\n') # In some versions of Python, literals may appear as actual values. This # ensures everything is string. code = map(str, generator.result) code = astor.source_repr.pretty_source(code).lstrip() return code
def generic_visit(self, node): if isinstance(node, ast.expr) and node in self.constant_expressions: fake_node = ast.Expression(node) code = compile(ast.gast_to_ast(fake_node), '<constant folding>', 'eval') try: value = eval(code, self.env) new_node = to_ast(value) if not ASTMatcher(node).match(new_node): self.update = True return new_node except DamnTooLongPattern as e: print("W: ", e, " Assume no update happened.") except ConversionError as e: print('error in constant folding: ', e) raise except ToNotEval: pass except AttributeError as e: # this may miss a few optimization logger.info('During constant folding, bailing out due to: ' + e.args[0]) except NameError as e: # FIXME dispatched function are not processed by constant # folding if "__dispatch__" in e.args[0]: return Transformation.generic_visit(self, node) # this may miss a few optimization logger.info('During constant folding, bailing out due to: ' + e.args[0]) except Exception as e: raise PythranSyntaxError(str(e), node) return Transformation.generic_visit(self, node)
def visit_For(self, node): # if the user added some OpenMP directive, trust him and no unroll if metadata.get(node, OMPDirective): return node # don't visit children because of collapse # first unroll children if needed or possible self.generic_visit(node) # a break or continue in the loop prevents unrolling too has_break = any(self.gather(HasBreak, n) for n in node.body) has_cont = any(self.gather(HasContinue, n) for n in node.body) if has_break or has_cont: return node # do not unroll too much to prevent code growth node_count = self.gather(NodeCount, node) def unroll(elt, body): return [ast.Assign([deepcopy(node.target)], elt)] + body def dc(body, i, n): if i == n - 1: return body else: return deepcopy(body) def getrange(n): return getattr(getattr(n, 'func', None), 'attr', None) if isinstance(node.iter, (ast.Tuple, ast.List)): elts_count = len(node.iter.elts) total_count = node_count * elts_count issmall = total_count < LoopFullUnrolling.MAX_NODE_COUNT if issmall: self.update = True return sum([unroll(elt, dc(node.body, i, elts_count)) for i, elt in enumerate(node.iter.elts)], []) code = compile(ast.gast_to_ast(ast.Expression(node.iter)), '<loop unrolling>', 'eval') try: values = list(eval(code, {'builtins': __import__('builtins')})) except Exception: return node values_count = len(values) total_count = node_count * values_count issmall = total_count < LoopFullUnrolling.MAX_NODE_COUNT if issmall: try: new_node = sum([unroll(to_ast(elt), dc(node.body, i, values_count)) for i, elt in enumerate(values)], []) self.update = True return new_node except Exception: return node return node
def unparse(node, indentation=None, include_encoding_marker=True): """Returns the source code of given AST. Args: node: The code to compile, as an AST object. indentation: Unused, deprecated. The returning code will always be indented at 4 spaces. include_encoding_marker: Bool, whether to include a comment on the first line to explicitly specify UTF-8 encoding. Returns: code: The source code generated from the AST object source_mapping: A mapping between the user and AutoGraph generated code. """ del indentation # astunparse doesn't allow configuring it. if not isinstance(node, (list, tuple)): node = (node,) codes = [] if include_encoding_marker: codes.append('# coding=utf-8') for n in node: if isinstance(n, gast.AST): ast_n = gast.gast_to_ast(n) else: ast_n = n if astunparse is ast: ast.fix_missing_locations(ast_n) # Only ast needs to call this. codes.append(astunparse.unparse(ast_n).strip()) return '\n'.join(codes)
def prepare(self, node): assert isinstance(node, ast.Module) self.env = { 'builtins': __import__('builtins'), } for module_name in MODULES: # __dispatch__ is the only fake top-level module if module_name != '__dispatch__': alias_module_name = mangle(module_name) try: self.env[alias_module_name] = __import__(module_name) except ImportError: pass # we need to parse the whole code to be able to apply user-defined pure # function but import are resolved before so we remove them to avoid # ImportError (for operator_ for example) dummy_module = ast.Module([s for s in node.body if not isinstance(s, ast.Import)], []) eval(compile(ast.gast_to_ast(dummy_module), '<constant_folding>', 'exec'), self.env) super(ConstantFolding, self).prepare(node)
def is_api_in_module(node, module_prefix): assert isinstance(node, gast.Call), "Input non-Call node for is_dygraph_api" # Python can have gast.Call as function, for example: covert_call(func)(x) # We only check the most outside function func_node = node.func while isinstance(func_node, gast.Call): func_node = func_node.func func_str = astor.to_source(gast.gast_to_ast(func_node)).strip() try: # TODO(liym27): # Consider a better to import modules like: # source_file = inspect.getfile(dyfunc) # import_statements = ImportVisitor(source_file).transform() # import_str = "".join(import_statements) import paddle import paddle.fluid as fluid import paddle.fluid.dygraph as dygraph import paddle.fluid.layers as layers from paddle.fluid.dygraph import to_variable from paddle import to_tensor return eval("_is_api_in_module_helper({}, '{}')".format( func_str, module_prefix)) except NameError: return False
def test_Bytes(self): code = 'b"0012"' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec') norm = ("Module(body=[Expr(value=Constant(value=b'0012', " "kind=None))], type_ignores=[])") self.assertEqual(gast.dump(tree), norm)
def ast_to_source(node, indentation=' '): """Return the source code of given AST. Args: node: The code to compile, as an AST object. indentation: The string to use for indentation. Returns: code: The source code generated from the AST object source_mapping: A mapping between the user and AutoGraph generated code. """ if not isinstance(node, (list, tuple)): node = (node,) generator = astor.codegen.SourceGenerator(indentation, False, astor.string_repr.pretty_string) for n in node: if isinstance(n, gast.AST): n = gast.gast_to_ast(n) generator.visit(n) generator.result.append('\n') # In some versions of Python, literals may appear as actual values. This # ensures everything is string. code = ''.join(map(str, generator.result)) # Strip leading blank lines. code_lines = code.split('\n') trimmed_code_lines = [] for l in code_lines: if l.rstrip() or trimmed_code_lines: trimmed_code_lines.append(l) code = '\n'.join(trimmed_code_lines) return code
def make_ast( target_ast_node_count, rng = None, distribution = ( python_numbers_control_flow.DATAFLOW_FNS_DISTRIBUTION) ): """Generates an AST for this task. Args: target_ast_node_count: How many nodes to put in the AST. rng: Random state to use. distribution: Sampling distribution to use when building the AST. May also be a callable that produces a distribution given a random state. Returns: AST of a generated program. """ def root_build(body): """Given a list of statements, puts them into a function in a module.""" return gast.Module( body=[ gast.FunctionDef( name="random_function", args=_make_arguments( python_numbers_control_flow.make_name("a"), python_numbers_control_flow.make_name("b")), body=body, decorator_list=[], returns=None, type_comment=None) ], type_ignores=[]) root_template = python_numbers_control_flow.ASTWithHoles( cost=5, holes=[ top_down_refinement.Hole( python_numbers_control_flow.ASTHoleType.STMTS_NONEMPTY, python_numbers_control_flow.ASTHoleMetadata(("a", "b"), True, False, 0)) ], build=root_build) if rng is None: rng = np.random.RandomState() if callable(distribution): distribution = distribution(rng) tree = top_down_refinement.top_down_construct( root_object=root_template, target_cost=target_ast_node_count, refinement_distribution=distribution, rng=rng) # Re-parse the tree so that it is valid. This is required for program graph # analysis to work. return gast.parse(astunparse.unparse(gast.gast_to_ast(tree)))
def test_NamedExpr(self): code = '(x := 1) ' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec') norm = ("Module(body=[Expr(value=NamedExpr(target=Name(id='x'," " ctx=Store(), annotation=None, type_comment=None), " "value=Constant(value=1, kind=None)))], type_ignores=" "[])") self.assertEqual(gast.dump(tree), norm)
def test_TryExcept(self): code = 'try:pass\nexcept e:pass\nelse:pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec') norm = ("Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(" "type=Name(id='e', ctx=Load(), annotation=None, " "type_comment=None), name=None, body=[Pass()])]" ", orelse=[Pass()], finalbody=[])], type_ignores=[])") self.assertEqual(gast.dump(tree), norm)
def _delete_keywords_from(node): assert isinstance(node, gast.Call) func_src = astor.to_source(gast.gast_to_ast(node.func)) import paddle.fluid as fluid full_args = eval("inspect.getargspec({})".format(func_src)) full_args_name = full_args[0] node.keywords = [k for k in node.keywords if k.arg in full_args_name] return
def to_source(node, indentation=' ' * 4): """Return source code of a given AST.""" if isinstance(node, gast.AST): node = gast.gast_to_ast(node) generator = SourceWithCommentGenerator(indentation, False, astor.string_repr.pretty_string) generator.visit(node) generator.result.append('\n') return astor.source_repr.pretty_source(generator.result).lstrip()
def ast_to_source(node, indentation): """Return the source code of given AST.""" if isinstance(node, gast.AST): node = gast.gast_to_ast(node) generator = astor.codegen.SourceGenerator(indentation, False, astor.string_repr.pretty_string) generator.visit(node) generator.result.append('\n') return astor.source_repr.pretty_source(generator.result).lstrip()
def test_Raise(self): codes = ('raise Exception', 'raise "Exception"', 'raise Exception, "err"', 'raise Exception("err")', 'raise E, V, T',) for code in codes: tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec')
def code_gast_ast(source): """ Transform source_code into gast.Node and modify it, then back to ast.Node. """ source = textwrap.dedent(source) root = gast.parse(source) new_root = GastNodeTransformer(root).apply() ast_root = gast.gast_to_ast(new_root) return ast.dump(ast_root)
def test_With(self): code = 'with open("any"): pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec') norm = ("Module(body=[With(items=[withitem(context_expr=Call(func=" "Name(id='open', ctx=Load(), annotation=None, " "type_comment=None), args=[Constant(value='any', " "kind=None)], keywords=[]), optional_vars=None)], body=[" "Pass()], type_comment=None)], type_ignores=[])") self.assertEqual(gast.dump(tree), norm)
def test_TypeIgnore(self): code = 'def foo(): pass # type: ignore[excuse]' tree = gast.parse(code, type_comments=True) compile(gast.gast_to_ast(tree), '<test>', 'exec') norm = ("Module(body=[FunctionDef(name='foo', args=arguments(" "args=[], posonlyargs=[], vararg=None, kwonlyargs=[], " "kw_defaults=[], kwarg=None, defaults=[]), body=[" "Pass()], decorator_list=[], returns=None, " "type_comment=None)], type_ignores=" "[TypeIgnore(lineno=1, tag='[excuse]')])") self.assertEqual(gast.dump(tree), norm)
def test_KeywordOnlyArgument(self): code = 'def foo(*, x=1): pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec') norm = ("Module(body=[FunctionDef(name='foo', args=arguments(args=" "[], posonlyargs=[], vararg=None, kwonlyargs=[Name" "(id='x', ctx=Param(), annotation=None, type_comment=None" ")], kw_defaults=[Constant(value=1, kind=None)], kwarg=" "None, defaults=[]), body=[Pass()], decorator_list=[], " "returns=None, type_comment=None)], type_ignores=[])") self.assertEqual(gast.dump(tree), norm)
def test_keyword_argument(self): code = 'def foo(**a): pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec') norm = ("Module(body=[FunctionDef(name='foo', args=arguments(args=[], " "posonlyargs=[], vararg=None, kwonlyargs=[], kw_defaults=[], " "kwarg=Name(id='a', ctx=Param(), annotation=None, " "type_comment=None), defaults=[]), body=[Pass()], " "decorator_list=[], returns=None, type_comment=None)], " "type_ignores=[])") self.assertEqual(gast.dump(tree), norm)
def ast_to_source(node, indentation): """Return the source code of given AST.""" if isinstance(node, gast.AST): node = gast.gast_to_ast(node) generator = astor.codegen.SourceGenerator(indentation, False, astor.string_repr.pretty_string) generator.visit(node) generator.result.append('\n') # In some versions of Python, literals may appear as actual values. This # ensures everything is string. code = map(str, generator.result) return astor.source_repr.pretty_source(code).lstrip()
def visit_For(self, node): # if the user added some OpenMP directive, trust him and no unroll if metadata.get(node, OMPDirective): return node # don't visit children because of collapse # first unroll children if needed or possible self.generic_visit(node) # a break or continue in the loop prevents unrolling too has_break = any(self.gather(HasBreak, n) for n in node.body) has_cont = any(self.gather(HasContinue, n) for n in node.body) if has_break or has_cont: return node # do not unroll too much to prevent code growth node_count = self.gather(NodeCount, node) def unroll(elt): return ([ast.Assign([deepcopy(node.target)], elt)] + deepcopy(node.body)) def getrange(n): return getattr(getattr(n, 'func', None), 'attr', None) if isinstance(node.iter, (ast.Tuple, ast.List)): total_count = node_count * len(node.iter.elts) issmall = total_count < LoopFullUnrolling.MAX_NODE_COUNT if issmall: self.update = True return sum([unroll(elt) for elt in node.iter.elts], []) code = compile(ast.gast_to_ast(ast.Expression(node.iter)), '<loop unrolling>', 'eval') try: values = list(eval(code, {'__builtin__': __import__('__builtin__')})) except Exception: return node total_count = node_count * len(values) issmall = total_count < LoopFullUnrolling.MAX_NODE_COUNT if issmall: try: new_node = sum([unroll(to_ast(elt)) for elt in values], []) self.update = True return new_node except Exception: return node return node
def generic_visit(self, node): if node in self.constant_expressions: try: fake_node = ast.Expression( node.value if isinstance(node, ast.Index) else node) code = compile(ast.gast_to_ast(fake_node), '<constant folding>', 'eval') value = eval(code, self.env) new_node = to_ast(value) if(isinstance(node, ast.Index) and not isinstance(new_node, ast.Index)): new_node = ast.Index(new_node) try: if not ASTMatcher(node).search(new_node): self.update = True return new_node except DamnTooLongPattern as e: print("W: ", e, " Assume no update happened.") return Transformation.generic_visit(self, node) except ConversionError as e: print('error in constant folding: ', e) raise except ToNotEval: return Transformation.generic_visit(self, node) except AttributeError as e: # FIXME union_ function is not handle by constant folding if "union_" in e.args[0]: return Transformation.generic_visit(self, node) elif "pythran" in e.args[0]: # FIXME: Can be fix giving a Python implementation for # these functions. return Transformation.generic_visit(self, node) raise except NameError as e: # FIXME dispatched function are not processed by constant # folding if "__dispatch__" in e.args[0]: return Transformation.generic_visit(self, node) raise except Exception as e: raise PythranSyntaxError(str(e), node) else: return Transformation.generic_visit(self, node)
def ast_to_source(node, indentation=' '): """Return the source code of given AST. Args: node: The code to compile, as an AST object. indentation: The string to use for indentation. Returns: code: The source code generated from the AST object source_mapping: A mapping between the user and AutoGraph generated code. """ if not isinstance(node, (list, tuple)): node = (node,) generator = astor.codegen.SourceGenerator(indentation, False, astor.string_repr.pretty_string) for n in node: if isinstance(n, gast.AST): n = gast.gast_to_ast(n) generator.visit(n) generator.result.append('\n') # In some versions of Python, literals may appear as actual values. This # ensures everything is string. code = ''.join(map(str, generator.result)) # Strip leading blank lines. code_lines = code.split('\n') trimmed_code_lines = [] for l in code_lines: if l.rstrip() or trimmed_code_lines: trimmed_code_lines.append(l) code = '\n'.join(trimmed_code_lines) # Work around the reference cycle generated by astor. # See https://github.com/berkerpeksag/astor/blob/55dd323f7d8d696610c703c0296763c567685c31/astor/code_gen.py#L162 # pylint:disable=line-too-long # Reference cycles are quite disliked by TensorFlow's tests. if hasattr(generator, 'write'): generator.write = None del generator return code
def prepare(self, node): assert isinstance(node, ast.Module) self.env = { '__builtin__': __import__('__builtin__'), } for module_name in MODULES: # __dispatch__ is the only fake top-level module if module_name != '__dispatch__': import_name = module_name # handle module name conflicting with c++ keywords if(module_name.endswith("_") and module_name[:-1] in cxx_keywords): import_name = module_name[:-1] alias_module_name = mangle(module_name) self.env[alias_module_name] = __import__(import_name) # handle functions conflicting with c++ keywords for fun in MODULES[module_name]: if fun in ("__theitemgetter__", "pythran"): # these ones do not exist in Python continue # Set attributs pointing to another for C++ keyword # case of __builtin__.int_ that point on __builtin__.int if not hasattr(self.env[alias_module_name], fun): setattr(self.env[alias_module_name], fun, getattr(self.env[alias_module_name], fun.strip("_"))) # we need to parse the whole code to be able to apply user-defined pure # function but import are resolved before so we remove them to avoid # ImportError (for operator_ for example) dummy_module = ast.Module([s for s in node.body if not isinstance(s, ast.Import)]) eval(compile(ast.gast_to_ast(dummy_module), '<constant_folding>', 'exec'), self.env) super(ConstantFolding, self).prepare(node)
def testCompile(self): for src_py in self.srcs: with open(src_py) as f: content = f.read() gnode = gast.parse(content) compile(gast.gast_to_ast(gnode), src_py, 'exec')
def test_With(self): code = 'with open("any"): pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_TryFinally(self): code = 'try:pass\nfinally:pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_ArgAnnotation(self): code = 'def foo(x:int): pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_KeywordOnlyArgument(self): code = 'def foo(*, x=1): pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_TryExceptNamed(self): code = 'try:pass\nexcept e as f:pass\nelse:pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_Call(self): code = 'foo(x, y=1, *args, **kwargs)' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_FormattedValue(self): code = 'e = 1; f"{e}"' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_JoinedStr(self): code = 'e = 1; f"e = {e}"' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec')
def test_keyword_argument(self): code = 'def foo(**a): pass' tree = gast.parse(code) compile(gast.gast_to_ast(tree), '<test>', 'exec') gast.dump(tree, include_attributes=True)
def test_unparse(self): for src_py in self.srcs: with open(src_py) as f: content = f.read() gnode = gast.parse(content) astunparse.unparse(gast.gast_to_ast(gnode))