def get_ast(self, source, p_mode="exec", flags=None): if flags is None: flags = consts.CO_FUTURE_WITH_STATEMENT info = pyparse.CompileInfo("<test>", p_mode, flags) tree = self.parser.parse_source(source, info) ast_node = ast_from_node(self.space, tree, info, self.parser) return ast_node
def f_string_compile(astbuilder, source, atom_node): # Note: a f-string is kept as a single literal up to here. # At this point only, we recursively call the AST compiler # on all the '{expr}' parts. The 'expr' part is not parsed # or even tokenized together with the rest of the source code! from pypy.interpreter.pyparser import pyparse from pypy.interpreter.astcompiler.astbuilder import ast_from_node # complain if 'source' is only whitespace or an empty string for c in source: if c not in ' \t\n\r\v\f': break else: astbuilder.error("f-string: empty expression not allowed", atom_node) if astbuilder.recursive_parser is None: astbuilder.error( "internal error: parser not available for parsing " "the expressions inside the f-string", atom_node) assert isinstance(source, str) # utf-8 encoded source = '(%s)' % source info = pyparse.CompileInfo("<fstring>", "eval", consts.PyCF_SOURCE_IS_UTF8 | consts.PyCF_IGNORE_COOKIE, optimize=astbuilder.compile_info.optimize) parser = astbuilder.recursive_parser parse_tree = parser.parse_source(source, info) return ast_from_node(astbuilder.space, parse_tree, info, recursive_parser=parser)
def descr_compile(self, space, filename="<syntax-tree>"): info = pyparse.CompileInfo(filename, self.mode) try: ast = ast_from_node(space, self.tree, info) result = compile_ast(space, ast, info) except error.IndentationError, e: raise OperationError(space.w_IndentationError, e.wrap_info(space))
def _compile_to_ast(self, source, info): space = self.space try: parse_tree = self.parser.parse_source(source, info) mod = astbuilder.ast_from_node(space, parse_tree, info) except parseerror.TabError, e: raise OperationError(space.w_TabError, e.wrap_info(space))
def descr_compile(self, space, filename="<syntax-tree>"): info = pyparse.CompileInfo(filename, self.mode) try: ast = ast_from_node(space, self.tree, info) result = compile_ast(space, ast, info) except error.IndentationError, e: raise OperationError(space.w_IndentationError, e.wrap_info(space))
def _compile_to_ast(self, source, info): space = self.space try: parse_tree = self.parser.parse_source(source, info) mod = astbuilder.ast_from_node(space, parse_tree, info) except parseerror.IndentationError, e: raise OperationError(space.w_IndentationError, e.wrap_info(space))
def mod_scope(self, source, mode="exec"): info = pyparse.CompileInfo("<test>", mode, consts.CO_FUTURE_WITH_STATEMENT) tree = self.parser.parse_source(source, info) module = astbuilder.ast_from_node(self.space, tree, info) builder = symtable.SymtableBuilder(self.space, module, info) scope = builder.find_scope(module) assert isinstance(scope, symtable.ModuleScope) return scope
def mod_scope(self, source, mode="exec"): info = pyparse.CompileInfo("<test>", mode, consts.CO_FUTURE_WITH_STATEMENT) tree = self.parser.parse_source(source, info) module = astbuilder.ast_from_node(self.space, tree, info) builder = symtable.SymtableBuilder(self.space, module, info) scope = builder.find_scope(module) assert isinstance(scope, symtable.ModuleScope) return scope
def test_string_bug(self): space = self.space source = '# -*- encoding: utf8 -*-\nstuff = "x \xc3\xa9 \\n"\n' info = pyparse.CompileInfo("<test>", "exec") tree = self.parser.parse_source(source, info) assert info.encoding == "utf8" s = ast_from_node(space, tree, info).body[0].value assert isinstance(s, ast.Str) expected = ['x', ' ', chr(0xc3), chr(0xa9), ' ', '\n'] assert space.eq_w(s.s, space.wrap(''.join(expected)))
def _compile_to_ast(self, source, info): space = self.space try: parse_tree = self.parser.parse_source(source, info) mod = astbuilder.ast_from_node(space, parse_tree, info) except parseerror.IndentationError as e: raise OperationError(space.w_IndentationError, e.find_sourceline_and_wrap_info(space)) except parseerror.SyntaxError as e: raise OperationError(space.w_SyntaxError, e.find_sourceline_and_wrap_info(space, source)) return mod
def test_string_bug(self): space = self.space source = '# -*- encoding: utf8 -*-\nstuff = "x \xc3\xa9 \\n"\n' info = pyparse.CompileInfo("<test>", "exec") tree = self.parser.parse_source(source, info) assert info.encoding == "utf8" s = ast_from_node(space, tree, info).body[0].value assert isinstance(s, ast.Str) expected = ["x", " ", chr(0xC3), chr(0xA9), " ", "\n"] assert space.eq_w(s.s, space.wrap("".join(expected)))
def descr_compile(self, space, filename="<syntax-tree>"): info = pyparse.CompileInfo(filename, self.mode) try: ast = ast_from_node(space, self.tree, info, self.recursive_parser) result = compile_ast(space, ast, info) except error.IndentationError as e: raise OperationError(space.w_IndentationError, e.find_sourceline_and_wrap_info(space)) except error.SyntaxError as e: raise OperationError(space.w_SyntaxError, e.find_sourceline_and_wrap_info(space)) return result
def f_string_compile(astbuilder, source, atom_node, fstr): # Note: a f-string is kept as a single literal up to here. # At this point only, we recursively call the AST compiler # on all the '{expr}' parts. The 'expr' part is not parsed # or even tokenized together with the rest of the source code! from pypy.interpreter.pyparser import pyparse from pypy.interpreter.astcompiler.astbuilder import ast_from_node # complain if 'source' is only whitespace or an empty string for c in source: if c not in ' \t\n\r\v\f': break else: astbuilder.error("f-string: empty expression not allowed", atom_node) if astbuilder.recursive_parser is None: astbuilder.error( "internal error: parser not available for parsing " "the expressions inside the f-string", atom_node) assert isinstance(source, str) # utf-8 encoded paren_source = '(%s)' % source # to deal with whitespace at the start of source lineno = 0 column_offset = 0 if fstr.stnode: stnode = fstr.stnode lineno = stnode.get_lineno() - 1 # one-based # CPython has an equivalent hack :-( value = stnode.get_value() if value is not None: offset = value.find(source) assert offset >= 0 last_nl = max(0, value.rfind('\n', 0, offset)) column_offset = offset - last_nl + stnode.get_column() lineno += value.count('\n', 0, last_nl + 1) info = pyparse.CompileInfo("<fstring>", "eval", consts.PyCF_SOURCE_IS_UTF8 | consts.PyCF_IGNORE_COOKIE, optimize=astbuilder.compile_info.optimize) parser = astbuilder.recursive_parser parse_tree = parser.parse_source(paren_source, info) ast = ast_from_node(astbuilder.space, parse_tree, info, recursive_parser=parser) # column_offset - 1 to exclude prefixed ( in paren_source fixup_fstring_positions(ast, lineno, column_offset - 1) return ast
def generate_function_code(expr, space): p = pyparse.PythonParser(space) info = pyparse.CompileInfo("<test>", 'exec') cst = p.parse_source(expr, info) ast = astbuilder.ast_from_node(space, cst, info) function_ast = optimize.optimize_ast(space, ast.body[0], info) function_ast = ast.body[0] symbols = symtable.SymtableBuilder(space, ast, info) generator = codegen.FunctionCodeGenerator(space, 'function', function_ast, 1, symbols, info) blocks = generator.first_block.post_order() generator._resolve_block_targets(blocks) return generator, blocks
def generate_function_code(expr, space): p = pyparse.PythonParser(space) info = pyparse.CompileInfo("<test>", 'exec') cst = p.parse_source(expr, info) ast = astbuilder.ast_from_node(space, cst, info) function_ast = optimize.optimize_ast(space, ast.body[0], info) function_ast = ast.body[0] symbols = symtable.SymtableBuilder(space, ast, info) generator = codegen.FunctionCodeGenerator( space, 'function', function_ast, 1, symbols, info) blocks = generator.first_block.post_order() generator._resolve_block_targets(blocks) return generator, blocks
def _compile_to_ast(self, source, info): from pypy.interpreter.pyparser.future import get_futures from pypy.interpreter.pyparser.error import (SyntaxError, IndentationError, TokenIndentationError) from pypy.interpreter.astcompiler.astbuilder import ast_from_node space = self.space try: f_flags, future_info = get_futures(self.future_flags, source) info.last_future_import = future_info info.flags |= f_flags parse_tree = self.parser.parse_source(source, info) mod = ast_from_node(space, parse_tree, info) except IndentationError, e: raise OperationError(space.w_IndentationError, e.wrap_info(space))
def test_string(self): space = self.space s = self.get_first_expr("'hi'") assert isinstance(s, ast.Str) assert space.eq_w(s.s, space.wrap("hi")) s = self.get_first_expr("'hi' ' implicitly' ' extra'") assert isinstance(s, ast.Str) assert space.eq_w(s.s, space.wrap("hi implicitly extra")) sentence = u"Die Männer ärgen sich!" source = u"# coding: utf-7\nstuff = u'%s'" % (sentence,) info = pyparse.CompileInfo("<test>", "exec") tree = self.parser.parse_source(source.encode("utf-7"), info) assert info.encoding == "utf-7" s = ast_from_node(space, tree, info).body[0].value assert isinstance(s, ast.Str) assert space.eq_w(s.s, space.wrap(sentence))
def _compile_to_ast(self, source, info): from pypy.interpreter.pyparser.future import get_futures from pypy.interpreter.pyparser.error import (SyntaxError, IndentationError, TokenIndentationError) from pypy.interpreter.astcompiler.astbuilder import ast_from_node space = self.space try: f_flags, future_info = get_futures(self.future_flags, source) info.last_future_import = future_info info.flags |= f_flags parse_tree = self.parser.parse_source(source, info) mod = ast_from_node(space, parse_tree, info) except IndentationError, e: raise OperationError(space.w_IndentationError, e.wrap_info(space))
def test_string(self): space = self.space s = self.get_first_expr("'hi'") assert isinstance(s, ast.Str) assert space.eq_w(s.s, space.wrap("hi")) s = self.get_first_expr("'hi' ' implicitly' ' extra'") assert isinstance(s, ast.Str) assert space.eq_w(s.s, space.wrap("hi implicitly extra")) sentence = u"Die Männer ärgen sich!" source = u"# coding: utf-7\nstuff = u'%s'" % (sentence, ) info = pyparse.CompileInfo("<test>", "exec") tree = self.parser.parse_source(source.encode("utf-7"), info) assert info.encoding == "utf-7" s = ast_from_node(space, tree, info).body[0].value assert isinstance(s, ast.Str) assert space.eq_w(s.s, space.wrap(sentence))
def source_to_tree(source): parsetree = pyparser.parse_source(source, compile_info) tree = ast_from_node(space, parsetree, compile_info) return tree
def get_ast(self, source, p_mode="exec"): info = pyparse.CompileInfo("<test>", p_mode, consts.CO_FUTURE_WITH_STATEMENT) tree = self.parser.parse_source(source, info) ast_node = ast_from_node(self.space, tree, info) return ast_node
def descr_compile(self, space, filename="<syntax-tree>"): info = pyparse.CompileInfo(filename, self.mode) ast = ast_from_node(space, self.tree, info) return space.wrap(compile_ast(space, ast, info))
def compile_with_astcompiler(expr, mode, space): p = pyparse.PythonParser(space) info = pyparse.CompileInfo("<test>", mode) cst = p.parse_source(expr, info) ast = astbuilder.ast_from_node(space, cst, info) return codegen.compile_ast(space, ast, info)
def compile_with_astcompiler(expr, mode, space): p = pyparse.PythonParser(space) info = pyparse.CompileInfo("<test>", mode) cst = p.parse_source(expr, info) ast = astbuilder.ast_from_node(space, cst, info) return codegen.compile_ast(space, ast, info)
def source_to_tree(source): parsetree = pyparser.parse_source(source, compile_info) tree = ast_from_node(space, parsetree, compile_info) return tree
def get_ast(self, source, p_mode="exec"): info = pyparse.CompileInfo("<test>", p_mode, consts.CO_FUTURE_WITH_STATEMENT) tree = self.parser.parse_source(source, info) ast_node = ast_from_node(self.space, tree, info) return ast_node
answer = ["(Delete"] answer.append(sexpy(node.targets)) answer.append(")") return ' '.join(answer) if isinstance(node, list): answer = ["("] for item in node: answer.append(sexpy(item)) answer.append(")") return ' '.join(answer) if node == None: return "empty" return "'(" + node.__repr__() + ")" targ_src = file("targ.py").read() config, parser = option.get_standard_options() spc = option.make_objspace(config) p = pyparse.PythonParser(spc) info = pyparse.CompileInfo("<test>", "exec") tree = p.parse_source(targ_src, info) ast_node = ast_from_node(spc, tree, info) # this is going to be an ast.Module object print sexpy(ast_node)
def descr_compile(self, space, filename="<syntax-tree>"): info = pyparse.CompileInfo(filename, self.mode) ast = ast_from_node(space, self.tree, info) return space.wrap(compile_ast(space, ast, info))