def pypy_parse(source, mode='exec', lineno=False): from pypy.interpreter.pyparser.pythonparse import PythonParser, make_pyparser # parser = build_parser_for_version("2.4", PythonParser()) parser = make_pyparser('stable') builder = TupleBuilder(parser) parser.parse_source(source, mode, builder) return builder.stack[-1].as_tuple(lineno)
def source2ast(source, mode='exec', version='2.4', space=None): from pypy.interpreter.pyparser.pythonparse import PythonParser, make_pyparser from pypy.interpreter.pyparser.astbuilder import AstBuilder parser = make_pyparser(version) builder = AstBuilder(parser, version, space=space) parser.parse_source(source, mode, builder) return builder.rule_stack[-1]
def source2ast(source, mode='exec', version='2.4', space=None): from pypy.interpreter.pyparser.pythonparse import PythonParser, make_pyparser from pypy.interpreter.pyparser.astbuilder import AstBuilder parser = make_pyparser(version) builder = AstBuilder(parser, version, space=space) parser.parse_source(source, mode, builder) return builder.rule_stack[-1]
def pypy_parse(source, mode='exec', lineno=False): from pypy.interpreter.pyparser.pythonparse import PythonParser, make_pyparser # parser = build_parser_for_version("2.4", PythonParser()) parser = make_pyparser('stable') builder = TupleBuilder(parser) parser.parse_source(source, mode, builder) return builder.stack[-1].as_tuple(lineno)
def parse_python_source(space, source, mode): parser = make_pyparser(space.config.objspace.pyversion) builder = grammar.BaseGrammarBuilder(debug=False, parser=parser) builder.space = space try: parser.parse_source(source, mode, builder) return builder.stack[-1] except SyntaxError, e: raise OperationError(space.w_SyntaxError, e.wrap_info(space, '<string>'))
def parse_python_source(space, source, mode): parser = make_pyparser(space.config.objspace.pyversion) builder = grammar.BaseGrammarBuilder(debug=False, parser=parser) builder.space = space try: parser.parse_source(source, mode, builder) return builder.stack[-1] except SyntaxError, e: raise OperationError(space.w_SyntaxError, e.wrap_info(space, '<string>'))
def _init_symbols(grammar_version): global sym_name sym_name = {} from pypy.interpreter.pyparser.pythonparse import make_pyparser parser = make_pyparser(grammar_version) for name, val in parser.symbols.items(): if val >= 0: Module.interpleveldefs[name] = 'space.wrap(%d)' % val sym_name[val] = name Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,)
def _init_symbols(grammar_version): global sym_name sym_name = {} from pypy.interpreter.pyparser.pythonparse import make_pyparser parser = make_pyparser(grammar_version) for name, val in parser.symbols.items(): if val >= 0: Module.interpleveldefs[name] = 'space.wrap(%d)' % val sym_name[val] = name Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,)
"a[100, 1:2:3,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2), Const(3)])])))]))", "a[100, :2:3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2), Const(3)])])))]))", "a[100, 1::3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None), Const(3)])])))]))", "a[100, 1:2:,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2), Const(None)])])))]))", "a[100, 1:2]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2)])])))]))", "a[100, 1:]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None)])])))]))", "a[100, :2,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2)])])))]))", "a[100, :]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(None)])])))]))", # stablecompiler produces a Pass statement which does not seem very consistent # (a module should only have a Stmt child) "\t # hello\n": "Module(None, Stmt([]))", } # Create parser from Grammar_stable, not current grammar. stable_parser = pythonparse.make_pyparser('stable') def tuple_parse_expr(expr, target='single'): t = Transformer("dummyfile") return ast_from_input(expr, target, t, stable_parser) def source2ast(source, mode, space=FakeSpace()): version = '2.4' python_parser = pythonparse.make_pyparser(version) builder = AstBuilder(python_parser, version, space=space) python_parser.parse_source(source, mode, builder) return builder.rule_stack[-1] def check_expression(expr, mode='single'): pypy_ast = source2ast(expr, mode) try:
"a[100, :2:3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2), Const(3)])])))]))", "a[100, 1::3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None), Const(3)])])))]))", "a[100, 1:2:,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2), Const(None)])])))]))", "a[100, 1:2]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2)])])))]))", "a[100, 1:]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None)])])))]))", "a[100, :2,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2)])])))]))", "a[100, :]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(None)])])))]))", # stablecompiler produces a Pass statement which does not seem very consistent # (a module should only have a Stmt child) "\t # hello\n": "Module(None, Stmt([]))", } # Create parser from Grammar_stable, not current grammar. stable_parser = pythonparse.make_pyparser('stable') python_parser = pythonparse.make_pyparser() # 'native') # 2.5a') def tuple_parse_expr(expr, target='single'): t = Transformer("dummyfile") return ast_from_input(expr, target, t, stable_parser) def source2ast(source, mode, space=FakeSpace()): builder = AstBuilder(space=space, parser=python_parser) python_parser.parse_source(source, mode, builder) return builder.rule_stack[-1] def check_expression(expr, mode='single'): pypy_ast = source2ast(expr, mode) try: python_ast = EXPECTED[expr]
def source2ast(source, mode, space=FakeSpace(), version='2.5'): python_parser = pythonparse.make_pyparser(version) builder = AstBuilder(python_parser, version, space=space) python_parser.parse_source(source, mode, builder) return builder.rule_stack[-1]
from pypy.interpreter.pyparser.pythonlexer import Source, TokenError, \ match_encoding_declaration from pypy.interpreter.pyparser.grammar import Token, GrammarElement from pypy.interpreter.pyparser.pythonparse import make_pyparser, _check_for_encoding P = make_pyparser('2.4') EQUAL = P.tokens['EQUAL'] ENDMARKER = P.tokens['ENDMARKER'] LSQB = P.tokens['LSQB'] MINUS = P.tokens['MINUS'] NAME = P.tokens['NAME'] NEWLINE = P.tokens['NEWLINE'] NULLTOKEN = P.tokens['NULLTOKEN'] NUMBER = P.tokens['NUMBER'] RSQB = P.tokens['RSQB'] STRING = P.tokens['STRING'] def parse_source(source): """returns list of parsed tokens""" lexer = Source( P, source.splitlines(True), {}) tokens = [] last_token = Token( P, NULLTOKEN, None) while last_token.codename != ENDMARKER: last_token = lexer.next() tokens.append(last_token) return tokens ## class TestSuite: ## """Tokenizer test suite""" PUNCTS = [
from pypy.interpreter.pyparser.pythonlexer import Source, TokenError, \ match_encoding_declaration from pypy.interpreter.pyparser.grammar import Token, GrammarElement from pypy.interpreter.pyparser.pythonparse import make_pyparser P = make_pyparser() EQUAL = P.tokens['EQUAL'] ENDMARKER = P.tokens['ENDMARKER'] LSQB = P.tokens['LSQB'] MINUS = P.tokens['MINUS'] NAME = P.tokens['NAME'] NEWLINE = P.tokens['NEWLINE'] NULLTOKEN = P.tokens['NULLTOKEN'] NUMBER = P.tokens['NUMBER'] RSQB = P.tokens['RSQB'] STRING = P.tokens['STRING'] def parse_source(source): """returns list of parsed tokens""" lexer = Source( P, source.splitlines(True)) tokens = [] last_token = Token( P, NULLTOKEN, None) while last_token.codename != ENDMARKER: last_token = lexer.next() tokens.append(last_token) return tokens ## class TestSuite: ## """Tokenizer test suite""" PUNCTS = [
"a[:, 100]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Sliceobj([Const(None), Const(None)]), Const(100)])))]))", "a[100, 1:2:3,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2), Const(3)])])))]))", "a[100, :2:3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2), Const(3)])])))]))", "a[100, 1::3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None), Const(3)])])))]))", "a[100, 1:2:,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2), Const(None)])])))]))", "a[100, 1:2]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2)])])))]))", "a[100, 1:]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None)])])))]))", "a[100, :2,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2)])])))]))", "a[100, :]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(None)])])))]))", # stablecompiler produces a Pass statement which does not seem very consistent # (a module should only have a Stmt child) "\t # hello\n": "Module(None, Stmt([]))", } # Create parser from Grammar_stable, not current grammar. stable_parser = pythonparse.make_pyparser("stable") def tuple_parse_expr(expr, target="single"): t = Transformer("dummyfile") return ast_from_input(expr, target, t, stable_parser) def source2ast(source, mode, space=FakeSpace()): version = "2.4" python_parser = pythonparse.make_pyparser(version) builder = AstBuilder(python_parser, version, space=space) python_parser.parse_source(source, mode, builder) return builder.rule_stack[-1]
def compile_with_astcompiler(expr, mode='exec', space=FakeSpace()): ast = source2ast(expr, mode, space) # xxx exec: single not really tested, mumble misc.set_filename('<?>', ast) if mode == 'exec': Generator = pycodegen.ModuleCodeGenerator elif mode == 'single': Generator = pycodegen.InteractiveCodeGenerator elif mode == 'eval': Generator = pycodegen.ExpressionCodeGenerator codegen = Generator(space, ast) rcode = codegen.getCode() return rcode # Create parser from Grammar_stable, not current grammar. stable_parser = pythonparse.make_pyparser('stable') def compile_with_testcompiler(expr, mode='exec', space=FakeSpace()): mode2 = 'exec' # xxx exec: single not really tested builder = TupleBuilder(stable_parser) stable_parser.parse_source(expr, mode2, builder) tuples = builder.stack[-1].as_tuple(True) from pypy.interpreter.stablecompiler import transformer, pycodegen, misc ast = transformer.Transformer('<?>').compile_node(tuples) misc.set_filename('<?>', ast) if mode == 'exec': Generator = pycodegen.ModuleCodeGenerator elif mode == 'single': Generator = pycodegen.InteractiveCodeGenerator elif mode == 'eval': Generator = pycodegen.ExpressionCodeGenerator
def source2ast(source, mode, space=FakeSpace()): version = '2.4' python_parser = pythonparse.make_pyparser(version) builder = AstBuilder(python_parser, version, space=space) python_parser.parse_source(source, mode, builder) return builder.rule_stack[-1]
def setup_class(self): from pypy.interpreter.pyparser.pythonparse import make_pyparser self.parser = make_pyparser('2.5a')
# make it so that pyparser matches the 'parser' module interface from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, W_Root from pypy.interpreter.gateway import interp2app, applevel from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef from pypy.interpreter.typedef import interp_attrproperty, GetSetProperty from pypy.interpreter.pycode import PyCode from pypy.interpreter.pyparser.syntaxtree import TokenNode, SyntaxNode, AbstractSyntaxVisitor from pypy.interpreter.pyparser.pythonparse import make_pyparser from pypy.interpreter.pyparser.error import SyntaxError from pypy.interpreter.pyparser import grammar, symbol, pytoken from pypy.interpreter.argument import Arguments # backward compat (temp) PYTHON_PARSER = make_pyparser() __all__ = [ "ASTType", "STType", "suite", "expr" ] class SyntaxToTupleVisitor(AbstractSyntaxVisitor): def __init__(self, space, line_info): self.space = space self.line_info = line_info self.tuple_stack_w = [] def w_result( self ): return self.tuple_stack_w[-1] def visit_syntaxnode( self, node ): space = self.space