def pypy_parse(source, mode='exec', lineno=False):
from pypy.interpreter.pyparser.pythonparse import PythonParser, make_pyparser
# parser = build_parser_for_version("2.4", PythonParser())
parser = make_pyparser('stable')
builder = TupleBuilder(parser)
parser.parse_source(source, mode, builder)
return builder.stack[-1].as_tuple(lineno)
def source2ast(source, mode='exec', version='2.4', space=None):
from pypy.interpreter.pyparser.pythonparse import PythonParser, make_pyparser
from pypy.interpreter.pyparser.astbuilder import AstBuilder
parser = make_pyparser(version)
builder = AstBuilder(parser, version, space=space)
parser.parse_source(source, mode, builder)
return builder.rule_stack[-1]
def source2ast(source, mode='exec', version='2.4', space=None):
from pypy.interpreter.pyparser.pythonparse import PythonParser, make_pyparser
from pypy.interpreter.pyparser.astbuilder import AstBuilder
parser = make_pyparser(version)
builder = AstBuilder(parser, version, space=space)
parser.parse_source(source, mode, builder)
return builder.rule_stack[-1]
def pypy_parse(source, mode='exec', lineno=False):
from pypy.interpreter.pyparser.pythonparse import PythonParser, make_pyparser
# parser = build_parser_for_version("2.4", PythonParser())
parser = make_pyparser('stable')
builder = TupleBuilder(parser)
parser.parse_source(source, mode, builder)
return builder.stack[-1].as_tuple(lineno)
def parse_python_source(space, source, mode):
parser = make_pyparser(space.config.objspace.pyversion)
builder = grammar.BaseGrammarBuilder(debug=False, parser=parser)
builder.space = space
try:
parser.parse_source(source, mode, builder)
return builder.stack[-1]
except SyntaxError, e:
raise OperationError(space.w_SyntaxError,
e.wrap_info(space, '<string>'))
def parse_python_source(space, source, mode):
parser = make_pyparser(space.config.objspace.pyversion)
builder = grammar.BaseGrammarBuilder(debug=False, parser=parser)
builder.space = space
try:
parser.parse_source(source, mode, builder)
return builder.stack[-1]
except SyntaxError, e:
raise OperationError(space.w_SyntaxError,
e.wrap_info(space, '<string>'))
def _init_symbols(grammar_version):
global sym_name
sym_name = {}
from pypy.interpreter.pyparser.pythonparse import make_pyparser
parser = make_pyparser(grammar_version)
for name, val in parser.symbols.items():
if val >= 0:
Module.interpleveldefs[name] = 'space.wrap(%d)' % val
sym_name[val] = name
Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,)
def _init_symbols(grammar_version):
global sym_name
sym_name = {}
from pypy.interpreter.pyparser.pythonparse import make_pyparser
parser = make_pyparser(grammar_version)
for name, val in parser.symbols.items():
if val >= 0:
Module.interpleveldefs[name] = 'space.wrap(%d)' % val
sym_name[val] = name
Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,)
"a[100, 1:2:3,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2), Const(3)])])))]))",
"a[100, :2:3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2), Const(3)])])))]))",
"a[100, 1::3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None), Const(3)])])))]))",
"a[100, 1:2:,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2), Const(None)])])))]))",
"a[100, 1:2]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2)])])))]))",
"a[100, 1:]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None)])])))]))",
"a[100, :2,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2)])])))]))",
"a[100, :]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(None)])])))]))",
# stablecompiler produces a Pass statement which does not seem very consistent
# (a module should only have a Stmt child)
"\t # hello\n": "Module(None, Stmt([]))",
}
# Create parser from Grammar_stable, not current grammar.
stable_parser = pythonparse.make_pyparser('stable')
def tuple_parse_expr(expr, target='single'):
t = Transformer("dummyfile")
return ast_from_input(expr, target, t, stable_parser)
def source2ast(source, mode, space=FakeSpace()):
version = '2.4'
python_parser = pythonparse.make_pyparser(version)
builder = AstBuilder(python_parser, version, space=space)
python_parser.parse_source(source, mode, builder)
return builder.rule_stack[-1]
def check_expression(expr, mode='single'):
pypy_ast = source2ast(expr, mode)
try:
"a[100, :2:3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2), Const(3)])])))]))",
"a[100, 1::3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None), Const(3)])])))]))",
"a[100, 1:2:,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2), Const(None)])])))]))",
"a[100, 1:2]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2)])])))]))",
"a[100, 1:]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None)])])))]))",
"a[100, :2,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2)])])))]))",
"a[100, :]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(None)])])))]))",
# stablecompiler produces a Pass statement which does not seem very consistent
# (a module should only have a Stmt child)
"\t # hello\n": "Module(None, Stmt([]))",
}
# Create parser from Grammar_stable, not current grammar.
stable_parser = pythonparse.make_pyparser('stable')
python_parser = pythonparse.make_pyparser() # 'native') # 2.5a')
def tuple_parse_expr(expr, target='single'):
t = Transformer("dummyfile")
return ast_from_input(expr, target, t, stable_parser)
def source2ast(source, mode, space=FakeSpace()):
builder = AstBuilder(space=space, parser=python_parser)
python_parser.parse_source(source, mode, builder)
return builder.rule_stack[-1]
def check_expression(expr, mode='single'):
pypy_ast = source2ast(expr, mode)
try:
python_ast = EXPECTED[expr]
def source2ast(source, mode, space=FakeSpace(), version='2.5'):
python_parser = pythonparse.make_pyparser(version)
builder = AstBuilder(python_parser, version, space=space)
python_parser.parse_source(source, mode, builder)
return builder.rule_stack[-1]
from pypy.interpreter.pyparser.pythonlexer import Source, TokenError, \
match_encoding_declaration
from pypy.interpreter.pyparser.grammar import Token, GrammarElement
from pypy.interpreter.pyparser.pythonparse import make_pyparser, _check_for_encoding
P = make_pyparser('2.4')
EQUAL = P.tokens['EQUAL']
ENDMARKER = P.tokens['ENDMARKER']
LSQB = P.tokens['LSQB']
MINUS = P.tokens['MINUS']
NAME = P.tokens['NAME']
NEWLINE = P.tokens['NEWLINE']
NULLTOKEN = P.tokens['NULLTOKEN']
NUMBER = P.tokens['NUMBER']
RSQB = P.tokens['RSQB']
STRING = P.tokens['STRING']
def parse_source(source):
"""returns list of parsed tokens"""
lexer = Source( P, source.splitlines(True), {})
tokens = []
last_token = Token( P, NULLTOKEN, None)
while last_token.codename != ENDMARKER:
last_token = lexer.next()
tokens.append(last_token)
return tokens
## class TestSuite:
## """Tokenizer test suite"""
PUNCTS = [
from pypy.interpreter.pyparser.pythonlexer import Source, TokenError, \
match_encoding_declaration
from pypy.interpreter.pyparser.grammar import Token, GrammarElement
from pypy.interpreter.pyparser.pythonparse import make_pyparser
P = make_pyparser()
EQUAL = P.tokens['EQUAL']
ENDMARKER = P.tokens['ENDMARKER']
LSQB = P.tokens['LSQB']
MINUS = P.tokens['MINUS']
NAME = P.tokens['NAME']
NEWLINE = P.tokens['NEWLINE']
NULLTOKEN = P.tokens['NULLTOKEN']
NUMBER = P.tokens['NUMBER']
RSQB = P.tokens['RSQB']
STRING = P.tokens['STRING']
def parse_source(source):
"""returns list of parsed tokens"""
lexer = Source( P, source.splitlines(True))
tokens = []
last_token = Token( P, NULLTOKEN, None)
while last_token.codename != ENDMARKER:
last_token = lexer.next()
tokens.append(last_token)
return tokens
## class TestSuite:
## """Tokenizer test suite"""
PUNCTS = [
"a[:, 100]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Sliceobj([Const(None), Const(None)]), Const(100)])))]))",
"a[100, 1:2:3,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2), Const(3)])])))]))",
"a[100, :2:3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2), Const(3)])])))]))",
"a[100, 1::3]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None), Const(3)])])))]))",
"a[100, 1:2:,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2), Const(None)])])))]))",
"a[100, 1:2]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(2)])])))]))",
"a[100, 1:]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(1), Const(None)])])))]))",
"a[100, :2,]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(2)])])))]))",
"a[100, :]": "Module(None, Stmt([Discard(Subscript(Name('a'), 2, Tuple([Const(100), Sliceobj([Const(None), Const(None)])])))]))",
# stablecompiler produces a Pass statement which does not seem very consistent
# (a module should only have a Stmt child)
"\t # hello\n": "Module(None, Stmt([]))",
}
# Create parser from Grammar_stable, not current grammar.
stable_parser = pythonparse.make_pyparser("stable")
def tuple_parse_expr(expr, target="single"):
t = Transformer("dummyfile")
return ast_from_input(expr, target, t, stable_parser)
def source2ast(source, mode, space=FakeSpace()):
version = "2.4"
python_parser = pythonparse.make_pyparser(version)
builder = AstBuilder(python_parser, version, space=space)
python_parser.parse_source(source, mode, builder)
return builder.rule_stack[-1]
def compile_with_astcompiler(expr, mode='exec', space=FakeSpace()):
ast = source2ast(expr, mode, space) # xxx exec: single not really tested, mumble
misc.set_filename('<?>', ast)
if mode == 'exec':
Generator = pycodegen.ModuleCodeGenerator
elif mode == 'single':
Generator = pycodegen.InteractiveCodeGenerator
elif mode == 'eval':
Generator = pycodegen.ExpressionCodeGenerator
codegen = Generator(space, ast)
rcode = codegen.getCode()
return rcode
# Create parser from Grammar_stable, not current grammar.
stable_parser = pythonparse.make_pyparser('stable')
def compile_with_testcompiler(expr, mode='exec', space=FakeSpace()):
mode2 = 'exec' # xxx exec: single not really tested
builder = TupleBuilder(stable_parser)
stable_parser.parse_source(expr, mode2, builder)
tuples = builder.stack[-1].as_tuple(True)
from pypy.interpreter.stablecompiler import transformer, pycodegen, misc
ast = transformer.Transformer('<?>').compile_node(tuples)
misc.set_filename('<?>', ast)
if mode == 'exec':
Generator = pycodegen.ModuleCodeGenerator
elif mode == 'single':
Generator = pycodegen.InteractiveCodeGenerator
elif mode == 'eval':
Generator = pycodegen.ExpressionCodeGenerator
def source2ast(source, mode, space=FakeSpace()):
version = '2.4'
python_parser = pythonparse.make_pyparser(version)
builder = AstBuilder(python_parser, version, space=space)
python_parser.parse_source(source, mode, builder)
return builder.rule_stack[-1]
def setup_class(self):
from pypy.interpreter.pyparser.pythonparse import make_pyparser
self.parser = make_pyparser('2.5a')
# make it so that pyparser matches the 'parser' module interface
from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, W_Root
from pypy.interpreter.gateway import interp2app, applevel
from pypy.interpreter.error import OperationError
from pypy.interpreter.typedef import TypeDef
from pypy.interpreter.typedef import interp_attrproperty, GetSetProperty
from pypy.interpreter.pycode import PyCode
from pypy.interpreter.pyparser.syntaxtree import TokenNode, SyntaxNode, AbstractSyntaxVisitor
from pypy.interpreter.pyparser.pythonparse import make_pyparser
from pypy.interpreter.pyparser.error import SyntaxError
from pypy.interpreter.pyparser import grammar, symbol, pytoken
from pypy.interpreter.argument import Arguments
# backward compat (temp)
PYTHON_PARSER = make_pyparser()
__all__ = [ "ASTType", "STType", "suite", "expr" ]
class SyntaxToTupleVisitor(AbstractSyntaxVisitor):
def __init__(self, space, line_info):
self.space = space
self.line_info = line_info
self.tuple_stack_w = []
def w_result( self ):
return self.tuple_stack_w[-1]
def visit_syntaxnode( self, node ):
space = self.space
