def parse_python(space, source, mode):
info = pyparse.CompileInfo("<string>", mode)
parser = pyparse.PythonParser(space)
try:
tree = parser.parse_source(source, info)
except error.IndentationError, e:
raise OperationError(space.w_IndentationError, e.wrap_info(space))
def get_ast(self, source, p_mode="exec", flags=None):
if flags is None:
flags = consts.CO_FUTURE_WITH_STATEMENT
info = pyparse.CompileInfo("<test>", p_mode, flags)
tree = self.parser.parse_source(source, info)
ast_node = ast_from_node(self.space, tree, info, self.parser)
return ast_node
def compile(self, source, filename, mode, flags, hidden_applevel=False):
info = pyparse.CompileInfo(filename,
mode,
flags,
hidden_applevel=hidden_applevel)
mod = self._compile_to_ast(source, info)
return self._compile_ast(mod, info)
def descr_compile(self, space, filename="<syntax-tree>"):
info = pyparse.CompileInfo(filename, self.mode)
try:
ast = ast_from_node(space, self.tree, info)
result = compile_ast(space, ast, info)
except error.IndentationError, e:
raise OperationError(space.w_IndentationError, e.wrap_info(space))
def f_string_compile(astbuilder, source, atom_node):
# Note: a f-string is kept as a single literal up to here.
# At this point only, we recursively call the AST compiler
# on all the '{expr}' parts. The 'expr' part is not parsed
# or even tokenized together with the rest of the source code!
from pypy.interpreter.pyparser import pyparse
from pypy.interpreter.astcompiler.astbuilder import ast_from_node
# complain if 'source' is only whitespace or an empty string
for c in source:
if c not in ' \t\n\r\v\f':
break
else:
astbuilder.error("f-string: empty expression not allowed", atom_node)
if astbuilder.recursive_parser is None:
astbuilder.error(
"internal error: parser not available for parsing "
"the expressions inside the f-string", atom_node)
assert isinstance(source, str) # utf-8 encoded
source = '(%s)' % source
info = pyparse.CompileInfo("<fstring>",
"eval",
consts.PyCF_SOURCE_IS_UTF8
| consts.PyCF_IGNORE_COOKIE,
optimize=astbuilder.compile_info.optimize)
parser = astbuilder.recursive_parser
parse_tree = parser.parse_source(source, info)
return ast_from_node(astbuilder.space,
parse_tree,
info,
recursive_parser=parser)
def bench(title):
a = time.clock()
info = pyparse.CompileInfo("<string>", "exec")
parser = pyparse.PythonParser(fakespace)
tree = parser._parse(s, info)
b = time.clock()
print title, (b - a)
def test_encoding(self):
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parse("""# coding: latin-1
stuff = "nothing"
""",
info=info)
assert tree.type == syms.file_input
assert info.encoding == "iso-8859-1"
sentence = u"u'Die Männer ärgen sich!'"
input = (u"# coding: utf-7\nstuff = %s" % (sentence, )).encode("utf-7")
tree = self.parse(input, info=info)
assert info.encoding == "utf-7"
input = "# coding: iso-8859-15\nx"
self.parse(input, info=info)
assert info.encoding == "iso-8859-15"
input = "\xEF\xBB\xBF# coding: utf-8\nx"
self.parse(input, info=info)
assert info.encoding == "utf-8"
input = "# coding: utf-8\nx"
info.flags |= consts.PyCF_SOURCE_IS_UTF8
exc = py.test.raises(SyntaxError, self.parse, input, info=info).value
info.flags &= ~consts.PyCF_SOURCE_IS_UTF8
assert exc.msg == "coding declaration in unicode string"
input = "\xEF\xBB\xBF# coding: latin-1\nx"
exc = py.test.raises(SyntaxError, self.parse, input).value
assert exc.msg == "UTF-8 BOM with latin-1 coding cookie"
input = "# coding: not-here"
exc = py.test.raises(SyntaxError, self.parse, input).value
assert exc.msg == "Unknown encoding: not-here"
input = u"# coding: ascii\n\xe2".encode('utf-8')
exc = py.test.raises(SyntaxError, self.parse, input).value
assert exc.msg == ("'ascii' codec can't decode byte 0xc3 "
"in position 16: ordinal not in range(128)")
def mod_scope(self, source, mode="exec"):
info = pyparse.CompileInfo("<test>", mode,
consts.CO_FUTURE_WITH_STATEMENT)
tree = self.parser.parse_source(source, info)
module = astbuilder.ast_from_node(self.space, tree, info)
builder = symtable.SymtableBuilder(self.space, module, info)
scope = builder.find_scope(module)
assert isinstance(scope, symtable.ModuleScope)
return scope
def parse_python(space, source, mode):
info = pyparse.CompileInfo("<string>", mode)
parser = pyparse.PythonParser(space)
try:
tree = parser.parse_source(source, info)
except error.IndentationError as e:
raise OperationError(space.w_IndentationError, e.wrap_info(space))
except error.SyntaxError as e:
raise OperationError(space.w_SyntaxError, e.wrap_info(space))
return W_STType(tree, mode, recursive_parser=parser)
def test_string_bug(self):
space = self.space
source = '# -*- encoding: utf8 -*-\nstuff = "x \xc3\xa9 \\n"\n'
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parser.parse_source(source, info)
assert info.encoding == "utf8"
s = ast_from_node(space, tree, info).body[0].value
assert isinstance(s, ast.Str)
expected = ['x', ' ', chr(0xc3), chr(0xa9), ' ', '\n']
assert space.eq_w(s.s, space.wrap(''.join(expected)))
def descr_compile(self, space, filename="<syntax-tree>"):
info = pyparse.CompileInfo(filename, self.mode)
try:
ast = ast_from_node(space, self.tree, info, self.recursive_parser)
result = compile_ast(space, ast, info)
except error.IndentationError as e:
raise OperationError(space.w_IndentationError,
e.find_sourceline_and_wrap_info(space))
except error.SyntaxError as e:
raise OperationError(space.w_SyntaxError,
e.find_sourceline_and_wrap_info(space))
return result
def f_string_compile(astbuilder, source, atom_node, fstr):
# Note: a f-string is kept as a single literal up to here.
# At this point only, we recursively call the AST compiler
# on all the '{expr}' parts. The 'expr' part is not parsed
# or even tokenized together with the rest of the source code!
from pypy.interpreter.pyparser import pyparse
from pypy.interpreter.astcompiler.astbuilder import ast_from_node
# complain if 'source' is only whitespace or an empty string
for c in source:
if c not in ' \t\n\r\v\f':
break
else:
astbuilder.error("f-string: empty expression not allowed", atom_node)
if astbuilder.recursive_parser is None:
astbuilder.error(
"internal error: parser not available for parsing "
"the expressions inside the f-string", atom_node)
assert isinstance(source, str) # utf-8 encoded
paren_source = '(%s)' % source # to deal with whitespace at the start of source
lineno = 0
column_offset = 0
if fstr.stnode:
stnode = fstr.stnode
lineno = stnode.get_lineno() - 1 # one-based
# CPython has an equivalent hack :-(
value = stnode.get_value()
if value is not None:
offset = value.find(source)
assert offset >= 0
last_nl = max(0, value.rfind('\n', 0, offset))
column_offset = offset - last_nl + stnode.get_column()
lineno += value.count('\n', 0, last_nl + 1)
info = pyparse.CompileInfo("<fstring>",
"eval",
consts.PyCF_SOURCE_IS_UTF8
| consts.PyCF_IGNORE_COOKIE,
optimize=astbuilder.compile_info.optimize)
parser = astbuilder.recursive_parser
parse_tree = parser.parse_source(paren_source, info)
ast = ast_from_node(astbuilder.space,
parse_tree,
info,
recursive_parser=parser)
# column_offset - 1 to exclude prefixed ( in paren_source
fixup_fstring_positions(ast, lineno, column_offset - 1)
return ast
def generate_function_code(expr, space):
p = pyparse.PythonParser(space)
info = pyparse.CompileInfo("<test>", 'exec')
cst = p.parse_source(expr, info)
ast = astbuilder.ast_from_node(space, cst, info)
function_ast = optimize.optimize_ast(space, ast.body[0], info)
function_ast = ast.body[0]
symbols = symtable.SymtableBuilder(space, ast, info)
generator = codegen.FunctionCodeGenerator(space, 'function', function_ast,
1, symbols, info)
blocks = generator.first_block.post_order()
generator._resolve_block_targets(blocks)
return generator, blocks
def test_string(self):
space = self.space
s = self.get_first_expr("'hi'")
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap("hi"))
s = self.get_first_expr("'hi' ' implicitly' ' extra'")
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap("hi implicitly extra"))
sentence = u"Die Männer ärgen sich!"
source = u"# coding: utf-7\nstuff = u'%s'" % (sentence, )
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parser.parse_source(source.encode("utf-7"), info)
assert info.encoding == "utf-7"
s = ast_from_node(space, tree, info).body[0].value
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap(sentence))
def compile(self,
source,
filename,
mode,
flags,
hidden_applevel=False,
optimize=-1):
if optimize == -1:
optimize = self.space.sys.get_optimize()
assert optimize >= 0
info = pyparse.CompileInfo(filename,
mode,
flags,
hidden_applevel=hidden_applevel,
optimize=optimize)
mod = self._compile_to_ast(source, info)
return self._compile_ast(mod, info, source)
def compile_ast(self, node, filename, mode, flags):
if mode == 'eval':
check = isinstance(node, ast.Expression)
elif mode == 'exec':
check = isinstance(node, ast.Module)
elif mode == 'input':
check = isinstance(node, ast.Interactive)
else:
check = True
if not check:
raise oefmt(self.space.w_TypeError, "invalid node type")
fut = misc.parse_future(node, self.future_flags.compiler_features)
f_flags, f_lineno, f_col = fut
future_pos = f_lineno, f_col
flags |= f_flags
info = pyparse.CompileInfo(filename, mode, flags, future_pos)
return self._compile_ast(node, info)
def test_dont_imply_dedent(self):
info = pyparse.CompileInfo("<test>", "single",
consts.PyCF_DONT_IMPLY_DEDENT)
self.parse('if 1:\n x\n', info=info)
self.parse('x = 5 ', info=info)
def parse(self, source, mode="exec", info=None):
if info is None:
info = pyparse.CompileInfo("<test>", mode)
return self.parser.parse_source(source, info)
def compile_to_ast(self, source, filename, mode, flags):
info = pyparse.CompileInfo(filename, mode, flags)
return self._compile_to_ast(source, info)
def compile_with_astcompiler(expr, mode, space):
p = pyparse.PythonParser(space)
info = pyparse.CompileInfo("<test>", mode)
cst = p.parse_source(expr, info)
ast = astbuilder.ast_from_node(space, cst, info)
return codegen.compile_ast(space, ast, info)
# <codecell>
from pypy.interpreter.pyparser import pyparse
from pypy.interpreter.astcompiler import ast
from pypy.interpreter.astcompiler.astbuilder import ast_from_node
from pypy.tool.pytest.objspace import TinyObjSpace
from .codecommon import normalize_name, assert_tree
from .ogm import OG
# <codecell>
# Source code to tree (AST)
space = TinyObjSpace()
pyparser = pyparse.PythonParser(space)
compile_info = pyparse.CompileInfo("<filename>", "exec")
def source_to_tree(source):
parsetree = pyparser.parse_source(source, compile_info)
tree = ast_from_node(space, parsetree, compile_info)
return tree
# <codecell>
# Find nodes that define new names
def make_def_finders():
# Class Definition
def ClassDef(node, links):
yield node.name
def get_ast(self, source, p_mode="exec"):
info = pyparse.CompileInfo("<test>", p_mode,
consts.CO_FUTURE_WITH_STATEMENT)
tree = self.parser.parse_source(source, info)
ast_node = ast_from_node(self.space, tree, info)
return ast_node
def descr_compile(self, space, filename="<syntax-tree>"):
info = pyparse.CompileInfo(filename, self.mode)
ast = ast_from_node(space, self.tree, info)
return space.wrap(compile_ast(space, ast, info))
def test_encoding_pep3120(self):
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parse("""foo = '日本'""", info=info)
assert info.encoding == 'utf-8'