def get_ast(self, source, p_mode="exec", flags=None):
if flags is None:
flags = consts.CO_FUTURE_WITH_STATEMENT
info = pyparse.CompileInfo("<test>", p_mode, flags)
tree = self.parser.parse_source(source, info)
ast_node = ast_from_node(self.space, tree, info, self.parser)
return ast_node
def f_string_compile(astbuilder, source, atom_node):
# Note: a f-string is kept as a single literal up to here.
# At this point only, we recursively call the AST compiler
# on all the '{expr}' parts. The 'expr' part is not parsed
# or even tokenized together with the rest of the source code!
from pypy.interpreter.pyparser import pyparse
from pypy.interpreter.astcompiler.astbuilder import ast_from_node
# complain if 'source' is only whitespace or an empty string
for c in source:
if c not in ' \t\n\r\v\f':
break
else:
astbuilder.error("f-string: empty expression not allowed", atom_node)
if astbuilder.recursive_parser is None:
astbuilder.error(
"internal error: parser not available for parsing "
"the expressions inside the f-string", atom_node)
assert isinstance(source, str) # utf-8 encoded
source = '(%s)' % source
info = pyparse.CompileInfo("<fstring>",
"eval",
consts.PyCF_SOURCE_IS_UTF8
| consts.PyCF_IGNORE_COOKIE,
optimize=astbuilder.compile_info.optimize)
parser = astbuilder.recursive_parser
parse_tree = parser.parse_source(source, info)
return ast_from_node(astbuilder.space,
parse_tree,
info,
recursive_parser=parser)
def descr_compile(self, space, filename="<syntax-tree>"):
info = pyparse.CompileInfo(filename, self.mode)
try:
ast = ast_from_node(space, self.tree, info)
result = compile_ast(space, ast, info)
except error.IndentationError, e:
raise OperationError(space.w_IndentationError, e.wrap_info(space))
def _compile_to_ast(self, source, info):
space = self.space
try:
parse_tree = self.parser.parse_source(source, info)
mod = astbuilder.ast_from_node(space, parse_tree, info)
except parseerror.TabError, e:
raise OperationError(space.w_TabError, e.wrap_info(space))
def descr_compile(self, space, filename="<syntax-tree>"):
info = pyparse.CompileInfo(filename, self.mode)
try:
ast = ast_from_node(space, self.tree, info)
result = compile_ast(space, ast, info)
except error.IndentationError, e:
raise OperationError(space.w_IndentationError,
e.wrap_info(space))
def _compile_to_ast(self, source, info):
space = self.space
try:
parse_tree = self.parser.parse_source(source, info)
mod = astbuilder.ast_from_node(space, parse_tree, info)
except parseerror.IndentationError, e:
raise OperationError(space.w_IndentationError,
e.wrap_info(space))
def mod_scope(self, source, mode="exec"):
info = pyparse.CompileInfo("<test>", mode,
consts.CO_FUTURE_WITH_STATEMENT)
tree = self.parser.parse_source(source, info)
module = astbuilder.ast_from_node(self.space, tree, info)
builder = symtable.SymtableBuilder(self.space, module, info)
scope = builder.find_scope(module)
assert isinstance(scope, symtable.ModuleScope)
return scope
def mod_scope(self, source, mode="exec"):
info = pyparse.CompileInfo("<test>", mode,
consts.CO_FUTURE_WITH_STATEMENT)
tree = self.parser.parse_source(source, info)
module = astbuilder.ast_from_node(self.space, tree, info)
builder = symtable.SymtableBuilder(self.space, module, info)
scope = builder.find_scope(module)
assert isinstance(scope, symtable.ModuleScope)
return scope
def test_string_bug(self):
space = self.space
source = '# -*- encoding: utf8 -*-\nstuff = "x \xc3\xa9 \\n"\n'
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parser.parse_source(source, info)
assert info.encoding == "utf8"
s = ast_from_node(space, tree, info).body[0].value
assert isinstance(s, ast.Str)
expected = ['x', ' ', chr(0xc3), chr(0xa9), ' ', '\n']
assert space.eq_w(s.s, space.wrap(''.join(expected)))
def _compile_to_ast(self, source, info):
space = self.space
try:
parse_tree = self.parser.parse_source(source, info)
mod = astbuilder.ast_from_node(space, parse_tree, info)
except parseerror.IndentationError as e:
raise OperationError(space.w_IndentationError, e.find_sourceline_and_wrap_info(space))
except parseerror.SyntaxError as e:
raise OperationError(space.w_SyntaxError, e.find_sourceline_and_wrap_info(space, source))
return mod
def test_string_bug(self):
space = self.space
source = '# -*- encoding: utf8 -*-\nstuff = "x \xc3\xa9 \\n"\n'
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parser.parse_source(source, info)
assert info.encoding == "utf8"
s = ast_from_node(space, tree, info).body[0].value
assert isinstance(s, ast.Str)
expected = ["x", " ", chr(0xC3), chr(0xA9), " ", "\n"]
assert space.eq_w(s.s, space.wrap("".join(expected)))
def descr_compile(self, space, filename="<syntax-tree>"):
info = pyparse.CompileInfo(filename, self.mode)
try:
ast = ast_from_node(space, self.tree, info, self.recursive_parser)
result = compile_ast(space, ast, info)
except error.IndentationError as e:
raise OperationError(space.w_IndentationError,
e.find_sourceline_and_wrap_info(space))
except error.SyntaxError as e:
raise OperationError(space.w_SyntaxError,
e.find_sourceline_and_wrap_info(space))
return result
def f_string_compile(astbuilder, source, atom_node, fstr):
# Note: a f-string is kept as a single literal up to here.
# At this point only, we recursively call the AST compiler
# on all the '{expr}' parts. The 'expr' part is not parsed
# or even tokenized together with the rest of the source code!
from pypy.interpreter.pyparser import pyparse
from pypy.interpreter.astcompiler.astbuilder import ast_from_node
# complain if 'source' is only whitespace or an empty string
for c in source:
if c not in ' \t\n\r\v\f':
break
else:
astbuilder.error("f-string: empty expression not allowed", atom_node)
if astbuilder.recursive_parser is None:
astbuilder.error(
"internal error: parser not available for parsing "
"the expressions inside the f-string", atom_node)
assert isinstance(source, str) # utf-8 encoded
paren_source = '(%s)' % source # to deal with whitespace at the start of source
lineno = 0
column_offset = 0
if fstr.stnode:
stnode = fstr.stnode
lineno = stnode.get_lineno() - 1 # one-based
# CPython has an equivalent hack :-(
value = stnode.get_value()
if value is not None:
offset = value.find(source)
assert offset >= 0
last_nl = max(0, value.rfind('\n', 0, offset))
column_offset = offset - last_nl + stnode.get_column()
lineno += value.count('\n', 0, last_nl + 1)
info = pyparse.CompileInfo("<fstring>",
"eval",
consts.PyCF_SOURCE_IS_UTF8
| consts.PyCF_IGNORE_COOKIE,
optimize=astbuilder.compile_info.optimize)
parser = astbuilder.recursive_parser
parse_tree = parser.parse_source(paren_source, info)
ast = ast_from_node(astbuilder.space,
parse_tree,
info,
recursive_parser=parser)
# column_offset - 1 to exclude prefixed ( in paren_source
fixup_fstring_positions(ast, lineno, column_offset - 1)
return ast
def generate_function_code(expr, space):
p = pyparse.PythonParser(space)
info = pyparse.CompileInfo("<test>", 'exec')
cst = p.parse_source(expr, info)
ast = astbuilder.ast_from_node(space, cst, info)
function_ast = optimize.optimize_ast(space, ast.body[0], info)
function_ast = ast.body[0]
symbols = symtable.SymtableBuilder(space, ast, info)
generator = codegen.FunctionCodeGenerator(space, 'function', function_ast,
1, symbols, info)
blocks = generator.first_block.post_order()
generator._resolve_block_targets(blocks)
return generator, blocks
def generate_function_code(expr, space):
p = pyparse.PythonParser(space)
info = pyparse.CompileInfo("<test>", 'exec')
cst = p.parse_source(expr, info)
ast = astbuilder.ast_from_node(space, cst, info)
function_ast = optimize.optimize_ast(space, ast.body[0], info)
function_ast = ast.body[0]
symbols = symtable.SymtableBuilder(space, ast, info)
generator = codegen.FunctionCodeGenerator(
space, 'function', function_ast, 1, symbols, info)
blocks = generator.first_block.post_order()
generator._resolve_block_targets(blocks)
return generator, blocks
def _compile_to_ast(self, source, info):
from pypy.interpreter.pyparser.future import get_futures
from pypy.interpreter.pyparser.error import (SyntaxError,
IndentationError,
TokenIndentationError)
from pypy.interpreter.astcompiler.astbuilder import ast_from_node
space = self.space
try:
f_flags, future_info = get_futures(self.future_flags, source)
info.last_future_import = future_info
info.flags |= f_flags
parse_tree = self.parser.parse_source(source, info)
mod = ast_from_node(space, parse_tree, info)
except IndentationError, e:
raise OperationError(space.w_IndentationError, e.wrap_info(space))
def test_string(self):
space = self.space
s = self.get_first_expr("'hi'")
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap("hi"))
s = self.get_first_expr("'hi' ' implicitly' ' extra'")
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap("hi implicitly extra"))
sentence = u"Die Männer ärgen sich!"
source = u"# coding: utf-7\nstuff = u'%s'" % (sentence,)
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parser.parse_source(source.encode("utf-7"), info)
assert info.encoding == "utf-7"
s = ast_from_node(space, tree, info).body[0].value
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap(sentence))
def _compile_to_ast(self, source, info):
from pypy.interpreter.pyparser.future import get_futures
from pypy.interpreter.pyparser.error import (SyntaxError,
IndentationError,
TokenIndentationError)
from pypy.interpreter.astcompiler.astbuilder import ast_from_node
space = self.space
try:
f_flags, future_info = get_futures(self.future_flags, source)
info.last_future_import = future_info
info.flags |= f_flags
parse_tree = self.parser.parse_source(source, info)
mod = ast_from_node(space, parse_tree, info)
except IndentationError, e:
raise OperationError(space.w_IndentationError,
e.wrap_info(space))
def test_string(self):
space = self.space
s = self.get_first_expr("'hi'")
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap("hi"))
s = self.get_first_expr("'hi' ' implicitly' ' extra'")
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap("hi implicitly extra"))
sentence = u"Die Männer ärgen sich!"
source = u"# coding: utf-7\nstuff = u'%s'" % (sentence, )
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parser.parse_source(source.encode("utf-7"), info)
assert info.encoding == "utf-7"
s = ast_from_node(space, tree, info).body[0].value
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap(sentence))
def source_to_tree(source):
parsetree = pyparser.parse_source(source, compile_info)
tree = ast_from_node(space, parsetree, compile_info)
return tree
def get_ast(self, source, p_mode="exec"):
info = pyparse.CompileInfo("<test>", p_mode,
consts.CO_FUTURE_WITH_STATEMENT)
tree = self.parser.parse_source(source, info)
ast_node = ast_from_node(self.space, tree, info)
return ast_node
def descr_compile(self, space, filename="<syntax-tree>"):
info = pyparse.CompileInfo(filename, self.mode)
ast = ast_from_node(space, self.tree, info)
return space.wrap(compile_ast(space, ast, info))
def compile_with_astcompiler(expr, mode, space):
p = pyparse.PythonParser(space)
info = pyparse.CompileInfo("<test>", mode)
cst = p.parse_source(expr, info)
ast = astbuilder.ast_from_node(space, cst, info)
return codegen.compile_ast(space, ast, info)
def compile_with_astcompiler(expr, mode, space):
p = pyparse.PythonParser(space)
info = pyparse.CompileInfo("<test>", mode)
cst = p.parse_source(expr, info)
ast = astbuilder.ast_from_node(space, cst, info)
return codegen.compile_ast(space, ast, info)
def source_to_tree(source):
parsetree = pyparser.parse_source(source, compile_info)
tree = ast_from_node(space, parsetree, compile_info)
return tree
def get_ast(self, source, p_mode="exec"):
info = pyparse.CompileInfo("<test>", p_mode,
consts.CO_FUTURE_WITH_STATEMENT)
tree = self.parser.parse_source(source, info)
ast_node = ast_from_node(self.space, tree, info)
return ast_node
answer = ["(Delete"]
answer.append(sexpy(node.targets))
answer.append(")")
return ' '.join(answer)
if isinstance(node, list):
answer = ["("]
for item in node:
answer.append(sexpy(item))
answer.append(")")
return ' '.join(answer)
if node == None:
return "empty"
return "'(" + node.__repr__() + ")"
targ_src = file("targ.py").read()
config, parser = option.get_standard_options()
spc = option.make_objspace(config)
p = pyparse.PythonParser(spc)
info = pyparse.CompileInfo("<test>", "exec")
tree = p.parse_source(targ_src, info)
ast_node = ast_from_node(spc, tree, info) # this is going to be an ast.Module object
print sexpy(ast_node)
def descr_compile(self, space, filename="<syntax-tree>"):
info = pyparse.CompileInfo(filename, self.mode)
ast = ast_from_node(space, self.tree, info)
return space.wrap(compile_ast(space, ast, info))
