def visit_Nonlocal(self, nonl):
for name in nonl.names:
old_role = self.scope.lookup_role(name)
msg = ""
if old_role & SYM_GLOBAL:
msg = "name '%s' is nonlocal and global" % (name, )
if old_role & SYM_PARAM:
msg = "name '%s' is parameter and nonlocal" % (name, )
if isinstance(self.scope, ModuleScope):
msg = "nonlocal declaration not allowed at module level"
if old_role & SYM_ANNOTATED:
msg = "annotated name '%s' can't be nonlocal" \
% (name,)
if msg is not "":
raise SyntaxError(msg, nonl.lineno, nonl.col_offset)
if (old_role & (SYM_USED | SYM_ASSIGNED)
and not (name == '__class__'
and self.scope._hide_bound_from_nested_scopes)):
if old_role & SYM_ASSIGNED:
msg = "name '%s' is assigned to before nonlocal declaration" \
% (name,)
else:
msg = "name '%s' is used prior to nonlocal declaration" % \
(name,)
raise SyntaxError(msg, nonl.lineno, nonl.col_offset)
self.note_symbol(name, SYM_NONLOCAL)
def visit_Global(self, glob):
for name in glob.names:
old_role = self.scope.lookup_role(name)
if (self.scope._hide_bound_from_nested_scopes
and name == '__class__'):
msg = ("'global __class__' inside a class statement is not "
"implemented in PyPy")
raise SyntaxError(msg,
glob.lineno,
glob.col_offset,
filename=self.compile_info.filename)
if old_role & SYM_PARAM:
msg = "name '%s' is parameter and global" % (name, )
raise SyntaxError(msg, glob.lineno, glob.col_offset)
if old_role & SYM_NONLOCAL:
msg = "name '%s' is nonlocal and global" % (name, )
raise SyntaxError(msg, glob.lineno, glob.col_offset)
if old_role & (SYM_USED | SYM_ASSIGNED | SYM_ANNOTATED):
if old_role & SYM_ASSIGNED:
msg = "name '%s' is assigned to before global declaration"\
% (name,)
elif old_role & SYM_ANNOTATED:
msg = "annotated name '%s' can't be global" \
% (name,)
else:
msg = "name '%s' is used prior to global declaration" % \
(name,)
raise SyntaxError(msg, glob.lineno, glob.col_offset)
self.note_symbol(name, SYM_GLOBAL)
def visit_AnnAssign(self, assign):
# __annotations__ is not setup or used in functions.
if not isinstance(self.scope, FunctionScope):
self.scope.contains_annotated = True
target = assign.target
if isinstance(target, ast.Name):
name = target.id
old_role = self.scope.lookup_role(name)
if assign.simple:
if old_role & SYM_GLOBAL:
raise SyntaxError(
"annotated name '%s' can't be global" % name,
assign.lineno, assign.col_offset)
if old_role & SYM_NONLOCAL:
raise SyntaxError(
"annotated name '%s' can't be nonlocal" % name,
assign.lineno, assign.col_offset)
scope = SYM_BLANK
if assign.simple:
scope |= SYM_ANNOTATED
if assign.value:
scope |= SYM_USED
if scope:
self.note_symbol(name, scope)
else:
target.walkabout(self)
if assign.value is not None:
assign.value.walkabout(self)
if assign.annotation is not None:
assign.annotation.walkabout(self)
def _use_var(self):
# some extra checks just for CPython compatibility -- the logic
# of build_var_scopes() in symbols.py should be able to detect
# all the cases that would really produce broken code, but CPython
# insists on raising SyntaxError in some more cases
if self._is_nested_function():
if self.bare_exec:
raise SyntaxError("for CPython compatibility, an unqualified "
"exec is not allowed here")
if self.import_star:
raise SyntaxError("for CPython compatibility, import * "
"is not allowed here")
def _finalize_name(self, name, flags, local, bound, free, globs):
"""Decide on the scope of a name."""
if flags & SYM_GLOBAL:
if flags & SYM_PARAM:
err = "name '%s' is parameter and global" % (name, )
raise SyntaxError(err, self.lineno, self.col_offset)
if flags & SYM_NONLOCAL:
err = "name '%s' is nonlocal and global" % (name, )
raise SyntaxError(err, self.lineno, self.col_offset)
self.symbols[name] = SCOPE_GLOBAL_EXPLICIT
globs[name] = None
if bound:
try:
del bound[name]
except KeyError:
pass
elif flags & SYM_NONLOCAL:
if flags & SYM_PARAM:
err = "name '%s' is parameter and nonlocal" % (name, )
raise SyntaxError(err, self.lineno, self.col_offset)
if bound is None:
err = "nonlocal declaration not allowed at module level"
raise SyntaxError(err, self.lineno, self.col_offset)
if name not in bound:
err = "no binding for nonlocal '%s' found" % (name, )
raise SyntaxError(err, self.lineno, self.col_offset)
self.symbols[name] = SCOPE_FREE
self.free_vars[name] = None
free[name] = None
self.has_free = True
elif flags & SYM_BOUND:
self.symbols[name] = SCOPE_LOCAL
local[name] = None
try:
del globs[name]
except KeyError:
pass
elif bound and name in bound:
self.symbols[name] = SCOPE_FREE
self.free_vars[name] = None
free[name] = None
self.has_free = True
elif name in globs:
self.symbols[name] = SCOPE_GLOBAL_IMPLICIT
else:
if self.nested:
self.has_free = True
self.symbols[name] = SCOPE_GLOBAL_IMPLICIT
def visitFrom(self, node):
if node.modname != "__future__":
return
raise SyntaxError(
"from __future__ imports must occur at the beginning of the file",
filename=node.filename,
lineno=node.lineno)
def note_return(self, ret):
if ret.value:
if self.is_generator:
raise SyntaxError("'return' with argument inside generator",
ret.lineno, ret.col_offset)
self.return_with_value = True
self.ret = ret
def name_op(self, identifier, ctx):
"""Generate an operation appropiate for the scope of the identifier."""
scope = self.scope.lookup(identifier)
op = ops.NOP
container = self.names
if scope == symtable.SCOPE_LOCAL:
if self.scope.can_be_optimized:
container = self.var_names
op = name_ops_fast(ctx)
elif scope == symtable.SCOPE_FREE:
op = name_ops_deref(ctx)
container = self.free_vars
elif scope == symtable.SCOPE_CELL:
try:
op = name_ops_deref(ctx)
except KeyError:
assert ctx == ast.Del
raise SyntaxError("Can't delete variable used in "
"nested scopes: '%s'" % (identifier,))
container = self.cell_vars
elif scope == symtable.SCOPE_GLOBAL_IMPLICIT:
if self.scope.locals_fully_known:
op = name_ops_global(ctx)
elif scope == symtable.SCOPE_GLOBAL_EXPLICIT:
op = name_ops_global(ctx)
if op == ops.NOP:
op = name_ops_default(ctx)
self.emit_op_arg(op, self.add_name(container, identifier))
def note_yield(self, yield_node):
if self.return_with_value:
raise SyntaxError("'return' with argument inside generator",
self.ret.lineno, self.ret.col_offset)
self.is_generator = True
if self._in_try_body_depth > 0:
self.has_yield_inside_try = True
def build_expr_stmt(builder, nb):
"""expr_stmt: testlist (augassign testlist | ('=' testlist)*)
"""
atoms = get_atoms(builder, nb)
if atoms:
lineno = atoms[0].lineno
else:
lineno = -1
l = len(atoms)
if l == 1:
builder.push(ast.Discard(atoms[0], lineno))
return
op = atoms[1]
assert isinstance(op, TokenObject)
if op.name == builder.parser.tokens['EQUAL']:
nodes = []
for i in range(0, l - 2, 2):
lvalue = to_lvalue(atoms[i], consts.OP_ASSIGN)
nodes.append(lvalue)
rvalue = atoms[-1]
builder.push(ast.Assign(nodes, rvalue, lineno))
pass
else:
assert l == 3
lvalue = atoms[0]
if isinstance(lvalue, ast.GenExpr) or isinstance(lvalue, ast.Tuple):
raise SyntaxError(
"augmented assign to tuple literal or generator expression not possible",
lineno, 0, "")
assert isinstance(op, TokenObject)
builder.push(ast.AugAssign(lvalue, op.get_name(), atoms[2], lineno))
def _visit_comprehension(self, node, kind, comps, *consider):
from pypy.interpreter.error import OperationError
outer = comps[0]
assert isinstance(outer, ast.comprehension)
outer.iter.walkabout(self)
new_scope = FunctionScope("<genexpr>", node.lineno, node.col_offset)
self.push_scope(new_scope, node)
self.implicit_arg(0)
new_scope.is_coroutine |= outer.is_async
outer.target.walkabout(self)
self.visit_sequence(outer.ifs)
self.visit_sequence(comps[1:])
for item in list(consider):
item.walkabout(self)
self.pop_scope()
# http://bugs.python.org/issue10544: a DeprecationWarning from 3.7 on,
# 3.8 will forbid it
if new_scope.is_generator:
msg = "'yield' inside %s" % kind
space = self.space
try:
space.warn(space.newtext(msg), space.w_DeprecationWarning)
except OperationError as e:
# ok, it's actually an exception! turn it into a syntax error
if not e.match(space, space.w_DeprecationWarning):
raise
raise SyntaxError(msg, node.lineno, node.col_offset)
new_scope.is_generator |= isinstance(node, ast.GeneratorExp)
def note_return(self, ret):
if ret.value:
if self.is_coroutine and self.is_generator:
raise SyntaxError("'return' with value in async generator",
ret.lineno, ret.col_offset)
self.return_with_value = True
self.ret = ret
def _finalize_name(self, name, flags, local, bound, free, globs):
"""Decide on the scope of a name."""
if flags & SYM_GLOBAL:
if flags & SYM_PARAM:
err = "name '%s' is local and global" % (name,)
raise SyntaxError(err, self.lineno, self.col_offset)
self.symbols[name] = SCOPE_GLOBAL_EXPLICIT
globs[name] = None
if bound:
try:
del bound[name]
except KeyError:
pass
elif flags & SYM_BOUND:
self.symbols[name] = SCOPE_LOCAL
local[name] = None
try:
del globs[name]
except KeyError:
pass
elif bound and name in bound:
self.symbols[name] = SCOPE_FREE
self.free_vars.append(name)
free[name] = None
self.has_free = True
elif name in globs:
self.symbols[name] = SCOPE_GLOBAL_IMPLICIT
else:
if self.nested:
self.has_free = True
self.symbols[name] = SCOPE_GLOBAL_IMPLICIT
def error(self, msg, ast_node=None):
if ast_node is None:
lineno = self.lineno
col_offset = self.col_offset
else:
lineno = ast_node.lineno
col_offset = ast_node.col_offset
raise SyntaxError(msg, lineno, col_offset)
def to_lvalue(ast_node, flags):
lineno = ast_node.lineno
if isinstance(ast_node, ast.Name):
return ast.AssName(ast_node.varname, flags, lineno)
# return ast.AssName(ast_node.name, flags)
elif isinstance(ast_node, ast.Tuple):
nodes = []
# FIXME: should ast_node.getChildren() but it's not annotable
# because of flatten()
for node in ast_node.nodes:
nodes.append(to_lvalue(node, flags))
return ast.AssTuple(nodes, lineno)
elif isinstance(ast_node, ast.List):
nodes = []
# FIXME: should ast_node.getChildren() but it's not annotable
# because of flatten()
for node in ast_node.nodes:
nodes.append(to_lvalue(node, flags))
return ast.AssList(nodes, lineno)
elif isinstance(ast_node, ast.Getattr):
expr = ast_node.expr
assert isinstance(ast_node, ast.Getattr)
attrname = ast_node.attrname
return ast.AssAttr(expr, attrname, flags, lineno)
elif isinstance(ast_node, ast.Subscript):
ast_node.flags = flags
return ast_node
elif isinstance(ast_node, ast.Slice):
ast_node.flags = flags
return ast_node
else:
if isinstance(ast_node, ast.GenExpr):
raise SyntaxError("assign to generator expression not possible",
lineno, 0, '')
elif isinstance(ast_node, ast.ListComp):
raise SyntaxError("can't assign to list comprehension", lineno, 0,
'')
elif isinstance(ast_node, ast.CallFunc):
if flags == consts.OP_DELETE:
raise SyntaxError("can't delete function call", lineno, 0, '')
else:
raise SyntaxError("can't assign to function call", lineno, 0,
'')
else:
raise SyntaxError("can't assign to non-lvalue", lineno, 0, '')
def visit_ImportFrom(self, imp):
for alias in imp.names:
if self._visit_alias(alias):
if self.scope.note_import_star(imp):
msg = "import * only allowed at module level"
raise SyntaxError(msg,
imp.lineno,
imp.col_offset,
filename=self.compile_info.filename)
def parse_argument(tokens, builder):
"""parses function call arguments"""
l = len(tokens)
index = 0
arguments = []
last_token = None
building_kw = False
kw_built = False
stararg_token = None
dstararg_token = None
while index < l:
cur_token = tokens[index]
if not isinstance(cur_token, TokenObject):
index += 1
if not building_kw:
arguments.append(cur_token)
else:
last_token = arguments.pop()
assert isinstance(last_token, ast.Name) # used by rtyper
arguments.append(
ast.Keyword(last_token.varname, cur_token,
last_token.lineno))
building_kw = False
kw_built = True
continue
elif cur_token.name == builder.parser.tokens['COMMA']:
index += 1
continue
elif cur_token.name == builder.parser.tokens['EQUAL']:
index += 1
building_kw = True
continue
elif cur_token.name == builder.parser.tokens[
'STAR'] or cur_token.name == builder.parser.tokens[
'DOUBLESTAR']:
index += 1
if cur_token.name == builder.parser.tokens['STAR']:
stararg_token = tokens[index]
index += 1
if index >= l:
break
index += 2 # Skip COMMA and DOUBLESTAR
dstararg_token = tokens[index]
break
elif cur_token.get_value() == 'for':
if len(arguments) != 1:
raise SyntaxError("invalid syntax", cur_token.lineno,
cur_token.col)
expr = arguments[0]
genexpr_for = parse_genexpr_for(tokens[index:])
genexpr_for[0].is_outmost = True
gexp = ast.GenExpr(
ast.GenExprInner(expr, genexpr_for, expr.lineno), expr.lineno)
arguments[0] = gexp
break
return arguments, stararg_token, dstararg_token
def build_atom(builder, nb):
atoms = get_atoms(builder, nb)
top = atoms[0]
if isinstance(top, TokenObject):
# assert isinstance(top, TokenObject) # rtyper
if top.name == builder.parser.tokens['LPAR']:
if len(atoms) == 2:
builder.push(ast.Tuple([], top.lineno))
else:
builder.push(atoms[1])
elif top.name == builder.parser.tokens['LSQB']:
if len(atoms) == 2:
builder.push(ast.List([], top.lineno))
else:
list_node = atoms[1]
list_node.lineno = top.lineno
builder.push(list_node)
elif top.name == builder.parser.tokens['LBRACE']:
items = []
for index in range(1, len(atoms) - 1, 4):
# a : b , c : d
# ^ +1 +2 +3 +4
items.append((atoms[index], atoms[index + 2]))
builder.push(ast.Dict(items, top.lineno))
elif top.name == builder.parser.tokens['NAME']:
val = top.get_value()
builder.push(ast.Name(val, top.lineno))
elif top.name == builder.parser.tokens['NUMBER']:
builder.push(
ast.Const(builder.eval_number(top.get_value()), top.lineno))
elif top.name == builder.parser.tokens['STRING']:
# need to concatenate strings in atoms
s = ''
if len(atoms) == 1:
token = atoms[0]
assert isinstance(token, TokenObject)
builder.push(
ast.Const(
parsestr(builder.space, builder.source_encoding,
token.get_value()), top.lineno))
else:
space = builder.space
empty = space.wrap('')
accum = []
for token in atoms:
assert isinstance(token, TokenObject)
accum.append(
parsestr(builder.space, builder.source_encoding,
token.get_value()))
w_s = space.call_method(empty, 'join', space.newlist(accum))
builder.push(ast.Const(w_s, top.lineno))
elif top.name == builder.parser.tokens['BACKQUOTE']:
builder.push(ast.Backquote(atoms[1], atoms[1].lineno))
else:
raise SyntaxError("unexpected tokens", top.lineno, top.col)
def check_stmt(self, stmt, invalid):
if isinstance(stmt, ast.From):
stmt.valid_future = 0
if invalid:
return 0
if is_future(stmt):
assert isinstance(stmt, ast.From)
for name, asname in stmt.names:
if name in self.features:
self.found[name] = 1
elif name == "*":
raise SyntaxError(
"future statement does not support import *",
filename=stmt.filename,
lineno=stmt.lineno)
else:
raise SyntaxError("future feature %s is not defined" %
name,
filename=stmt.filename,
lineno=stmt.lineno)
stmt.valid_future = 1
return 1
return 0
def visitGlobal(self, node ):
scope = self.cur_scope()
for name in node.names:
prevrole = scope.add_global(name)
if prevrole == ROLE_PARAM:
msg = "name '%s' is a function parameter and declared global"
raise SyntaxError(msg % (name,))
elif prevrole == ROLE_DEFINED:
msg = "name '%s' is assigned to before global declaration"
issue_warning(self.space, msg % (name,),
node.filename, node.lineno)
elif prevrole == ROLE_USED:
msg = "name '%s' is used prior to global declaration"
issue_warning(self.space, msg % (name,),
node.filename, node.lineno)
def note_symbol(self, identifier, role):
"""Record that identifier occurs in this scope."""
mangled = self.mangle(identifier)
new_role = role
if mangled in self.roles:
old_role = self.roles[mangled]
if old_role & SYM_PARAM and role & SYM_PARAM:
err = "duplicate argument '%s' in function definition" % \
(identifier,)
raise SyntaxError(err, self.lineno, self.col_offset)
new_role |= old_role
self.roles[mangled] = new_role
if role & SYM_PARAM:
self.varnames.append(mangled)
return mangled
def build_arith_expr(builder, nb):
atoms = get_atoms(builder, nb)
l = len(atoms)
left = atoms[0]
for i in range(2, l, 2):
right = atoms[i]
op_node = atoms[i - 1]
assert isinstance(op_node, TokenObject)
if op_node.name == builder.parser.tokens['PLUS']:
left = ast.Add(left, right, left.lineno)
elif op_node.name == builder.parser.tokens['MINUS']:
left = ast.Sub(left, right, left.lineno)
else:
token = atoms[i - 1]
raise SyntaxError("unexpected token", token.lineno, token.col)
builder.push(left)
def build_shift_expr(builder, nb):
atoms = get_atoms(builder, nb)
l = len(atoms)
left = atoms[0]
lineno = left.lineno
for i in range(2, l, 2):
right = atoms[i]
op_node = atoms[i - 1]
assert isinstance(op_node, TokenObject)
if op_node.name == builder.parser.tokens['LEFTSHIFT']:
left = ast.LeftShift(left, right, lineno)
elif op_node.name == builder.parser.tokens['RIGHTSHIFT']:
left = ast.RightShift(left, right, lineno)
else:
token = atoms[i - 1]
raise SyntaxError("unexpected token", token.lineno, token.col)
builder.push(left)
def parse_lines(self, lines, goal, builder, flags=0):
goalnumber = self.symbols[goal]
target = self.root_rules[goalnumber]
keywords = {} # dict.fromkeys(self.keywords)
disable_with = not (flags & CO_FUTURE_WITH_STATEMENT)
for keyword in self.keywords:
if disable_with and keyword in ('with', 'as'):
continue
keywords[keyword] = None
src = Source(self, lines, keywords, flags)
if not target.match(src, builder):
line, lineno = src.debug()
# XXX needs better error messages
raise SyntaxError("invalid syntax", lineno, -1, line)
# return None
return builder
def build_term(builder, nb):
atoms = get_atoms(builder, nb)
l = len(atoms)
left = atoms[0]
for i in range(2, l, 2):
right = atoms[i]
op_node = atoms[i - 1]
assert isinstance(op_node, TokenObject)
if op_node.name == builder.parser.tokens['STAR']:
left = ast.Mul(left, right, left.lineno)
elif op_node.name == builder.parser.tokens['SLASH']:
left = ast.Div(left, right, left.lineno)
elif op_node.name == builder.parser.tokens['PERCENT']:
left = ast.Mod(left, right, left.lineno)
elif op_node.name == builder.parser.tokens['DOUBLESLASH']:
left = ast.FloorDiv(left, right, left.lineno)
else:
token = atoms[i - 1]
raise SyntaxError("unexpected token", token.lineno, token.col)
builder.push(left)
def parse_genexpr_for(tokens):
"""parses 'for j in k for i in j if i %2 == 0' and returns
a GenExprFor instance
XXX: if RPYTHON supports to pass a class object to a function,
we could refactor parse_listcomp and parse_genexpr_for,
and call :
- parse_listcomp(tokens, forclass=ast.GenExprFor, ifclass=...)
or:
- parse_listcomp(tokens, forclass=ast.ListCompFor, ifclass=...)
"""
genexpr_fors = []
ifs = []
index = 0
if tokens:
lineno = tokens[0].lineno
else:
lineno = -1
while index < len(tokens):
token = tokens[index]
assert isinstance(token, TokenObject) # rtyper info + check
if token.get_value() == 'for':
index += 1 # skip 'for'
ass_node = to_lvalue(tokens[index], consts.OP_ASSIGN)
index += 2 # skip 'in'
iterable = tokens[index]
index += 1
while index < len(tokens):
token = tokens[index]
assert isinstance(token, TokenObject) # rtyper info
if token.get_value() == 'if':
ifs.append(ast.GenExprIf(tokens[index + 1], token.lineno))
index += 2
else:
break
genexpr_fors.append(ast.GenExprFor(ass_node, iterable, ifs,
lineno))
ifs = []
else:
raise SyntaxError('invalid syntax', token.lineno, token.col)
return genexpr_fors
def _finalize_name(self, name, flags, local, bound, free, globs):
"""Decide on the scope of a name."""
if flags & SYM_GLOBAL:
self.symbols[name] = SCOPE_GLOBAL_EXPLICIT
globs[name] = None
if bound:
try:
del bound[name]
except KeyError:
pass
elif flags & SYM_NONLOCAL:
if name not in bound:
err = "no binding for nonlocal '%s' found" % (name, )
raise SyntaxError(err, self.lineno, self.col_offset)
self.symbols[name] = SCOPE_FREE
if not self._hide_bound_from_nested_scopes:
self.free_vars.append(name)
free[name] = None
self.has_free = True
elif flags & (SYM_BOUND | SYM_ANNOTATED):
self.symbols[name] = SCOPE_LOCAL
local[name] = None
try:
del globs[name]
except KeyError:
pass
elif bound and name in bound:
self.symbols[name] = SCOPE_FREE
self.free_vars.append(name)
free[name] = None
self.has_free = True
elif name in globs:
self.symbols[name] = SCOPE_GLOBAL_IMPLICIT
else:
if self.nested:
self.has_free = True
self.symbols[name] = SCOPE_GLOBAL_IMPLICIT
def _check_optimization(self):
if (self.has_free or self.child_has_free) and not self.optimized:
err = None
if self.child_has_free:
trailer = "contains a nested function with free variables"
else:
trailer = "is a nested function"
name = self.name
if self.import_star:
node = self.import_star
if self.bare_exec:
err = "function '%s' uses import * and bare exec, " \
"which are illegal because it %s" % (name, trailer)
else:
err = "import * is not allowed in function '%s' because " \
"it %s" % (name, trailer)
elif self.bare_exec:
node = self.bare_exec
err = "unqualified exec is not allowed in function '%s' " \
"because it %s" % (name, trailer)
else:
raise AssertionError("unknown reason for unoptimization")
raise SyntaxError(err, node.lineno, node.col_offset)
self.locals_fully_known = self.optimized and not self.has_exec
def note_return(self, ret):
"""Called when a return statement is found."""
raise SyntaxError("return outside function", ret.lineno,
ret.col_offset)
def note_yield(self, yield_node):
"""Called when a yield is found."""
raise SyntaxError("'yield' outside function", yield_node.lineno,
yield_node.col_offset)
def __init__(self, msg, line, strstart, token_stack):
lineno, offset = strstart
# offset is zero-based here, but should be 1-based on the SyntaxError
SyntaxError.__init__(self, msg, lineno, offset + 1, line)
self.token_stack = token_stack
def __init__(self, msg, line, strstart, token_stack):
lineno, offset = strstart
SyntaxError.__init__(self, msg, lineno, offset, line)
self.token_stack = token_stack