def visit_Nonlocal(self, nonl): for name in nonl.names: old_role = self.scope.lookup_role(name) msg = "" if old_role & SYM_GLOBAL: msg = "name '%s' is nonlocal and global" % (name, ) if old_role & SYM_PARAM: msg = "name '%s' is parameter and nonlocal" % (name, ) if isinstance(self.scope, ModuleScope): msg = "nonlocal declaration not allowed at module level" if old_role & SYM_ANNOTATED: msg = "annotated name '%s' can't be nonlocal" \ % (name,) if msg is not "": raise SyntaxError(msg, nonl.lineno, nonl.col_offset) if (old_role & (SYM_USED | SYM_ASSIGNED) and not (name == '__class__' and self.scope._hide_bound_from_nested_scopes)): if old_role & SYM_ASSIGNED: msg = "name '%s' is assigned to before nonlocal declaration" \ % (name,) else: msg = "name '%s' is used prior to nonlocal declaration" % \ (name,) raise SyntaxError(msg, nonl.lineno, nonl.col_offset) self.note_symbol(name, SYM_NONLOCAL)
def visit_Global(self, glob): for name in glob.names: old_role = self.scope.lookup_role(name) if (self.scope._hide_bound_from_nested_scopes and name == '__class__'): msg = ("'global __class__' inside a class statement is not " "implemented in PyPy") raise SyntaxError(msg, glob.lineno, glob.col_offset, filename=self.compile_info.filename) if old_role & SYM_PARAM: msg = "name '%s' is parameter and global" % (name, ) raise SyntaxError(msg, glob.lineno, glob.col_offset) if old_role & SYM_NONLOCAL: msg = "name '%s' is nonlocal and global" % (name, ) raise SyntaxError(msg, glob.lineno, glob.col_offset) if old_role & (SYM_USED | SYM_ASSIGNED | SYM_ANNOTATED): if old_role & SYM_ASSIGNED: msg = "name '%s' is assigned to before global declaration"\ % (name,) elif old_role & SYM_ANNOTATED: msg = "annotated name '%s' can't be global" \ % (name,) else: msg = "name '%s' is used prior to global declaration" % \ (name,) raise SyntaxError(msg, glob.lineno, glob.col_offset) self.note_symbol(name, SYM_GLOBAL)
def visit_AnnAssign(self, assign): # __annotations__ is not setup or used in functions. if not isinstance(self.scope, FunctionScope): self.scope.contains_annotated = True target = assign.target if isinstance(target, ast.Name): name = target.id old_role = self.scope.lookup_role(name) if assign.simple: if old_role & SYM_GLOBAL: raise SyntaxError( "annotated name '%s' can't be global" % name, assign.lineno, assign.col_offset) if old_role & SYM_NONLOCAL: raise SyntaxError( "annotated name '%s' can't be nonlocal" % name, assign.lineno, assign.col_offset) scope = SYM_BLANK if assign.simple: scope |= SYM_ANNOTATED if assign.value: scope |= SYM_USED if scope: self.note_symbol(name, scope) else: target.walkabout(self) if assign.value is not None: assign.value.walkabout(self) if assign.annotation is not None: assign.annotation.walkabout(self)
def _use_var(self): # some extra checks just for CPython compatibility -- the logic # of build_var_scopes() in symbols.py should be able to detect # all the cases that would really produce broken code, but CPython # insists on raising SyntaxError in some more cases if self._is_nested_function(): if self.bare_exec: raise SyntaxError("for CPython compatibility, an unqualified " "exec is not allowed here") if self.import_star: raise SyntaxError("for CPython compatibility, import * " "is not allowed here")
def _finalize_name(self, name, flags, local, bound, free, globs): """Decide on the scope of a name.""" if flags & SYM_GLOBAL: if flags & SYM_PARAM: err = "name '%s' is parameter and global" % (name, ) raise SyntaxError(err, self.lineno, self.col_offset) if flags & SYM_NONLOCAL: err = "name '%s' is nonlocal and global" % (name, ) raise SyntaxError(err, self.lineno, self.col_offset) self.symbols[name] = SCOPE_GLOBAL_EXPLICIT globs[name] = None if bound: try: del bound[name] except KeyError: pass elif flags & SYM_NONLOCAL: if flags & SYM_PARAM: err = "name '%s' is parameter and nonlocal" % (name, ) raise SyntaxError(err, self.lineno, self.col_offset) if bound is None: err = "nonlocal declaration not allowed at module level" raise SyntaxError(err, self.lineno, self.col_offset) if name not in bound: err = "no binding for nonlocal '%s' found" % (name, ) raise SyntaxError(err, self.lineno, self.col_offset) self.symbols[name] = SCOPE_FREE self.free_vars[name] = None free[name] = None self.has_free = True elif flags & SYM_BOUND: self.symbols[name] = SCOPE_LOCAL local[name] = None try: del globs[name] except KeyError: pass elif bound and name in bound: self.symbols[name] = SCOPE_FREE self.free_vars[name] = None free[name] = None self.has_free = True elif name in globs: self.symbols[name] = SCOPE_GLOBAL_IMPLICIT else: if self.nested: self.has_free = True self.symbols[name] = SCOPE_GLOBAL_IMPLICIT
def visitFrom(self, node): if node.modname != "__future__": return raise SyntaxError( "from __future__ imports must occur at the beginning of the file", filename=node.filename, lineno=node.lineno)
def note_return(self, ret): if ret.value: if self.is_generator: raise SyntaxError("'return' with argument inside generator", ret.lineno, ret.col_offset) self.return_with_value = True self.ret = ret
def name_op(self, identifier, ctx): """Generate an operation appropiate for the scope of the identifier.""" scope = self.scope.lookup(identifier) op = ops.NOP container = self.names if scope == symtable.SCOPE_LOCAL: if self.scope.can_be_optimized: container = self.var_names op = name_ops_fast(ctx) elif scope == symtable.SCOPE_FREE: op = name_ops_deref(ctx) container = self.free_vars elif scope == symtable.SCOPE_CELL: try: op = name_ops_deref(ctx) except KeyError: assert ctx == ast.Del raise SyntaxError("Can't delete variable used in " "nested scopes: '%s'" % (identifier,)) container = self.cell_vars elif scope == symtable.SCOPE_GLOBAL_IMPLICIT: if self.scope.locals_fully_known: op = name_ops_global(ctx) elif scope == symtable.SCOPE_GLOBAL_EXPLICIT: op = name_ops_global(ctx) if op == ops.NOP: op = name_ops_default(ctx) self.emit_op_arg(op, self.add_name(container, identifier))
def note_yield(self, yield_node): if self.return_with_value: raise SyntaxError("'return' with argument inside generator", self.ret.lineno, self.ret.col_offset) self.is_generator = True if self._in_try_body_depth > 0: self.has_yield_inside_try = True
def build_expr_stmt(builder, nb): """expr_stmt: testlist (augassign testlist | ('=' testlist)*) """ atoms = get_atoms(builder, nb) if atoms: lineno = atoms[0].lineno else: lineno = -1 l = len(atoms) if l == 1: builder.push(ast.Discard(atoms[0], lineno)) return op = atoms[1] assert isinstance(op, TokenObject) if op.name == builder.parser.tokens['EQUAL']: nodes = [] for i in range(0, l - 2, 2): lvalue = to_lvalue(atoms[i], consts.OP_ASSIGN) nodes.append(lvalue) rvalue = atoms[-1] builder.push(ast.Assign(nodes, rvalue, lineno)) pass else: assert l == 3 lvalue = atoms[0] if isinstance(lvalue, ast.GenExpr) or isinstance(lvalue, ast.Tuple): raise SyntaxError( "augmented assign to tuple literal or generator expression not possible", lineno, 0, "") assert isinstance(op, TokenObject) builder.push(ast.AugAssign(lvalue, op.get_name(), atoms[2], lineno))
def _visit_comprehension(self, node, kind, comps, *consider): from pypy.interpreter.error import OperationError outer = comps[0] assert isinstance(outer, ast.comprehension) outer.iter.walkabout(self) new_scope = FunctionScope("<genexpr>", node.lineno, node.col_offset) self.push_scope(new_scope, node) self.implicit_arg(0) new_scope.is_coroutine |= outer.is_async outer.target.walkabout(self) self.visit_sequence(outer.ifs) self.visit_sequence(comps[1:]) for item in list(consider): item.walkabout(self) self.pop_scope() # http://bugs.python.org/issue10544: a DeprecationWarning from 3.7 on, # 3.8 will forbid it if new_scope.is_generator: msg = "'yield' inside %s" % kind space = self.space try: space.warn(space.newtext(msg), space.w_DeprecationWarning) except OperationError as e: # ok, it's actually an exception! turn it into a syntax error if not e.match(space, space.w_DeprecationWarning): raise raise SyntaxError(msg, node.lineno, node.col_offset) new_scope.is_generator |= isinstance(node, ast.GeneratorExp)
def note_return(self, ret): if ret.value: if self.is_coroutine and self.is_generator: raise SyntaxError("'return' with value in async generator", ret.lineno, ret.col_offset) self.return_with_value = True self.ret = ret
def _finalize_name(self, name, flags, local, bound, free, globs): """Decide on the scope of a name.""" if flags & SYM_GLOBAL: if flags & SYM_PARAM: err = "name '%s' is local and global" % (name,) raise SyntaxError(err, self.lineno, self.col_offset) self.symbols[name] = SCOPE_GLOBAL_EXPLICIT globs[name] = None if bound: try: del bound[name] except KeyError: pass elif flags & SYM_BOUND: self.symbols[name] = SCOPE_LOCAL local[name] = None try: del globs[name] except KeyError: pass elif bound and name in bound: self.symbols[name] = SCOPE_FREE self.free_vars.append(name) free[name] = None self.has_free = True elif name in globs: self.symbols[name] = SCOPE_GLOBAL_IMPLICIT else: if self.nested: self.has_free = True self.symbols[name] = SCOPE_GLOBAL_IMPLICIT
def error(self, msg, ast_node=None): if ast_node is None: lineno = self.lineno col_offset = self.col_offset else: lineno = ast_node.lineno col_offset = ast_node.col_offset raise SyntaxError(msg, lineno, col_offset)
def to_lvalue(ast_node, flags): lineno = ast_node.lineno if isinstance(ast_node, ast.Name): return ast.AssName(ast_node.varname, flags, lineno) # return ast.AssName(ast_node.name, flags) elif isinstance(ast_node, ast.Tuple): nodes = [] # FIXME: should ast_node.getChildren() but it's not annotable # because of flatten() for node in ast_node.nodes: nodes.append(to_lvalue(node, flags)) return ast.AssTuple(nodes, lineno) elif isinstance(ast_node, ast.List): nodes = [] # FIXME: should ast_node.getChildren() but it's not annotable # because of flatten() for node in ast_node.nodes: nodes.append(to_lvalue(node, flags)) return ast.AssList(nodes, lineno) elif isinstance(ast_node, ast.Getattr): expr = ast_node.expr assert isinstance(ast_node, ast.Getattr) attrname = ast_node.attrname return ast.AssAttr(expr, attrname, flags, lineno) elif isinstance(ast_node, ast.Subscript): ast_node.flags = flags return ast_node elif isinstance(ast_node, ast.Slice): ast_node.flags = flags return ast_node else: if isinstance(ast_node, ast.GenExpr): raise SyntaxError("assign to generator expression not possible", lineno, 0, '') elif isinstance(ast_node, ast.ListComp): raise SyntaxError("can't assign to list comprehension", lineno, 0, '') elif isinstance(ast_node, ast.CallFunc): if flags == consts.OP_DELETE: raise SyntaxError("can't delete function call", lineno, 0, '') else: raise SyntaxError("can't assign to function call", lineno, 0, '') else: raise SyntaxError("can't assign to non-lvalue", lineno, 0, '')
def visit_ImportFrom(self, imp): for alias in imp.names: if self._visit_alias(alias): if self.scope.note_import_star(imp): msg = "import * only allowed at module level" raise SyntaxError(msg, imp.lineno, imp.col_offset, filename=self.compile_info.filename)
def parse_argument(tokens, builder): """parses function call arguments""" l = len(tokens) index = 0 arguments = [] last_token = None building_kw = False kw_built = False stararg_token = None dstararg_token = None while index < l: cur_token = tokens[index] if not isinstance(cur_token, TokenObject): index += 1 if not building_kw: arguments.append(cur_token) else: last_token = arguments.pop() assert isinstance(last_token, ast.Name) # used by rtyper arguments.append( ast.Keyword(last_token.varname, cur_token, last_token.lineno)) building_kw = False kw_built = True continue elif cur_token.name == builder.parser.tokens['COMMA']: index += 1 continue elif cur_token.name == builder.parser.tokens['EQUAL']: index += 1 building_kw = True continue elif cur_token.name == builder.parser.tokens[ 'STAR'] or cur_token.name == builder.parser.tokens[ 'DOUBLESTAR']: index += 1 if cur_token.name == builder.parser.tokens['STAR']: stararg_token = tokens[index] index += 1 if index >= l: break index += 2 # Skip COMMA and DOUBLESTAR dstararg_token = tokens[index] break elif cur_token.get_value() == 'for': if len(arguments) != 1: raise SyntaxError("invalid syntax", cur_token.lineno, cur_token.col) expr = arguments[0] genexpr_for = parse_genexpr_for(tokens[index:]) genexpr_for[0].is_outmost = True gexp = ast.GenExpr( ast.GenExprInner(expr, genexpr_for, expr.lineno), expr.lineno) arguments[0] = gexp break return arguments, stararg_token, dstararg_token
def build_atom(builder, nb): atoms = get_atoms(builder, nb) top = atoms[0] if isinstance(top, TokenObject): # assert isinstance(top, TokenObject) # rtyper if top.name == builder.parser.tokens['LPAR']: if len(atoms) == 2: builder.push(ast.Tuple([], top.lineno)) else: builder.push(atoms[1]) elif top.name == builder.parser.tokens['LSQB']: if len(atoms) == 2: builder.push(ast.List([], top.lineno)) else: list_node = atoms[1] list_node.lineno = top.lineno builder.push(list_node) elif top.name == builder.parser.tokens['LBRACE']: items = [] for index in range(1, len(atoms) - 1, 4): # a : b , c : d # ^ +1 +2 +3 +4 items.append((atoms[index], atoms[index + 2])) builder.push(ast.Dict(items, top.lineno)) elif top.name == builder.parser.tokens['NAME']: val = top.get_value() builder.push(ast.Name(val, top.lineno)) elif top.name == builder.parser.tokens['NUMBER']: builder.push( ast.Const(builder.eval_number(top.get_value()), top.lineno)) elif top.name == builder.parser.tokens['STRING']: # need to concatenate strings in atoms s = '' if len(atoms) == 1: token = atoms[0] assert isinstance(token, TokenObject) builder.push( ast.Const( parsestr(builder.space, builder.source_encoding, token.get_value()), top.lineno)) else: space = builder.space empty = space.wrap('') accum = [] for token in atoms: assert isinstance(token, TokenObject) accum.append( parsestr(builder.space, builder.source_encoding, token.get_value())) w_s = space.call_method(empty, 'join', space.newlist(accum)) builder.push(ast.Const(w_s, top.lineno)) elif top.name == builder.parser.tokens['BACKQUOTE']: builder.push(ast.Backquote(atoms[1], atoms[1].lineno)) else: raise SyntaxError("unexpected tokens", top.lineno, top.col)
def check_stmt(self, stmt, invalid): if isinstance(stmt, ast.From): stmt.valid_future = 0 if invalid: return 0 if is_future(stmt): assert isinstance(stmt, ast.From) for name, asname in stmt.names: if name in self.features: self.found[name] = 1 elif name == "*": raise SyntaxError( "future statement does not support import *", filename=stmt.filename, lineno=stmt.lineno) else: raise SyntaxError("future feature %s is not defined" % name, filename=stmt.filename, lineno=stmt.lineno) stmt.valid_future = 1 return 1 return 0
def visitGlobal(self, node ): scope = self.cur_scope() for name in node.names: prevrole = scope.add_global(name) if prevrole == ROLE_PARAM: msg = "name '%s' is a function parameter and declared global" raise SyntaxError(msg % (name,)) elif prevrole == ROLE_DEFINED: msg = "name '%s' is assigned to before global declaration" issue_warning(self.space, msg % (name,), node.filename, node.lineno) elif prevrole == ROLE_USED: msg = "name '%s' is used prior to global declaration" issue_warning(self.space, msg % (name,), node.filename, node.lineno)
def note_symbol(self, identifier, role): """Record that identifier occurs in this scope.""" mangled = self.mangle(identifier) new_role = role if mangled in self.roles: old_role = self.roles[mangled] if old_role & SYM_PARAM and role & SYM_PARAM: err = "duplicate argument '%s' in function definition" % \ (identifier,) raise SyntaxError(err, self.lineno, self.col_offset) new_role |= old_role self.roles[mangled] = new_role if role & SYM_PARAM: self.varnames.append(mangled) return mangled
def build_arith_expr(builder, nb): atoms = get_atoms(builder, nb) l = len(atoms) left = atoms[0] for i in range(2, l, 2): right = atoms[i] op_node = atoms[i - 1] assert isinstance(op_node, TokenObject) if op_node.name == builder.parser.tokens['PLUS']: left = ast.Add(left, right, left.lineno) elif op_node.name == builder.parser.tokens['MINUS']: left = ast.Sub(left, right, left.lineno) else: token = atoms[i - 1] raise SyntaxError("unexpected token", token.lineno, token.col) builder.push(left)
def build_shift_expr(builder, nb): atoms = get_atoms(builder, nb) l = len(atoms) left = atoms[0] lineno = left.lineno for i in range(2, l, 2): right = atoms[i] op_node = atoms[i - 1] assert isinstance(op_node, TokenObject) if op_node.name == builder.parser.tokens['LEFTSHIFT']: left = ast.LeftShift(left, right, lineno) elif op_node.name == builder.parser.tokens['RIGHTSHIFT']: left = ast.RightShift(left, right, lineno) else: token = atoms[i - 1] raise SyntaxError("unexpected token", token.lineno, token.col) builder.push(left)
def parse_lines(self, lines, goal, builder, flags=0): goalnumber = self.symbols[goal] target = self.root_rules[goalnumber] keywords = {} # dict.fromkeys(self.keywords) disable_with = not (flags & CO_FUTURE_WITH_STATEMENT) for keyword in self.keywords: if disable_with and keyword in ('with', 'as'): continue keywords[keyword] = None src = Source(self, lines, keywords, flags) if not target.match(src, builder): line, lineno = src.debug() # XXX needs better error messages raise SyntaxError("invalid syntax", lineno, -1, line) # return None return builder
def build_term(builder, nb): atoms = get_atoms(builder, nb) l = len(atoms) left = atoms[0] for i in range(2, l, 2): right = atoms[i] op_node = atoms[i - 1] assert isinstance(op_node, TokenObject) if op_node.name == builder.parser.tokens['STAR']: left = ast.Mul(left, right, left.lineno) elif op_node.name == builder.parser.tokens['SLASH']: left = ast.Div(left, right, left.lineno) elif op_node.name == builder.parser.tokens['PERCENT']: left = ast.Mod(left, right, left.lineno) elif op_node.name == builder.parser.tokens['DOUBLESLASH']: left = ast.FloorDiv(left, right, left.lineno) else: token = atoms[i - 1] raise SyntaxError("unexpected token", token.lineno, token.col) builder.push(left)
def parse_genexpr_for(tokens): """parses 'for j in k for i in j if i %2 == 0' and returns a GenExprFor instance XXX: if RPYTHON supports to pass a class object to a function, we could refactor parse_listcomp and parse_genexpr_for, and call : - parse_listcomp(tokens, forclass=ast.GenExprFor, ifclass=...) or: - parse_listcomp(tokens, forclass=ast.ListCompFor, ifclass=...) """ genexpr_fors = [] ifs = [] index = 0 if tokens: lineno = tokens[0].lineno else: lineno = -1 while index < len(tokens): token = tokens[index] assert isinstance(token, TokenObject) # rtyper info + check if token.get_value() == 'for': index += 1 # skip 'for' ass_node = to_lvalue(tokens[index], consts.OP_ASSIGN) index += 2 # skip 'in' iterable = tokens[index] index += 1 while index < len(tokens): token = tokens[index] assert isinstance(token, TokenObject) # rtyper info if token.get_value() == 'if': ifs.append(ast.GenExprIf(tokens[index + 1], token.lineno)) index += 2 else: break genexpr_fors.append(ast.GenExprFor(ass_node, iterable, ifs, lineno)) ifs = [] else: raise SyntaxError('invalid syntax', token.lineno, token.col) return genexpr_fors
def _finalize_name(self, name, flags, local, bound, free, globs): """Decide on the scope of a name.""" if flags & SYM_GLOBAL: self.symbols[name] = SCOPE_GLOBAL_EXPLICIT globs[name] = None if bound: try: del bound[name] except KeyError: pass elif flags & SYM_NONLOCAL: if name not in bound: err = "no binding for nonlocal '%s' found" % (name, ) raise SyntaxError(err, self.lineno, self.col_offset) self.symbols[name] = SCOPE_FREE if not self._hide_bound_from_nested_scopes: self.free_vars.append(name) free[name] = None self.has_free = True elif flags & (SYM_BOUND | SYM_ANNOTATED): self.symbols[name] = SCOPE_LOCAL local[name] = None try: del globs[name] except KeyError: pass elif bound and name in bound: self.symbols[name] = SCOPE_FREE self.free_vars.append(name) free[name] = None self.has_free = True elif name in globs: self.symbols[name] = SCOPE_GLOBAL_IMPLICIT else: if self.nested: self.has_free = True self.symbols[name] = SCOPE_GLOBAL_IMPLICIT
def _check_optimization(self): if (self.has_free or self.child_has_free) and not self.optimized: err = None if self.child_has_free: trailer = "contains a nested function with free variables" else: trailer = "is a nested function" name = self.name if self.import_star: node = self.import_star if self.bare_exec: err = "function '%s' uses import * and bare exec, " \ "which are illegal because it %s" % (name, trailer) else: err = "import * is not allowed in function '%s' because " \ "it %s" % (name, trailer) elif self.bare_exec: node = self.bare_exec err = "unqualified exec is not allowed in function '%s' " \ "because it %s" % (name, trailer) else: raise AssertionError("unknown reason for unoptimization") raise SyntaxError(err, node.lineno, node.col_offset) self.locals_fully_known = self.optimized and not self.has_exec
def note_return(self, ret): """Called when a return statement is found.""" raise SyntaxError("return outside function", ret.lineno, ret.col_offset)
def note_yield(self, yield_node): """Called when a yield is found.""" raise SyntaxError("'yield' outside function", yield_node.lineno, yield_node.col_offset)
def __init__(self, msg, line, strstart, token_stack): lineno, offset = strstart # offset is zero-based here, but should be 1-based on the SyntaxError SyntaxError.__init__(self, msg, lineno, offset + 1, line) self.token_stack = token_stack
def __init__(self, msg, line, strstart, token_stack): lineno, offset = strstart SyntaxError.__init__(self, msg, lineno, offset, line) self.token_stack = token_stack