def build_atom(builder, nb): atoms = get_atoms(builder, nb) top = atoms[0] if isinstance(top, TokenObject): # assert isinstance(top, TokenObject) # rtyper if top.name == builder.parser.tokens['LPAR']: if len(atoms) == 2: builder.push(ast.Tuple([], top.lineno)) else: builder.push(atoms[1]) elif top.name == builder.parser.tokens['LSQB']: if len(atoms) == 2: builder.push(ast.List([], top.lineno)) else: list_node = atoms[1] list_node.lineno = top.lineno builder.push(list_node) elif top.name == builder.parser.tokens['LBRACE']: items = [] for index in range(1, len(atoms) - 1, 4): # a : b , c : d # ^ +1 +2 +3 +4 items.append((atoms[index], atoms[index + 2])) builder.push(ast.Dict(items, top.lineno)) elif top.name == builder.parser.tokens['NAME']: val = top.get_value() builder.push(ast.Name(val, top.lineno)) elif top.name == builder.parser.tokens['NUMBER']: builder.push( ast.Const(builder.eval_number(top.get_value()), top.lineno)) elif top.name == builder.parser.tokens['STRING']: # need to concatenate strings in atoms s = '' if len(atoms) == 1: token = atoms[0] assert isinstance(token, TokenObject) builder.push( ast.Const( parsestr(builder.space, builder.source_encoding, token.get_value()), top.lineno)) else: space = builder.space empty = space.wrap('') accum = [] for token in atoms: assert isinstance(token, TokenObject) accum.append( parsestr(builder.space, builder.source_encoding, token.get_value())) w_s = space.call_method(empty, 'join', space.newlist(accum)) builder.push(ast.Const(w_s, top.lineno)) elif top.name == builder.parser.tokens['BACKQUOTE']: builder.push(ast.Backquote(atoms[1], atoms[1].lineno)) else: raise SyntaxError("unexpected tokens", top.lineno, top.col)
def test_mutate_add(self): c1 = ast.Const(1) c2 = ast.Const(2) add = ast.Add(c1, c2) class Visitor(BaseVisitor): def visitAdd(self, node): return ast.Const(3) c3 = add.mutate(Visitor()) assert isinstance(c3, ast.Const)
def build_subscript(builder, nb): """'.' '.' '.' | [test] ':' [test] [':' [test]] | test""" atoms = get_atoms(builder, nb) token = atoms[0] lineno = token.lineno if isinstance(token, TokenObject) and token.name == builder.parser.tokens['DOT']: # Ellipsis: builder.push(ast.Ellipsis(lineno)) elif len(atoms) == 1: if isinstance(token, TokenObject) and token.name == builder.parser.tokens['COLON']: sliceinfos = [None, None, None] builder.push(SlicelistObject('slice', sliceinfos, lineno)) else: # test builder.push(token) else: # elif len(atoms) > 1: sliceinfos = [None, None, None] infosindex = 0 for token in atoms: if isinstance(token, TokenObject) and token.name == builder.parser.tokens['COLON']: infosindex += 1 else: sliceinfos[infosindex] = token if infosindex == 2: sliceobj_infos = [] for value in sliceinfos: if value is None: sliceobj_infos.append(ast.Const(builder.wrap_none(), lineno)) else: sliceobj_infos.append(value) builder.push(SlicelistObject('sliceobj', sliceobj_infos, lineno)) else: builder.push(SlicelistObject('slice', sliceinfos, lineno))
def visit_UnaryOp(self, unary): w_operand = unary.operand.as_constant() op = unary.op if w_operand is not None: try: for op_kind, folder in unrolling_unary_folders: if op_kind == op: w_const = folder(self.space, w_operand) break else: raise AssertionError("unknown unary operation") w_minint = self.space.wrap(-sys.maxint - 1) # This makes sure the result is an integer. if self.space.eq_w(w_minint, w_const): w_const = w_minint except OperationError: pass else: return ast.Const(w_const, unary.lineno, unary.col_offset) elif op == ast.Not: compare = unary.operand if isinstance(compare, ast.Compare) and len(compare.ops) == 1: cmp_op = compare.ops[0] try: opposite = opposite_compare_operations(cmp_op) except KeyError: pass else: compare.ops[0] = opposite return compare return unary
def visitTuple(self, node): consts_w = [] for subnode in node.nodes: if not isinstance(subnode, ast.Const): return node # not all constants consts_w.append(subnode.value) return ast.Const(self.space.newtuple(consts_w))
def visit_Name(self, name): # Turn loading None into a constant lookup. Eventaully, we can do this # for True and False, too. if name.id == "None": assert name.ctx == ast.Load return ast.Const(self.space.w_None, name.lineno, name.col_offset) return name
def visit_BinOp(self, binop): left = binop.left.as_constant() if left is not None: right = binop.right.as_constant() if right is not None: op = binop.op # Can't fold straight division without "from __future_ import # division" because it might be affected at runtime by the -Q # flag. if op == ast.Div and \ not self.compile_info.flags & consts.CO_FUTURE_DIVISION: return binop try: for op_kind, folder in unrolling_binary_folders: if op_kind == op: w_const = folder(self.space, left, right) break else: raise AssertionError("unknown binary operation") # Let all errors be found at runtime. except OperationError: pass else: # To avoid blowing up the size of pyc files, we only fold # reasonably sized sequences. try: w_len = self.space.len(w_const) except OperationError: pass else: if self.space.int_w(w_len) > 20: return binop return ast.Const(w_const, binop.lineno, binop.col_offset) return binop
def visit_BinOp(self, binop): left = binop.left.as_constant() if left is not None: right = binop.right.as_constant() if right is not None: op = binop.op try: for op_kind, folder in unrolling_binary_folders: if op_kind == op: w_const = folder(self.space, left, right) break else: raise AssertionError("unknown binary operation") # Let all errors be found at runtime. except OperationError: pass else: # To avoid blowing up the size of pyc files, we only fold # reasonably sized sequences. try: w_len = self.space.len(w_const) except OperationError: pass else: if self.space.int_w(w_len) > 20: return binop return ast.Const(w_const, binop.lineno, binop.col_offset) return binop
def as_node_list(self, space): try: values_w = space.unpackiterable(self.value) except OperationError: return None line = self.lineno column = self.col_offset return [ast.Const(w_obj, line, column) for w_obj in values_w]
def _visitUnaryOp(self, node, constant_fold): expr = node.expr if isinstance(expr, ast.Const): try: w_newvalue = constant_fold(self.space, expr.value) except OperationError: pass else: return ast.Const(w_newvalue) return node
def visit_Name(self, name): # Turn loading None into a constant lookup. We cannot do this # for True and False, because rebinding them is allowed (2.7). if name.id == "None": # The compiler refuses to parse "None = ...", but "del None" # is allowed (if pointless). Check anyway: custom asts that # correspond to "None = ..." can be made by hand. if name.ctx == ast.Load: return ast.Const(self.space.w_None, name.lineno, name.col_offset) return name
def build_trailer(builder, nb): """trailer: '(' ')' | '(' arglist ')' | '[' subscriptlist ']' | '.' NAME """ atoms = get_atoms(builder, nb) first_token = atoms[0] # Case 1 : '(' ... if isinstance( first_token, TokenObject) and first_token.name == builder.parser.tokens['LPAR']: if len(atoms ) == 2: # and atoms[1].token == builder.parser.tokens['RPAR']: builder.push(ArglistObject([], None, None, first_token.lineno)) elif len(atoms) == 3: # '(' Arglist ')' # push arglist on the stack builder.push(atoms[1]) elif isinstance( first_token, TokenObject) and first_token.name == builder.parser.tokens['LSQB']: if len(atoms) == 3 and isinstance(atoms[1], SlicelistObject): builder.push(atoms[1]) else: # atoms is a list of, alternatively, values and comma tokens, # with '[' and ']' tokens at the end subs = [] for index in range(1, len(atoms) - 1, 2): atom = atoms[index] if isinstance(atom, SlicelistObject): num_slicevals = 3 slicevals = [] if atom.fake_rulename == 'slice': num_slicevals = 2 for val in atom.value[:num_slicevals]: if val is None: slicevals.append( ast.Const(builder.wrap_none(), atom.lineno)) else: slicevals.append(val) subs.append(ast.Sliceobj(slicevals, atom.lineno)) else: subs.append(atom) if len(atoms) > 3: # at least one comma sub = ast.Tuple(subs, first_token.lineno) else: [sub] = subs builder.push(SubscriptObject('subscript', sub, first_token.lineno)) elif len(atoms) == 2: # Attribute access: '.' NAME builder.push(atoms[0]) builder.push(atoms[1]) builder.push( TempRuleObject('pending-attr-access', 2, first_token.lineno)) else: assert False, "Trailer reducing implementation incomplete !"
def visit_Subscript(self, subs): if subs.ctx == ast.Load: w_obj = subs.value.as_constant() if w_obj is not None: w_idx = subs.slice.as_constant() if w_idx is not None: try: return ast.Const(self.space.getitem(w_obj, w_idx), subs.lineno, subs.col_offset) except OperationError: # Let exceptions propgate at runtime. pass return subs
def visit_Tuple(self, tup): """Try to turn tuple building into a constant.""" if tup.elts: consts_w = [None] * len(tup.elts) for i in range(len(tup.elts)): node = tup.elts[i] w_const = node.as_constant() if w_const is None: return tup consts_w[i] = w_const else: consts_w = [] w_consts = self.space.newtuple(consts_w) return ast.Const(w_consts, tup.lineno, tup.col_offset)
def build_simple_stmt(builder, nb): atoms = get_atoms(builder, nb) l = len(atoms) nodes = [] if atoms: lineno = atoms[0].lineno else: lineno = -1 for n in range(0,l,2): node = atoms[n] if isinstance(node, TokenObject) and node.name == builder.parser.tokens['NEWLINE']: nodes.append(ast.Discard(ast.Const(builder.wrap_none()), node.lineno)) else: nodes.append(node) builder.push(ast.Stmt(nodes, lineno))
def visit_Name(self, name): """Turn loading None, True, and False into a constant lookup.""" if name.ctx == ast.Del: return name space = self.space iden = name.id w_const = None if iden == "None": w_const = space.w_None elif iden == "True": w_const = space.w_True elif iden == "False": w_const = space.w_False if w_const is not None: return ast.Const(w_const, name.lineno, name.col_offset) return name
def _visitBitOp(self, node, constant_fold): while len(node.nodes) >= 2: left = node.nodes[0] if not isinstance(left, ast.Const): return node # done right = node.nodes[1] if not isinstance(right, ast.Const): return node # done try: w_newvalue = constant_fold(self.space, left.value, right.value) except OperationError: return node # done del node.nodes[1] node.nodes[0] = ast.Const(w_newvalue) else: # if reduced to a single node, just returns it return node.nodes[0]
def visit_Tuple(self, tup): """Try to turn tuple building into a constant.""" if tup.elts: consts_w = [None]*len(tup.elts) for i in range(len(tup.elts)): node = tup.elts[i] w_const = node.as_constant() if w_const is None: return tup consts_w[i] = w_const # intern the string constants packed into the tuple here, # because assemble.py will see the result as just a tuple constant for i in range(len(consts_w)): consts_w[i] = misc.intern_if_common_string( self.space, consts_w[i]) else: consts_w = [] w_consts = self.space.newtuple(consts_w) return ast.Const(w_consts, tup.lineno, tup.col_offset)
def _visitBinaryOp(self, node, constant_fold): left = node.left right = node.right if isinstance(left, ast.Const) and isinstance(right, ast.Const): try: w_newvalue = constant_fold(self.space, left.value, right.value) except OperationError: pass else: # to avoid creating too large .pyc files, we don't # constant-fold operations that create long sequences, # like '(5,) * 500'. This is the same as CPython. try: size = self.space.int_w(self.space.len(w_newvalue)) except OperationError: size = -1 if size <= 20: return ast.Const(w_newvalue) return node
def build_exprlist(builder, nb): """exprlist: expr (',' expr)* [',']""" atoms = get_atoms(builder, nb) if len(atoms) <= 2: builder.push(atoms[0]) else: names = [] values = [] isConst = True for index in range(0, len(atoms), 2): item = atoms[index] names.append(item) if isinstance(item, ast.Const): values.append(item) else: isConst = False if isConst: builder.push(ast.Const(builder.space.newtuple(values), atoms[0].lineno)) else: builder.push(ast.Tuple(names, atoms[0].lineno))
def build_testlist_gexp(builder, nb): atoms = get_atoms(builder, nb) if atoms: lineno = atoms[0].lineno else: lineno = -1 l = len(atoms) if l == 1: builder.push(atoms[0]) return items = [] token = atoms[1] if isinstance( token, TokenObject) and token.name == builder.parser.tokens['COMMA']: for i in range(0, l, 2): # this is atoms not 1 items.append(atoms[i]) else: # genfor: 'i for i in j' # GenExpr(GenExprInner(Name('i'), [GenExprFor(AssName('i', 'OP_ASSIGN'), Name('j'), [])])))])) expr = atoms[0] genexpr_for = parse_genexpr_for(atoms[1:]) genexpr_for[0].is_outmost = True builder.push( ast.GenExpr(ast.GenExprInner(expr, genexpr_for, lineno), lineno)) return isConst = True values = [] for item in items: if isinstance(item, ast.Const): values.append(item.value) else: isConst = False break if isConst: builder.push(ast.Const(builder.space.newtuple(values), lineno)) else: builder.push(ast.Tuple(items, lineno)) return
def visit_Subscript(self, subs): if subs.ctx == ast.Load: w_obj = subs.value.as_constant() if w_obj is not None: w_idx = subs.slice.as_constant() if w_idx is not None: try: w_const = self.space.getitem(w_obj, w_idx) except OperationError: # Let exceptions propagate at runtime. return subs # CPython issue5057: if v is unicode, there might # be differences between wide and narrow builds in # cases like u'\U00012345'[0]. # Wide builds will return a non-BMP char, whereas # narrow builds will return a surrogate. In both # the cases skip the optimization in order to # produce compatible pycs. if (self.space.isinstance_w(w_obj, self.space.w_unicode) and self.space.isinstance_w( w_const, self.space.w_unicode)): #unistr = self.space.unicode_w(w_const) #if len(unistr) == 1: # ch = ord(unistr[0]) #else: # ch = 0 #if (ch > 0xFFFF or # (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)): # --XXX-- for now we always disable optimization of # u'...'[constant] because the tests above are not # enough to fix issue5057 (CPython has the same # problem as of April 24, 2012). # See test_const_fold_unicode_subscr return subs return ast.Const(w_const, subs.lineno, subs.col_offset) return subs
def visitAdd(self, node): return ast.Const(3)
def nodes_equal(left, right, check_lineno=False): if isinstance(left, ast_ast.Node) and isinstance(right, ast_ast.Node): # direct comparison if left.__class__ is not right.__class__: print "Node type mismatch:", left, right return False if check_lineno and left.lineno != right.lineno: print "lineno mismatch in (%s) left: %s, right: %s" % (left, left.lineno, right.lineno) return False left_nodes = list(left.getChildren()) right_nodes = list(right.getChildren()) if len(left_nodes) != len(right_nodes): print "Number of children mismatch:", left, right return False for left_node, right_node in zip(left_nodes, right_nodes): if not nodes_equal(left_node, right_node, check_lineno): return False return True if not isinstance(left,test_ast.Node) or not isinstance(right,ast_ast.Node): return left==right if left.__class__.__name__ != right.__class__.__name__: print "Node type mismatch:", left, right return False if isinstance(left,test_ast.Function) and isinstance(right,ast_ast.Function): left_nodes = list(left.getChildren()) right_nodes = [] # generated ast differ here because argnames is a list of nodes in right_nodes.append(right.decorators) right_nodes.append(right.name) right_nodes.append(right.argnames) right_nodes.extend(flatten(right.defaults)) right_nodes.append(right.flags) right_nodes.append(right.w_doc) right_nodes.append(right.code) left_args = left_nodes[2] del left_nodes[2] right_args = right_nodes[2] del right_nodes[2] if not arglist_equal(left_args, right_args): return False elif isinstance(left,test_ast.Lambda) and isinstance(right,ast_ast.Lambda): left_nodes = list(left.getChildren()) right_nodes = [] # generated ast differ here because argnames is a list of nodes in right_nodes.append(right.argnames) right_nodes.extend(flatten(right.defaults)) right_nodes.append(right.flags) right_nodes.append(right.code) print "left", repr(left_nodes) print "right", repr(right_nodes) left_args = left_nodes[0] del left_nodes[0] right_args = right_nodes[0] del right_nodes[0] if not arglist_equal(left_args, right_args): return False elif isinstance(left,test_ast.Const): if isinstance(right,ast_ast.Const): r = left.value == right.value elif isinstance(right,ast_ast.NoneConst): r = left.value == None elif isinstance(right, ast_ast.NumberConst): r = left.value == right.number_value elif isinstance(right, ast_ast.StringConst): r = left.value == right.string_value else: print "Not const type %s" % repr(right) return False if not r: print "Constant mismatch:", left, right if check_lineno: # left is a stablecompiler.ast node which means and stable compiler # doesn't set a lineno on each Node if left.lineno is not None and left.lineno != right.lineno: print "(0) (%s) left: %s, right: %s" % (left, left.lineno, right.lineno) return False return True elif isinstance(right, ast_ast.Return) and isinstance(left, test_ast.Return): left_nodes = left.getChildren() if right.value is None: right_nodes = (ast_ast.Const(None),) else: right_nodes = right.getChildren() elif isinstance(left,test_ast.Subscript): # test_ast.Subscript is not expressive enough to tell the difference # between a[x] and a[x,] :-( left_nodes = list(left.getChildren()) if len(left.subs) > 1: left_nodes[-len(left.subs):] = [test_ast.Tuple(left_nodes[-len(left.subs):], left.lineno)] right_nodes = right.getChildren() else: left_nodes = left.getChildren() right_nodes = right.getChildren() if len(left_nodes)!=len(right_nodes): print "Number of children mismatch:", left, right return False for i,j in zip(left_nodes,right_nodes): if not nodes_equal(i,j, check_lineno): return False if check_lineno: # left is a stablecompiler.ast node which means and stable compiler # doesn't set a lineno on each Node. # (stablecompiler.ast.Expression doesn't have a lineno attribute) if hasattr(left, 'lineno') and left.lineno is not None and left.lineno != right.lineno: print "(1) (%s) left: %s, right: %s" % (left, left.lineno, right.lineno) return False return True
def visit_Repr(self, rep): w_const = rep.value.as_constant() if w_const is not None: w_repr = self.space.repr(w_const) return ast.Const(w_repr, rep.lineno, rep.col_offset) return rep