def build_atom(builder, nb):
atoms = get_atoms(builder, nb)
top = atoms[0]
if isinstance(top, TokenObject):
# assert isinstance(top, TokenObject) # rtyper
if top.name == builder.parser.tokens['LPAR']:
if len(atoms) == 2:
builder.push(ast.Tuple([], top.lineno))
else:
builder.push(atoms[1])
elif top.name == builder.parser.tokens['LSQB']:
if len(atoms) == 2:
builder.push(ast.List([], top.lineno))
else:
list_node = atoms[1]
list_node.lineno = top.lineno
builder.push(list_node)
elif top.name == builder.parser.tokens['LBRACE']:
items = []
for index in range(1, len(atoms) - 1, 4):
# a : b , c : d
# ^ +1 +2 +3 +4
items.append((atoms[index], atoms[index + 2]))
builder.push(ast.Dict(items, top.lineno))
elif top.name == builder.parser.tokens['NAME']:
val = top.get_value()
builder.push(ast.Name(val, top.lineno))
elif top.name == builder.parser.tokens['NUMBER']:
builder.push(
ast.Const(builder.eval_number(top.get_value()), top.lineno))
elif top.name == builder.parser.tokens['STRING']:
# need to concatenate strings in atoms
s = ''
if len(atoms) == 1:
token = atoms[0]
assert isinstance(token, TokenObject)
builder.push(
ast.Const(
parsestr(builder.space, builder.source_encoding,
token.get_value()), top.lineno))
else:
space = builder.space
empty = space.wrap('')
accum = []
for token in atoms:
assert isinstance(token, TokenObject)
accum.append(
parsestr(builder.space, builder.source_encoding,
token.get_value()))
w_s = space.call_method(empty, 'join', space.newlist(accum))
builder.push(ast.Const(w_s, top.lineno))
elif top.name == builder.parser.tokens['BACKQUOTE']:
builder.push(ast.Backquote(atoms[1], atoms[1].lineno))
else:
raise SyntaxError("unexpected tokens", top.lineno, top.col)
def test_mutate_add(self):
c1 = ast.Const(1)
c2 = ast.Const(2)
add = ast.Add(c1, c2)
class Visitor(BaseVisitor):
def visitAdd(self, node):
return ast.Const(3)
c3 = add.mutate(Visitor())
assert isinstance(c3, ast.Const)
def build_subscript(builder, nb):
"""'.' '.' '.' | [test] ':' [test] [':' [test]] | test"""
atoms = get_atoms(builder, nb)
token = atoms[0]
lineno = token.lineno
if isinstance(token, TokenObject) and token.name == builder.parser.tokens['DOT']:
# Ellipsis:
builder.push(ast.Ellipsis(lineno))
elif len(atoms) == 1:
if isinstance(token, TokenObject) and token.name == builder.parser.tokens['COLON']:
sliceinfos = [None, None, None]
builder.push(SlicelistObject('slice', sliceinfos, lineno))
else:
# test
builder.push(token)
else: # elif len(atoms) > 1:
sliceinfos = [None, None, None]
infosindex = 0
for token in atoms:
if isinstance(token, TokenObject) and token.name == builder.parser.tokens['COLON']:
infosindex += 1
else:
sliceinfos[infosindex] = token
if infosindex == 2:
sliceobj_infos = []
for value in sliceinfos:
if value is None:
sliceobj_infos.append(ast.Const(builder.wrap_none(), lineno))
else:
sliceobj_infos.append(value)
builder.push(SlicelistObject('sliceobj', sliceobj_infos, lineno))
else:
builder.push(SlicelistObject('slice', sliceinfos, lineno))
def visit_UnaryOp(self, unary):
w_operand = unary.operand.as_constant()
op = unary.op
if w_operand is not None:
try:
for op_kind, folder in unrolling_unary_folders:
if op_kind == op:
w_const = folder(self.space, w_operand)
break
else:
raise AssertionError("unknown unary operation")
w_minint = self.space.wrap(-sys.maxint - 1)
# This makes sure the result is an integer.
if self.space.eq_w(w_minint, w_const):
w_const = w_minint
except OperationError:
pass
else:
return ast.Const(w_const, unary.lineno, unary.col_offset)
elif op == ast.Not:
compare = unary.operand
if isinstance(compare, ast.Compare) and len(compare.ops) == 1:
cmp_op = compare.ops[0]
try:
opposite = opposite_compare_operations(cmp_op)
except KeyError:
pass
else:
compare.ops[0] = opposite
return compare
return unary
def visitTuple(self, node):
consts_w = []
for subnode in node.nodes:
if not isinstance(subnode, ast.Const):
return node # not all constants
consts_w.append(subnode.value)
return ast.Const(self.space.newtuple(consts_w))
def visit_Name(self, name):
# Turn loading None into a constant lookup. Eventaully, we can do this
# for True and False, too.
if name.id == "None":
assert name.ctx == ast.Load
return ast.Const(self.space.w_None, name.lineno, name.col_offset)
return name
def visit_BinOp(self, binop):
left = binop.left.as_constant()
if left is not None:
right = binop.right.as_constant()
if right is not None:
op = binop.op
# Can't fold straight division without "from __future_ import
# division" because it might be affected at runtime by the -Q
# flag.
if op == ast.Div and \
not self.compile_info.flags & consts.CO_FUTURE_DIVISION:
return binop
try:
for op_kind, folder in unrolling_binary_folders:
if op_kind == op:
w_const = folder(self.space, left, right)
break
else:
raise AssertionError("unknown binary operation")
# Let all errors be found at runtime.
except OperationError:
pass
else:
# To avoid blowing up the size of pyc files, we only fold
# reasonably sized sequences.
try:
w_len = self.space.len(w_const)
except OperationError:
pass
else:
if self.space.int_w(w_len) > 20:
return binop
return ast.Const(w_const, binop.lineno, binop.col_offset)
return binop
def visit_BinOp(self, binop):
left = binop.left.as_constant()
if left is not None:
right = binop.right.as_constant()
if right is not None:
op = binop.op
try:
for op_kind, folder in unrolling_binary_folders:
if op_kind == op:
w_const = folder(self.space, left, right)
break
else:
raise AssertionError("unknown binary operation")
# Let all errors be found at runtime.
except OperationError:
pass
else:
# To avoid blowing up the size of pyc files, we only fold
# reasonably sized sequences.
try:
w_len = self.space.len(w_const)
except OperationError:
pass
else:
if self.space.int_w(w_len) > 20:
return binop
return ast.Const(w_const, binop.lineno, binop.col_offset)
return binop
def as_node_list(self, space):
try:
values_w = space.unpackiterable(self.value)
except OperationError:
return None
line = self.lineno
column = self.col_offset
return [ast.Const(w_obj, line, column) for w_obj in values_w]
def _visitUnaryOp(self, node, constant_fold):
expr = node.expr
if isinstance(expr, ast.Const):
try:
w_newvalue = constant_fold(self.space, expr.value)
except OperationError:
pass
else:
return ast.Const(w_newvalue)
return node
def visit_Name(self, name):
# Turn loading None into a constant lookup. We cannot do this
# for True and False, because rebinding them is allowed (2.7).
if name.id == "None":
# The compiler refuses to parse "None = ...", but "del None"
# is allowed (if pointless). Check anyway: custom asts that
# correspond to "None = ..." can be made by hand.
if name.ctx == ast.Load:
return ast.Const(self.space.w_None, name.lineno,
name.col_offset)
return name
def build_trailer(builder, nb):
"""trailer: '(' ')' | '(' arglist ')' | '[' subscriptlist ']' | '.' NAME
"""
atoms = get_atoms(builder, nb)
first_token = atoms[0]
# Case 1 : '(' ...
if isinstance(
first_token,
TokenObject) and first_token.name == builder.parser.tokens['LPAR']:
if len(atoms
) == 2: # and atoms[1].token == builder.parser.tokens['RPAR']:
builder.push(ArglistObject([], None, None, first_token.lineno))
elif len(atoms) == 3: # '(' Arglist ')'
# push arglist on the stack
builder.push(atoms[1])
elif isinstance(
first_token,
TokenObject) and first_token.name == builder.parser.tokens['LSQB']:
if len(atoms) == 3 and isinstance(atoms[1], SlicelistObject):
builder.push(atoms[1])
else:
# atoms is a list of, alternatively, values and comma tokens,
# with '[' and ']' tokens at the end
subs = []
for index in range(1, len(atoms) - 1, 2):
atom = atoms[index]
if isinstance(atom, SlicelistObject):
num_slicevals = 3
slicevals = []
if atom.fake_rulename == 'slice':
num_slicevals = 2
for val in atom.value[:num_slicevals]:
if val is None:
slicevals.append(
ast.Const(builder.wrap_none(), atom.lineno))
else:
slicevals.append(val)
subs.append(ast.Sliceobj(slicevals, atom.lineno))
else:
subs.append(atom)
if len(atoms) > 3: # at least one comma
sub = ast.Tuple(subs, first_token.lineno)
else:
[sub] = subs
builder.push(SubscriptObject('subscript', sub, first_token.lineno))
elif len(atoms) == 2:
# Attribute access: '.' NAME
builder.push(atoms[0])
builder.push(atoms[1])
builder.push(
TempRuleObject('pending-attr-access', 2, first_token.lineno))
else:
assert False, "Trailer reducing implementation incomplete !"
def visit_Subscript(self, subs):
if subs.ctx == ast.Load:
w_obj = subs.value.as_constant()
if w_obj is not None:
w_idx = subs.slice.as_constant()
if w_idx is not None:
try:
return ast.Const(self.space.getitem(w_obj, w_idx),
subs.lineno, subs.col_offset)
except OperationError:
# Let exceptions propgate at runtime.
pass
return subs
def visit_Tuple(self, tup):
"""Try to turn tuple building into a constant."""
if tup.elts:
consts_w = [None] * len(tup.elts)
for i in range(len(tup.elts)):
node = tup.elts[i]
w_const = node.as_constant()
if w_const is None:
return tup
consts_w[i] = w_const
else:
consts_w = []
w_consts = self.space.newtuple(consts_w)
return ast.Const(w_consts, tup.lineno, tup.col_offset)
def build_simple_stmt(builder, nb):
atoms = get_atoms(builder, nb)
l = len(atoms)
nodes = []
if atoms:
lineno = atoms[0].lineno
else:
lineno = -1
for n in range(0,l,2):
node = atoms[n]
if isinstance(node, TokenObject) and node.name == builder.parser.tokens['NEWLINE']:
nodes.append(ast.Discard(ast.Const(builder.wrap_none()), node.lineno))
else:
nodes.append(node)
builder.push(ast.Stmt(nodes, lineno))
def visit_Name(self, name):
"""Turn loading None, True, and False into a constant lookup."""
if name.ctx == ast.Del:
return name
space = self.space
iden = name.id
w_const = None
if iden == "None":
w_const = space.w_None
elif iden == "True":
w_const = space.w_True
elif iden == "False":
w_const = space.w_False
if w_const is not None:
return ast.Const(w_const, name.lineno, name.col_offset)
return name
def _visitBitOp(self, node, constant_fold):
while len(node.nodes) >= 2:
left = node.nodes[0]
if not isinstance(left, ast.Const):
return node # done
right = node.nodes[1]
if not isinstance(right, ast.Const):
return node # done
try:
w_newvalue = constant_fold(self.space, left.value,
right.value)
except OperationError:
return node # done
del node.nodes[1]
node.nodes[0] = ast.Const(w_newvalue)
else:
# if reduced to a single node, just returns it
return node.nodes[0]
def visit_Tuple(self, tup):
"""Try to turn tuple building into a constant."""
if tup.elts:
consts_w = [None]*len(tup.elts)
for i in range(len(tup.elts)):
node = tup.elts[i]
w_const = node.as_constant()
if w_const is None:
return tup
consts_w[i] = w_const
# intern the string constants packed into the tuple here,
# because assemble.py will see the result as just a tuple constant
for i in range(len(consts_w)):
consts_w[i] = misc.intern_if_common_string(
self.space, consts_w[i])
else:
consts_w = []
w_consts = self.space.newtuple(consts_w)
return ast.Const(w_consts, tup.lineno, tup.col_offset)
def _visitBinaryOp(self, node, constant_fold):
left = node.left
right = node.right
if isinstance(left, ast.Const) and isinstance(right, ast.Const):
try:
w_newvalue = constant_fold(self.space, left.value,
right.value)
except OperationError:
pass
else:
# to avoid creating too large .pyc files, we don't
# constant-fold operations that create long sequences,
# like '(5,) * 500'. This is the same as CPython.
try:
size = self.space.int_w(self.space.len(w_newvalue))
except OperationError:
size = -1
if size <= 20:
return ast.Const(w_newvalue)
return node
def build_exprlist(builder, nb):
"""exprlist: expr (',' expr)* [',']"""
atoms = get_atoms(builder, nb)
if len(atoms) <= 2:
builder.push(atoms[0])
else:
names = []
values = []
isConst = True
for index in range(0, len(atoms), 2):
item = atoms[index]
names.append(item)
if isinstance(item, ast.Const):
values.append(item)
else:
isConst = False
if isConst:
builder.push(ast.Const(builder.space.newtuple(values), atoms[0].lineno))
else:
builder.push(ast.Tuple(names, atoms[0].lineno))
def build_testlist_gexp(builder, nb):
atoms = get_atoms(builder, nb)
if atoms:
lineno = atoms[0].lineno
else:
lineno = -1
l = len(atoms)
if l == 1:
builder.push(atoms[0])
return
items = []
token = atoms[1]
if isinstance(
token,
TokenObject) and token.name == builder.parser.tokens['COMMA']:
for i in range(0, l, 2): # this is atoms not 1
items.append(atoms[i])
else:
# genfor: 'i for i in j'
# GenExpr(GenExprInner(Name('i'), [GenExprFor(AssName('i', 'OP_ASSIGN'), Name('j'), [])])))]))
expr = atoms[0]
genexpr_for = parse_genexpr_for(atoms[1:])
genexpr_for[0].is_outmost = True
builder.push(
ast.GenExpr(ast.GenExprInner(expr, genexpr_for, lineno), lineno))
return
isConst = True
values = []
for item in items:
if isinstance(item, ast.Const):
values.append(item.value)
else:
isConst = False
break
if isConst:
builder.push(ast.Const(builder.space.newtuple(values), lineno))
else:
builder.push(ast.Tuple(items, lineno))
return
def visit_Subscript(self, subs):
if subs.ctx == ast.Load:
w_obj = subs.value.as_constant()
if w_obj is not None:
w_idx = subs.slice.as_constant()
if w_idx is not None:
try:
w_const = self.space.getitem(w_obj, w_idx)
except OperationError:
# Let exceptions propagate at runtime.
return subs
# CPython issue5057: if v is unicode, there might
# be differences between wide and narrow builds in
# cases like u'\U00012345'[0].
# Wide builds will return a non-BMP char, whereas
# narrow builds will return a surrogate. In both
# the cases skip the optimization in order to
# produce compatible pycs.
if (self.space.isinstance_w(w_obj, self.space.w_unicode)
and self.space.isinstance_w(
w_const, self.space.w_unicode)):
#unistr = self.space.unicode_w(w_const)
#if len(unistr) == 1:
# ch = ord(unistr[0])
#else:
# ch = 0
#if (ch > 0xFFFF or
# (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)):
# --XXX-- for now we always disable optimization of
# u'...'[constant] because the tests above are not
# enough to fix issue5057 (CPython has the same
# problem as of April 24, 2012).
# See test_const_fold_unicode_subscr
return subs
return ast.Const(w_const, subs.lineno, subs.col_offset)
return subs
def visitAdd(self, node):
return ast.Const(3)
def nodes_equal(left, right, check_lineno=False):
if isinstance(left, ast_ast.Node) and isinstance(right, ast_ast.Node):
# direct comparison
if left.__class__ is not right.__class__:
print "Node type mismatch:", left, right
return False
if check_lineno and left.lineno != right.lineno:
print "lineno mismatch in (%s) left: %s, right: %s" % (left, left.lineno, right.lineno)
return False
left_nodes = list(left.getChildren())
right_nodes = list(right.getChildren())
if len(left_nodes) != len(right_nodes):
print "Number of children mismatch:", left, right
return False
for left_node, right_node in zip(left_nodes, right_nodes):
if not nodes_equal(left_node, right_node, check_lineno):
return False
return True
if not isinstance(left,test_ast.Node) or not isinstance(right,ast_ast.Node):
return left==right
if left.__class__.__name__ != right.__class__.__name__:
print "Node type mismatch:", left, right
return False
if isinstance(left,test_ast.Function) and isinstance(right,ast_ast.Function):
left_nodes = list(left.getChildren())
right_nodes = [] # generated ast differ here because argnames is a list of nodes in
right_nodes.append(right.decorators)
right_nodes.append(right.name)
right_nodes.append(right.argnames)
right_nodes.extend(flatten(right.defaults))
right_nodes.append(right.flags)
right_nodes.append(right.w_doc)
right_nodes.append(right.code)
left_args = left_nodes[2]
del left_nodes[2]
right_args = right_nodes[2]
del right_nodes[2]
if not arglist_equal(left_args, right_args):
return False
elif isinstance(left,test_ast.Lambda) and isinstance(right,ast_ast.Lambda):
left_nodes = list(left.getChildren())
right_nodes = [] # generated ast differ here because argnames is a list of nodes in
right_nodes.append(right.argnames)
right_nodes.extend(flatten(right.defaults))
right_nodes.append(right.flags)
right_nodes.append(right.code)
print "left", repr(left_nodes)
print "right", repr(right_nodes)
left_args = left_nodes[0]
del left_nodes[0]
right_args = right_nodes[0]
del right_nodes[0]
if not arglist_equal(left_args, right_args):
return False
elif isinstance(left,test_ast.Const):
if isinstance(right,ast_ast.Const):
r = left.value == right.value
elif isinstance(right,ast_ast.NoneConst):
r = left.value == None
elif isinstance(right, ast_ast.NumberConst):
r = left.value == right.number_value
elif isinstance(right, ast_ast.StringConst):
r = left.value == right.string_value
else:
print "Not const type %s" % repr(right)
return False
if not r:
print "Constant mismatch:", left, right
if check_lineno:
# left is a stablecompiler.ast node which means and stable compiler
# doesn't set a lineno on each Node
if left.lineno is not None and left.lineno != right.lineno:
print "(0) (%s) left: %s, right: %s" % (left, left.lineno, right.lineno)
return False
return True
elif isinstance(right, ast_ast.Return) and isinstance(left, test_ast.Return):
left_nodes = left.getChildren()
if right.value is None:
right_nodes = (ast_ast.Const(None),)
else:
right_nodes = right.getChildren()
elif isinstance(left,test_ast.Subscript):
# test_ast.Subscript is not expressive enough to tell the difference
# between a[x] and a[x,] :-(
left_nodes = list(left.getChildren())
if len(left.subs) > 1:
left_nodes[-len(left.subs):] = [test_ast.Tuple(left_nodes[-len(left.subs):],
left.lineno)]
right_nodes = right.getChildren()
else:
left_nodes = left.getChildren()
right_nodes = right.getChildren()
if len(left_nodes)!=len(right_nodes):
print "Number of children mismatch:", left, right
return False
for i,j in zip(left_nodes,right_nodes):
if not nodes_equal(i,j, check_lineno):
return False
if check_lineno:
# left is a stablecompiler.ast node which means and stable compiler
# doesn't set a lineno on each Node.
# (stablecompiler.ast.Expression doesn't have a lineno attribute)
if hasattr(left, 'lineno') and left.lineno is not None and left.lineno != right.lineno:
print "(1) (%s) left: %s, right: %s" % (left, left.lineno, right.lineno)
return False
return True
def visit_Repr(self, rep):
w_const = rep.value.as_constant()
if w_const is not None:
w_repr = self.space.repr(w_const)
return ast.Const(w_repr, rep.lineno, rep.col_offset)
return rep