def ROT_TWO(self, instr):
one = self.ast_stack.pop()
two = self.ast_stack.pop()
if self.ilst[0].opname == 'STORE_NAME':
kw = dict(lineno=instr.lineno, col_offset=0)
stores = []
while self.ilst[0].opname == 'STORE_NAME':
stores.append(self.ilst.pop(0))
assert len(stores) <= 3, stores
elts_load = [one, two]
if len(stores) == 3:
elts_load.insert(0, self.ast_stack.pop())
tup_load = _ast.Tuple(elts=elts_load[::-1], ctx=_ast.Load(), **kw)
elts_store = [
_ast.Name(id=store.arg, ctx=_ast.Store(), **kw)
for store in stores
]
tup_store = _ast.Tuple(elts=elts_store, ctx=_ast.Store(), **kw)
assgn = _ast.Assign(value=tup_load, targets=[tup_store], **kw)
self.ast_stack.append(assgn)
# self.ast_stack.append(tup_store)
else:
self.ast_stack.append(one)
self.ast_stack.append(two)
def make_assign_unpack(i, bytecode, unpack_num=-1):
if unpack_num < 1:
logger.error("Could not find the number of unpacked items. ")
return i, None
store_exprs = []
value_exprs = []
store_state, value_state = True, False
while i >= 0:
op, arg = bytecode[i][2], bytecode[i][3]
if store_state:
if op == UNPACK_SEQUENCE:
store_state = False
prev_op = bytecode[i - 1][2] if i > 0 else -1
if prev_op == BUILD_TUPLE:
value_state = True
else:
i, value_exprs = Statement.make_expr(i - 1, bytecode)
break
elif op in STORE_OPCODES:
i, store_stmt = Statement.make_expr(i, bytecode, context=_ast.Store())
store_exprs.insert(0, store_stmt)
elif value_state:
i, value_stmt = Statement.make_expr(i, bytecode)
value_exprs.insert(0, value_stmt)
i -= 1
store_exprs = _ast.Tuple(store_exprs, _ast.Store())
if not isinstance(value_exprs, _ast.AST):
value_exprs = _ast.Tuple(value_exprs, _ast.Load())
return i, _ast.Assign([store_exprs], value_exprs)
def STORE_NAME(self, instr):
value = self.ast_stack.pop()
value = self.process_ifexpr(value)
if isinstance(value, _ast.Import):
if value.from_:
assert isinstance(self.ast_stack[-1], _ast.ImportFrom)
from_ = self.ast_stack.pop()
as_name = instr.arg
name = from_.names[0].name
if as_name != name:
from_.names[0].asname = as_name
self.ast_stack.append(from_)
else:
as_name = instr.arg
if value.names[0].asname is None:
base_name = value.names[0].name.split('.')[0]
if base_name != as_name:
value.names[0].asname = as_name
self.ast_stack.append(value)
elif isinstance(value, (_ast.Attribute)) and isinstance(
value.value, (_ast.Import)):
asname = instr.arg
value = value.value
value.names[0].asname = asname
self.ast_stack.append(value)
elif isinstance(value, (_ast.ClassDef, _ast.FunctionDef)):
as_name = instr.arg
value.name = as_name
self.ast_stack.append(value)
elif isinstance(value, _ast.AugAssign):
self.ast_stack.append(value)
elif isinstance(value, _ast.Assign):
_ = self.ast_stack.pop()
assname = _ast.Name(instr.arg,
_ast.Store(),
lineno=instr.lineno,
col_offset=0)
value.targets.append(assname)
self.ast_stack.append(value)
else:
assname = _ast.Name(instr.arg,
_ast.Store(),
lineno=instr.lineno,
col_offset=0)
assign = _ast.Assign(targets=[assname],
value=value,
lineno=instr.lineno,
col_offset=0)
self.ast_stack.append(assign)
def make_assign_opt_unpack(i, bytecode):
store_exprs = []
value_exprs = []
store_state, value_state = True, False
while i >= 0:
op, arg = bytecode[i][2], bytecode[i][3]
if store_state:
if op == ROT_TWO:
prev_op = bytecode[i - 1][2] if i > 0 else -1
if prev_op == ROT_THREE:
i -= 1
value_state = True
store_state = False
elif op in STORE_OPCODES:
i, store_stmt = Statement.make_expr(i, bytecode, context=_ast.Store())
store_exprs.insert(0, store_stmt)
elif value_state:
i, value_stmt = Statement.make_expr(i, bytecode)
value_exprs.insert(0, value_stmt)
i -= 1
store_exprs = _ast.Tuple(store_exprs, _ast.Store())
if not isinstance(value_exprs, _ast.AST):
value_exprs = _ast.Tuple(value_exprs, _ast.Load())
return i, _ast.Assign([store_exprs], value_exprs)
def UNPACK_SEQUENCE(self, instr):
nargs = instr.oparg
nodes = []
ast_tuple = _ast.Tuple(elts=nodes,
ctx=_ast.Store(),
lineno=instr.lineno,
col_offset=0)
for i in range(nargs):
nex_instr = self.ilst.pop(0)
self.ast_stack.append(None)
self.visit(nex_instr)
node = self.ast_stack.pop()
nodes.append(node.targets[0])
expr = self.ast_stack.pop()
if isinstance(expr, _ast.Assign):
assgn = expr
assgn.targets.append(ast_tuple)
value_dup = self.ast_stack.pop()
assert cmp_ast(assgn.value, value_dup)
else:
assgn = _ast.Assign(targets=[ast_tuple],
value=expr,
lineno=instr.lineno,
col_offset=0)
self.ast_stack.append(assgn)
def test_constructor(self):
ast = self.ast
body = []
mod = ast.Module(body)
assert mod.body is body
target = ast.Name("hi", ast.Store())
expr = ast.Name("apples", ast.Load())
otherwise = []
fr = ast.For(target, expr, body, otherwise, lineno=0, col_offset=1)
assert fr.target is target
assert fr.iter is expr
assert fr.orelse is otherwise
assert fr.body is body
assert fr.lineno == 0
assert fr.col_offset == 1
fr = ast.For(body=body,
target=target,
iter=expr,
col_offset=1,
lineno=0,
orelse=otherwise)
assert fr.target is target
assert fr.iter is expr
assert fr.orelse is otherwise
assert fr.body is body
assert fr.lineno == 0
assert fr.col_offset == 1
exc = raises(TypeError, ast.Module, 1, 2).value
msg = str(exc)
assert msg == "Module constructor takes either 0 or 1 positional argument"
ast.Module(nothing=23)
def make_subscript(i, bytecode, context=None):
op = bytecode[i][2]
if op == STORE_SUBSCR:
# TOS1[TOS] = TOS2
i, index_expr = Statement.make_expr(i - 1, bytecode)
i, arr_expr = Statement.make_expr(i - 1, bytecode, context=_ast.Store())
i, rhs_expr = Statement.make_expr(i - 1, bytecode)
lhs_expr = _ast.Subscript(arr_expr, index_expr, _ast.Store())
return i, _ast.Assign([lhs_expr], rhs_expr)
else:
if context is None:
context = _ast.Load()
# BINARY_SUBSCR: TOS1[TOS] and DELETE_SUBSCR TOS1[TOS]
i, index_expr = Statement.make_expr(i - 1, bytecode)
i, arr_expr = Statement.make_expr(i - 1, bytecode)
return i, _ast.Subscript(arr_expr, index_expr, context)
def INPLACE_OP(self, instr):
right = self.ast_stack.pop()
left = self.ast_stack.pop()
left.ctx = _ast.Store()
aug_assign = _ast.AugAssign(target=left, op=OP(), value=right, lineno=instr.lineno, col_offset=0)
self.ast_stack.append(aug_assign)
def test_list_syncing(self):
ast = self.ast
mod = ast.Module([ast.Lt()])
raises(TypeError, compile, mod, "<string>", "exec")
mod = self.get_ast("x = y = 3")
assign = mod.body[0]
assert len(assign.targets) == 2
assign.targets[1] = ast.Name("lemon", ast.Store(),
lineno=0, col_offset=0)
name = ast.Name("apple", ast.Store(),
lineno=0, col_offset=0)
mod.body.append(ast.Assign([name], ast.Num(4, lineno=0, col_offset=0),
lineno=0, col_offset=0))
co = compile(mod, "<test>", "exec")
ns = {}
exec co in ns
assert "y" not in ns
assert ns["x"] == ns["lemon"] == 3
assert ns["apple"] == 4
def GetCtx(ctx_type):
"""Creates Load, Store, Del, and Param, used in the ctx kwarg."""
if ctx_type == CtxEnum.LOAD:
return _ast.Load()
elif ctx_type == CtxEnum.STORE:
return _ast.Store()
elif ctx_type == CtxEnum.DEL:
return _ast.Del()
elif ctx_type == CtxEnum.PARAM:
return _ast.Param()
raise InvalidCtx('ctx_type {} isn\'t a valid type'.format(ctx_type))
def STORE_SLICE_0(self, instr):
'obj[:] = expr'
value = self.ast_stack.pop()
expr = self.ast_stack.pop()
kw = dict(lineno=instr.lineno, col_offset=0)
slice = _ast.Slice(lower=None, step=None, upper=None, **kw)
subscr = _ast.Subscript(value=value, slice=slice, ctx=_ast.Store(), **kw)
assign = _ast.Assign(targets=[subscr], value=expr, **kw)
self.ast_stack.append(assign)
def STORE_ATTR(self, instr):
attrname = instr.arg
node = self.ast_stack.pop()
expr = self.ast_stack.pop()
expr = self.process_ifexpr(expr)
assattr = _ast.Attribute(value=node, attr=attrname, ctx=_ast.Store(), lineno=instr.lineno, col_offset=0)
set_attr = _ast.Assign(targets=[assattr], value=expr, lineno=instr.lineno, col_offset=0)
self.ast_stack.append(set_attr)
def INPLACE_OP(self, instr):
right = self.pop_ast_item()
left = self.pop_ast_item()
left.ctx = _ast.Store()
aug_assign = _ast.AugAssign(target=left,
op=OP(),
value=right,
lineno=instr.lineno,
col_offset=0)
self.push_ast_item(aug_assign)
def make_store_delete_slice(i, bytecode, context=None):
op = bytecode[i][2]
is_delete = op in DELETE_SLICE_OPCODES
if context is None:
context = _ast.Store() if not is_delete else _ast.Del()
lhs_expr = None
if op in (STORE_SLICE_0, DELETE_SLICE_0):
i, lhs_expr = Statement.make_expr(i - 1, bytecode, context=context)
lhs_expr = _ast.Subscript(lhs_expr,
_ast.Slice(None, None, None),
_ast.Store())
elif op in (STORE_SLICE_1, STORE_SLICE_2, DELETE_SLICE_1, DELETE_SLICE_2):
i, index_expr = Statement.make_expr(i - 1, bytecode)
i, arr_expr = Statement.make_expr(i - 1, bytecode, context=context)
args = [None] * 3
index_index = 0 if op in (STORE_SLICE_1, DELETE_SLICE_1) else 1
args[index_index] = index_expr
lhs_expr = _ast.Subscript(arr_expr,
_ast.Slice(*args),
_ast.Store())
else:
i, end_index_expr = Statement.make_expr(i - 1, bytecode)
i, start_index_expr = Statement.make_expr(i - 1, bytecode)
i, arr_expr = Statement.make_expr(i - 1, bytecode, context=context)
lhs_expr = _ast.Subscript(arr_expr,
_ast.Slice(start_index_expr, end_index_expr, None),
_ast.Store())
if is_delete:
return i, _ast.Delete([lhs_expr])
else:
i, rhs_expr = Statement.make_expr(i - 1, bytecode)
return i, _ast.Assign([lhs_expr], rhs_expr)
def make_assign_chained(i, bytecode):
store_exprs = []
value_exprs = []
store_state, value_state = True, False
while i >= 0:
op, arg = bytecode[i][2], bytecode[i][3]
if store_state:
if op == DUP_TOP:
prev_op = bytecode[i - 1][2] if i > 0 else -1
if prev_op not in STORE_OPCODES:
value_state = True
store_state = False
elif op in STORE_OPCODES:
i, store_stmt = Statement.make_expr(i, bytecode, context=_ast.Store())
store_exprs.insert(0, store_stmt)
elif value_state:
i, value_exprs = Statement.make_expr(i, bytecode)
break
i -= 1
store_exprs = _ast.Tuple(store_exprs, _ast.Store())
return i, _ast.Assign([store_exprs], value_exprs)
def STORE_SLICE_2(self, instr):
'obj[:upper] = expr'
upper = self.pop_ast_item()
value = self.pop_ast_item()
expr = self.pop_ast_item()
kw = dict(lineno=instr.lineno, col_offset=0)
slice = _ast.Slice(lower=None, step=None, upper=upper, **kw)
subscr = _ast.Subscript(value=value,
slice=slice,
ctx=_ast.Store(),
**kw)
assign = _ast.Assign(targets=[subscr], value=expr, **kw)
self.push_ast_item(assign)
def visit_Expr(self, expression_node):
value = expression_node.value
if not isinstance(value, _ast.BinOp) or not (isinstance(
value.op, _ast.RShift) or isinstance(value.op, _ast.LShift)):
return expression_node
mock_attribute = 'return_value'
if isinstance(value.op, _ast.LShift):
mock_attribute = 'side_effect'
return _ast.Assign(targets=[
_ast.Attribute(attr=mock_attribute,
ctx=_ast.Store(),
value=value.left.func)
],
value=value.right)
def _assign_list_form_variables(self, where_function_ast_node):
copy_of_body = copy.deepcopy(where_function_ast_node.body)
where_function_ast_node.body = []
for assignment_expression in copy_of_body:
variable_name = assignment_expression.targets[0].id
self.spec_metadata.add_feature_variable(self.feature_name,
variable_name)
variable_values = assignment_expression.value
where_function_ast_node.body.append(
_ast.Assign(targets=[
_ast.Subscript(
value=_ast.Name(id='injectable_values',
ctx=_ast.Load()),
slice=_ast.Index(value=_ast.Str(s=variable_name)),
ctx=_ast.Store())
],
value=variable_values))
def STORE_SUBSCR(self, instr):
index = self.ast_stack.pop()
value = self.ast_stack.pop()
expr = self.ast_stack.pop()
expr = self.process_ifexpr(expr)
if isinstance(expr, _ast.AugAssign):
self.ast_stack.append(expr)
else:
kw = dict(lineno=instr.lineno, col_offset=0)
index = self.format_slice(index, kw)
subscr = _ast.Subscript(value=value, slice=index, ctx=_ast.Store(), **kw)
assign = _ast.Assign(targets=[subscr], value=expr, **kw)
self.ast_stack.append(assign)
def _assign_matrix_form_variables(self, where_function_ast_node):
copy_of_body = copy.deepcopy(where_function_ast_node.body)
where_function_ast_node.body = []
variables_and_values = WhereBlockFunctions._get_variables_and_values(copy_of_body)
# We might be screwing with line numbers here
for variable_name, variable_values in variables_and_values.items():
self.spec_metadata.add_feature_variable(self.feature_name, variable_name)
where_function_ast_node.body.append(
_ast.Assign(
targets=[
_ast.Subscript(
value=_ast.Name(id='injectable_values', ctx=_ast.Load()),
slice=_ast.Index(value=ast_proxy.ast_str(s=variable_name)),
ctx=_ast.Store()
)
],
value=_ast.List(elts=variable_values, ctx=_ast.Load())
))
def test_bug_null_in_objspace_type(self):
import ast
code = ast.Expression(
lineno=1,
col_offset=1,
body=ast.ListComp(
lineno=1,
col_offset=1,
elt=ast.Call(lineno=1,
col_offset=1,
func=ast.Name(lineno=1,
col_offset=1,
id='str',
ctx=ast.Load(lineno=1,
col_offset=1)),
args=[
ast.Name(lineno=1,
col_offset=1,
id='x',
ctx=ast.Load(lineno=1, col_offset=1))
],
keywords=[]),
generators=[
ast.comprehension(
lineno=1,
col_offset=1,
target=ast.Name(lineno=1,
col_offset=1,
id='x',
ctx=ast.Store(lineno=1, col_offset=1)),
iter=ast.List(
lineno=1,
col_offset=1,
elts=[ast.Num(lineno=1, col_offset=1, n=23)],
ctx=ast.Load(
lineno=1,
col_offset=1,
)),
ifs=[])
]))
compile(code, '<template>', 'eval')
def STORE_SLICE_3(self, instr):
'obj[lower:upper] = expr'
upper = self.ast_stack.pop()
lower = self.ast_stack.pop()
value = self.ast_stack.pop()
expr = self.ast_stack.pop()
kw = dict(lineno=instr.lineno, col_offset=0)
slice = _ast.Slice(lower=lower, step=None, upper=upper, **kw)
subscr = _ast.Subscript(value=value, slice=slice, ctx=_ast.Store(), **kw)
if isinstance(expr, _ast.AugAssign):
assign = expr
result = cmp_ast(expr.target, subscr)
assert result
else:
assign = _ast.Assign(targets=[subscr], value=expr, **kw)
self.ast_stack.append(assign)
def _cfa(body, state, on_gen):
assert isinstance(on_gen, list)
for c in on_gen:
assert callable(c)
cfg = state.cfg
def make_connect(cur_bid):
assert isinstance(cur_bid, int)
def inner(new_bid):
assert isinstance(new_bid, int)
state.connect([cur_bid], [new_bid])
return inner
def push_block(block):
block_id = len(cfg.blocks)
assert block_id not in cfg.blocks
cfg.blocks[block_id] = block
for c in on_gen:
c(block_id)
return block_id
cur_block = []
for b in body:
if REDUCE_FORS_TO_WHILES and isinstance(b, _ast.For):
if isinstance(b.iter, _ast.Call) and isinstance(
b.iter.func,
_ast.Name) and b.iter.func.id in ("range", "xrange"):
if not b.iter.keywords and not b.iter.starargs and not b.iter.kwargs:
end_var = "__wfend_%d_%d_" % (b.lineno, b.col_offset)
iter_var = "__wfiter_%d_%d_" % (b.lineno, b.col_offset)
if len(b.iter.args) in (1, 2):
if len(b.iter.args) == 1:
start = _ast.Num(0)
end = b.iter.args[0]
elif len(b.iter.args) == 2:
start = b.iter.args[0]
end = b.iter.args[1]
else:
start = b.iter.args[0]
end = b.iter.args[1]
cur_block.append(
_ast.Assign([
_ast.Name(
iter_var, _ast.Store(), not_real=True)
],
start,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True))
cur_block.append(
_ast.Assign([
_ast.Name(end_var, _ast.Store(), not_real=True)
],
end,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True))
body = [
_ast.Assign([b.target],
_ast.Name(iter_var,
_ast.Load(),
not_real=True),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True),
_ast.Assign([
_ast.Name(
iter_var, _ast.Store(), not_real=True)
],
_ast.BinOp(
_ast.Name(iter_var,
_ast.Load(),
not_real=True),
_ast.Add(), _ast.Num(1)),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
] + b.body
b = _ast.While(_ast.Compare(
_ast.Name(iter_var, _ast.Load(),
not_real=True), [_ast.Lt()],
[_ast.Name(end_var, _ast.Load(), not_real=True)],
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True),
body,
b.orelse,
not_real=True)
if isinstance(b, (
_ast.Assign,
_ast.AugAssign,
_ast.ClassDef,
_ast.Delete,
_ast.Exec,
_ast.Expr,
_ast.FunctionDef,
_ast.Global,
_ast.Import,
_ast.ImportFrom,
_ast.Print,
_ast.Pass,
)):
cur_block.append(b)
elif isinstance(b, _ast.Assert):
cur_block.append(b)
if isinstance(b.test, _ast.Call) and isinstance(
b.test.func, _ast.Name
) and b.test.func.id == "isinstance" and isinstance(
b.test.args[0], _ast.Name) and isinstance(
b.test.args[1], _ast.Name):
varname = b.test.args[0].id
cast = _ast.Call(_ast.Name(
"__cast__", _ast.Load(), not_real=True, **pos(b)), [
_ast.Name(
varname, _ast.Store(), not_real=True, **pos(b)),
b.test.args[1]
], [],
None,
None,
not_real=True,
**pos(b))
assign = _ast.Assign([
_ast.Name(varname, _ast.Store(), not_real=True, **pos(b))
],
cast,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
cur_block.append(assign)
elif isinstance(b, (_ast.Break, _ast.Continue)):
f = state.add_break if isinstance(
b, _ast.Break) else state.add_continue
if cur_block:
j = Jump()
cur_block.append(j)
block_id = push_block(cur_block)
f(j.set_dest)
f(make_connect(block_id))
else:
for c in on_gen:
f(c)
return []
elif isinstance(b, _ast.If):
br = Branch(b.test, lineno=b.lineno)
cur_block.append(br)
next_block = push_block(cur_block)
on_gen = None # make sure this doesn't get used
cur_block = []
gen_true = [br.set_true, make_connect(next_block)]
gen_false = [br.set_false, make_connect(next_block)]
if ENFORCE_NO_MULTIMULTI:
on_gen = gen_true
j1 = Jump()
gen_true = [make_connect(push_block([j1])), j1.set_dest]
on_gen = gen_false
j2 = Jump()
gen_false = [make_connect(push_block([j2])), j2.set_dest]
on_gen = None
assert b.body
body = b.body
if isinstance(b.test, _ast.Call) and isinstance(
b.test.func, _ast.Name
) and b.test.func.id == "isinstance" and isinstance(
b.test.args[0], _ast.Name) and isinstance(
b.test.args[1], _ast.Name):
varname = b.test.args[0].id
cast = _ast.Call(_ast.Name(
"__cast__", _ast.Load(), not_real=True, **pos(b)), [
_ast.Name(
varname, _ast.Store(), not_real=True, **pos(b)),
b.test.args[1]
], [],
None,
None,
not_real=True,
**pos(b))
assign = _ast.Assign([
_ast.Name(varname, _ast.Store(), not_real=True, **pos(b))
],
cast,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
body = [assign] + body
if ADD_IF_ASSERTS:
body = [_ast.Assert(b.test, None, not_real=True, **pos(b))
] + body
ending_gen = _cfa(body, state, gen_true)
if b.orelse:
ending_gen += _cfa(b.orelse, state, gen_false)
else:
ending_gen += gen_false
on_gen = ending_gen
if not on_gen and PRUNE_UNREACHABLE_BLOCKS:
return []
elif isinstance(b, _ast.TryExcept):
j = Jump()
cur_block.append(j)
next_block = push_block(cur_block)
on_gen = [j.set_dest, make_connect(next_block)]
cur_block = []
on_gen = _cfa(b.body, state, on_gen)
# Set this to evaluate a string to try to defeat simple flow analysis
br = Branch(_ast.Str("nonzero"))
next_block = push_block([br])
on_except = [br.set_false, make_connect(next_block)]
on_fine = [br.set_true, make_connect(next_block)]
assert len(b.handlers) >= 1
# for handler in b.handlers:
# on_except = _cfa(b.handlers[0].body, state, on_except)
if b.orelse:
on_fine = _cfa(b.orelse, state, on_fine)
if ENFORCE_NO_MULTIMULTI:
j = Jump()
on_gen = on_fine
next_block = push_block([j])
on_fine = [j.set_dest, make_connect(next_block)]
j = Jump()
on_gen = on_except
next_block = push_block([j])
on_except = [j.set_dest, make_connect(next_block)]
on_gen = on_fine + on_except
cur_block = []
elif isinstance(b, _ast.TryFinally):
j = Jump()
cur_block.append(j)
next_block = push_block(cur_block)
on_gen = [j.set_dest, make_connect(next_block)]
cur_block = []
on_gen = _cfa(b.body, state, on_gen)
on_gen = _cfa(b.finalbody, state, on_gen)
elif isinstance(b, _ast.While):
# This could also be architected as having no extra block and having two jump statements, but I don't like that
if cur_block:
j = Jump()
cur_block.append(j)
on_gen = [make_connect(push_block(cur_block)), j.set_dest]
cur_block = []
always_true = False
always_false = False
if isinstance(b.test, _ast.Name):
if b.test.id == "True":
always_true = True
elif b.test.id == "False":
always_false = True
elif isinstance(b.test, _ast.Num):
if b.test.n:
always_true = True
else:
always_false = True
if always_true:
br = Jump()
on_true = br.set_dest
elif always_false:
br = Jump()
on_false = br.set_dest
else:
br = Branch(b.test)
on_true = br.set_true
on_false = br.set_false
init_id = push_block([br])
on_gen = None
assert cur_block == [] # just checking
if not always_false:
gen_true = [on_true, make_connect(init_id)]
if not always_true:
gen_false = [on_false, make_connect(init_id)]
if ENFORCE_NO_MULTIMULTI:
if not always_false:
on_gen = gen_true
j1 = Jump()
gen_true = [make_connect(push_block([j1])), j1.set_dest]
if not always_true:
on_gen = gen_false
j2 = Jump()
gen_false = [make_connect(push_block([j2])), j2.set_dest]
on_gen = None
ending_gen = []
if not always_false:
state.push_loop()
assert b.body
loop_ending_gen = _cfa(b.body, state, gen_true)
loop_ending_gen += state.get_continues()
for c in loop_ending_gen:
c(init_id)
ending_gen = state.get_breaks()
state.pop_loop()
if not always_true:
if b.orelse:
ending_gen += _cfa(b.orelse, state, gen_false)
else:
ending_gen += gen_false
on_gen = ending_gen
if not on_gen and PRUNE_UNREACHABLE_BLOCKS:
return []
elif isinstance(b, _ast.For):
iter_func = _ast.Attribute(b.iter,
"__iter__",
_ast.Load(),
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
iter_call = _ast.Call(iter_func, [], [],
None,
None,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
# iter_var = _make_temp_name()
iter_var = "__foriter_%d_%d_" % (b.lineno, b.col_offset)
iter_assign = _ast.Assign(
[_ast.Name(iter_var, _ast.Store(), not_real=True, **pos(b))],
iter_call,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
cur_block.append(iter_assign)
j = Jump()
cur_block.append(j)
on_gen = [make_connect(push_block(cur_block)), j.set_dest]
cur_block = []
br = Branch(
HasNext(
_ast.Name(iter_var, _ast.Load(), not_real=True, **pos(b)),
**pos(b)), **pos(b))
init_id = push_block([br])
on_gen = None
assert cur_block == [] # just checking
gen_true = [br.set_true, make_connect(init_id)]
gen_false = [br.set_false, make_connect(init_id)]
if ENFORCE_NO_MULTIMULTI:
on_gen = gen_true
j1 = Jump()
gen_true = [make_connect(push_block([j1])), j1.set_dest]
on_gen = gen_false
j2 = Jump()
gen_false = [make_connect(push_block([j2])), j2.set_dest]
on_gen = None
ending_gen = []
state.push_loop()
next_func = _ast.Attribute(_ast.Name(iter_var,
_ast.Load(),
not_real=True,
**pos(b)),
"next",
_ast.Load(),
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
next = _ast.Call(next_func, [], [],
None,
None,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
next_assign = _ast.Assign([b.target],
next,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
next_iter_gen = _cfa([next_assign] + b.body, state, gen_true)
next_iter_gen += state.get_continues()
for c in next_iter_gen:
c(init_id)
loop_done_gen = list(state.get_breaks())
state.pop_loop()
if b.orelse:
# if b.orelse and loop_ending_blocks:
loop_done_gen += _cfa(b.orelse, state, gen_false)
else:
loop_done_gen += gen_false
on_gen = loop_done_gen
if not on_gen and PRUNE_UNREACHABLE_BLOCKS:
return []
elif isinstance(b, (_ast.Return, _ast.Raise)):
cur_block.append(b)
block_id = push_block(cur_block)
state.returns.append(make_connect(block_id))
return []
elif isinstance(b, _ast.With):
# XXX totally ignores the functionality of with statements
# Have to save the context manager because the expression might not be valid later
mgr_name = "__mgr_%s_%s_" % (b.lineno, b.col_offset)
save_mgr = _ast.Assign([
_ast.Name(mgr_name,
_ast.Store(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
],
b.context_expr,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
enter_func = _ast.Attribute(_ast.Name(mgr_name,
_ast.Load(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True),
"__enter__",
_ast.Load(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
bind = _ast.Call(enter_func, [], [],
None,
None,
lineno=b.lineno,
col_offset="__enter__()",
not_real=True)
if b.optional_vars:
assert isinstance(b.optional_vars, _ast.AST)
init = _ast.Assign([b.optional_vars],
bind,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
else:
init = _ast.Expr(bind,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
exit_func = _ast.Attribute(_ast.Name(mgr_name,
_ast.Load(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True),
"__exit__",
_ast.Load(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
if SIMPLE_WITH_EXIT:
exit_call = _ast.Call(exit_func, [], [],
None,
None,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
else:
none_ = _ast.Name("None",
_ast.Load(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
exit_call = _ast.Call(exit_func, [none_, none_, none_], [],
None,
None,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
exit = _ast.Expr(exit_call,
lineno=b.lineno,
col_offset="__exit__()",
not_real=True)
cur_block.extend([save_mgr, init])
j = Jump()
cur_block.append(j)
next_block = push_block(cur_block)
on_gen = [j.set_dest, make_connect(next_block)]
cur_block = []
body = b.body + [exit]
next_gen = _cfa(body, state, on_gen)
on_gen = next_gen
else:
raise Exception(b)
if cur_block:
j = Jump()
cur_block.append(j)
next = push_block(cur_block)
return [j.set_dest, make_connect(next)]
return on_gen
return on_gen
def ast_wrap_in_assn(var_name, ast_expr):
assn = _ast.Assign(targets=[_ast.Name(id=var_name, ctx=_ast.Store())],
value=ast_expr.value)
ast.fix_missing_locations(assn)
return assn
def visit_GeneratorExp(self, node):
"""Rewrite the GeneratorExp visitor function to turn the
generator expression into a iterator function. This is
necessary to be able to correctly label any random functions
that get called from within the generator expression.
Basically, this function creates a function, and transforms
the generator into a for loop that yields values from the.
The function name is then returned, so that the parent node
can handle the assignment properly.
"""
# make an identifier for the list
self.newline(node)
iden = self._gen_iden(node)
argids = []
for gen in node.generators:
argval = gen.iter
argid = self._gen_iden(gen.iter)
self.visit(
_ast.Assign(targets=[_ast.Name(id=argid, ctx=_ast.Store())],
value=argval))
argids.append(argid)
elt = node.elt
def parse_generator(nodes, ids):
node = nodes[0]
tempnode = _ast.For()
tempnode.target = node.target
tempnode.iter = _ast.Name(id=ids[0], ctx=_ast.Load())
if len(nodes) == 1:
yield_node = _ast.Expr(value=_ast.Yield(value=elt))
body = [yield_node]
else:
body = [parse_generator(nodes[1:], ids[1:])]
if len(node.ifs) == 1:
ifnode = _ast.If(test=node.ifs[0], body=body, orelse=[])
tempnode.body = [ifnode]
elif len(node.ifs) > 1:
ifnode = _ast.If(test=_ast.BoolOp(op=_ast.And(),
values=node.ifs),
body=body,
orelse=[])
tempnode.body = [ifnode]
else:
tempnode.body = body
tempnode.orelse = None
return tempnode
node = _ast.FunctionDef(
name=iden,
args=_ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]),
body=[parse_generator(node.generators, argids)],
decorator_list=[])
self.visit(node)
return iden
def STORE_NAME(self, instr):
value = self.pop_ast_item()
value = self.process_ifexpr(value)
if isinstance(value, _ast.Import):
if value.from_:
assert isinstance(self._ast_stack[-1], _ast.ImportFrom)
from_ = self.pop_ast_item()
as_name = instr.arg
name = from_.names[0].name
if as_name != name:
from_.names[0].asname = as_name
self.push_ast_item(from_)
else:
as_name = instr.arg
if value.names[0].asname is None:
base_name = value.names[0].name.split('.')[0]
if base_name != as_name:
value.names[0].asname = as_name
self.push_ast_item(value)
elif isinstance(value, (_ast.Attribute)) and isinstance(
value.value, (_ast.Import)):
asname = instr.arg
value = value.value
value.names[0].asname = asname
self.push_ast_item(value)
elif isinstance(value, (_ast.ClassDef, _ast.FunctionDef)):
as_name = instr.arg
value.name = as_name
self.push_ast_item(value)
elif isinstance(value, _ast.AugAssign):
self.push_ast_item(value)
elif isinstance(value, _ast.Assign):
_ = self.pop_ast_item()
assname = _ast.Name(instr.arg,
_ast.Store(),
lineno=instr.lineno,
col_offset=0)
if _ is value.value or isinstance(_, _ast.Assign):
value.targets.append(assname)
else:
if not isinstance(value.targets, _ast.Tuple):
value.targets = [_ast.Tuple(value.targets, _ast.Store())]
value.value = _ast.Tuple([value.value], _ast.Load())
value.targets[0].lineno = value.targets[0].elts[0].lineno
value.targets[0].col_offset = value.targets[0].elts[
0].col_offset
value.value.lineno = value.value.elts[0].lineno
value.value.col_offset = value.value.elts[0].col_offset
value.targets[0].elts.append(assname)
value.value.elts.append(_)
self.push_ast_item(value)
else:
assname = _ast.Name(instr.arg,
_ast.Store(),
lineno=instr.lineno,
col_offset=0)
assign = _ast.Assign(targets=[assname],
value=value,
lineno=instr.lineno,
col_offset=0)
self.push_ast_item(assign)
def split_handlers(self, handlers_blocks):
handlers = []
except_instrs = []
ends = []
while len(handlers_blocks):
instr = handlers_blocks.pop(0)
except_instrs.append(instr)
if (instr.opname == 'COMPARE_OP') and (instr.arg == 'exception match'):
jump = handlers_blocks.pop(0)
assert jump.opname == 'POP_JUMP_IF_FALSE'
next_handler = jump.oparg
instr = handlers_blocks.pop(0)
except_instrs.append(instr)
instr = handlers_blocks.pop(0)
except_instrs.append(instr)
instr = handlers_blocks.pop(0)
except_instrs.append(instr)
assert except_instrs[0].opname == 'DUP_TOP'
assert except_instrs[-3].opname == 'POP_TOP'
assert except_instrs[-1].opname == 'POP_TOP'
exec_stmnt = self.decompile_block(except_instrs[1:-4]).stmnt()
assert len(exec_stmnt) == 1
exc_type = exec_stmnt[0]
if except_instrs[-2].opname == 'STORE_NAME':
exc_name = _ast.Name(id=except_instrs[-2].arg, ctx=_ast.Store(), lineno=except_instrs[-2].lineno, col_offset=0)
else:
assert except_instrs[-2].opname == 'POP_TOP'
exc_name = None
handler_body = []
while len(handlers_blocks):
instr = handlers_blocks.pop(0)
if instr.i == next_handler:
handlers_blocks.insert(0, instr)
break
handler_body.append(instr)
assert handler_body[-1].opname == 'JUMP_FORWARD'
ends.append(handler_body[-1].arg)
exc_body = self.decompile_block(handler_body[:-1]).stmnt()
if not exc_body:
exc_body.append(_ast.Pass(lineno=except_instrs[-2].lineno, col_offset=0))
# is this for python 3?
if py3 and exc_name is not None:
exc_name = exc_name.id
handlers.append(_ast.ExceptHandler(type=exc_type, name=exc_name, body=exc_body, lineno=instr.lineno, col_offset=0))
except_instrs = []
assert except_instrs[-1].opname == 'END_FINALLY'
if len(except_instrs) == 1:
pass
else:
assert except_instrs[0].opname == 'POP_TOP'
assert except_instrs[1].opname == 'POP_TOP'
assert except_instrs[2].opname == 'POP_TOP'
assert except_instrs[-2].opname in ['JUMP_FORWARD', 'JUMP_ABSOLUTE'], except_instrs[-2]
ends.append(except_instrs[-2].arg)
exc_body = self.decompile_block(except_instrs[3:-2]).stmnt()
if not exc_body:
exc_body.append(_ast.Pass(lineno=except_instrs[-2].lineno, col_offset=0))
handlers.append(_ast.ExceptHandler(type=None, name=None, body=exc_body, lineno=except_instrs[0].lineno, col_offset=0))
assert all(e == ends[0] for e in ends)
end = ends[0]
return end, handlers
