def to_python_statment(bytecode):
i = len(bytecode) - 1
while i >= 0:
index, lineno, op, arg, cflow_in, code_object = bytecode[i]
prev_op = bytecode[i - 1][2] if i > 1 else -1
if op in (PRINT_ITEM, PRINT_NEWLINE, PRINT_ITEM_TO, PRINT_NEWLINE_TO):
_, stmt = Statement.to_print_statement(i, op, bytecode)
return stmt
elif op in CALL_OPCODES:
_, call = Statement.make_call(i, bytecode)
return _ast.Expr(call)
elif op == COMPARE_OP:
_, cmp_expr = Statement.make_comparator(i, bytecode)
return _ast.Expr(cmp_expr)
elif op in STORE_OPCODES and prev_op not in (MAKE_FUNCTION, BUILD_CLASS):
_, store_stmt = Statement.make_assign(i, bytecode)
return store_stmt
elif op in STORE_SLICE_OPCODES or op in DELETE_SLICE_OPCODES:
_, store_stmt = Statement.make_store_delete_slice(i, bytecode)
return store_stmt
elif op in DELETE_OPCODES:
_, store_stmt = Statement.make_delete(i, bytecode)
return store_stmt
else:
logger.debug("Unhandled >> STMT:\n%s", show_bytecode(bytecode))
i -= 1
return None
class ReflectionAtLineFixture:
empty_node = _ast.Pass
is_instance_at_first_lvl = _ast.FunctionDef(id='isinstance', lineno=1)
type_at_first_lvl = _ast.FunctionDef(id='type', lineno=1)
is_instance_at_second_lvl = _ast.FunctionDef(
body=[_ast.Expr(id='isinstance', lineno=2)], lineno=1)
type_at_second_lvl = _ast.FunctionDef(
body=[_ast.Expr(id='type', lineno=2)], lineno=1)
class ConstructorNonAttribsValueLineNumberFixture:
definition_is_none = None
definition__is_not_a_function = _ast.Pass
definitions_is_not_a_constructor = _ast.FunctionDef(name='test', body=[_ast.Expr(lineno=1)])
definitions_consist_of_assign_with_attribute = _ast.FunctionDef(
name='__init__', body=[_ast.Assign(targets=[_ast.Attribute(lineno=2)], lineno=1)],
)
definitions_consist_of_assign_without_attribute = _ast.FunctionDef(
name='__init__', body=[_ast.Assign(targets=[_ast.Expr(lineno=2)], lineno=1)],
)
definitions_consist_of_any_but_not_a_assign = _ast.FunctionDef(name='__init__', body=[_ast.Pass(lineno=3)])
def MAKE_FUNCTION(self, instr):
code = self.ast_stack.pop()
ndefaults = bitrange(instr.oparg, 0, 8)
nkwonly_defaults = bitrange(instr.oparg, 8, 16)
nannotations = bitrange(instr.oparg, 16, 32) - 1
annotations = []
for i in range(nannotations):
annotations.insert(0, self.ast_stack.pop())
kw_defaults = []
for i in range(nkwonly_defaults * 2):
kw_defaults.insert(0, self.ast_stack.pop())
defaults = []
for i in range(ndefaults):
defaults.insert(0, self.ast_stack.pop())
function = make_function(code, defaults, lineno=instr.lineno, annotations=annotations, kw_defaults=kw_defaults)
doc = code.co_consts[0] if code.co_consts else None
if isinstance(doc, str):
function.body.insert(0, _ast.Expr(value=_ast.Str(s=doc, lineno=instr.lineno, col_offset=0),
lineno=instr.lineno, col_offset=0))
self.ast_stack.append(function)
def parse_generator(nodes, ids):
node = nodes[0]
tempnode = _ast.For()
tempnode.target = node.target
tempnode.iter = _ast.Name(id=ids[0], ctx=_ast.Load())
if len(nodes) == 1:
yield_node = _ast.Expr(value=_ast.Yield(value=elt))
body = [yield_node]
else:
body = [parse_generator(nodes[1:], ids[1:])]
if len(node.ifs) == 1:
ifnode = _ast.If(test=node.ifs[0], body=body, orelse=[])
tempnode.body = [ifnode]
elif len(node.ifs) > 1:
ifnode = _ast.If(test=_ast.BoolOp(op=_ast.And(),
values=node.ifs),
body=body,
orelse=[])
tempnode.body = [ifnode]
else:
tempnode.body = body
tempnode.orelse = None
return tempnode
class ReturnedValueFixture:
node_is_none = None
assign_expression_as_input = _ast.Assign(lineno=1)
pass_expression_as_input = _ast.Pass(lineno=1)
plain_expression_as_input = _ast.Expr(lineno=1)
function_body_is_empty = _ast.FunctionDef(lineno=1, body=[])
function_body_without_return_expression = _ast.FunctionDef(
lineno=1,
body=[_ast.Pass(), _ast.Expr(), _ast.Assign],
)
function_body_with_return_expression = _ast.FunctionDef(
lineno=1,
body=[
_ast.Pass(),
_ast.Expr(),
_ast.Assign(),
_ast.Return(lineno=1, value=None)
],
)
class StaticOrPrivateFixture:
definition_is_assign = _ast.Assign()
definition_is_pass = _ast.Pass()
definition_is_expr = _ast.Expr()
empty_decorator_list_and_name = _ast.FunctionDef()
empty_decorator_list = _ast.FunctionDef(name='test')
is_private = _ast.FunctionDef(name='_test')
is_protected = _ast.FunctionDef(name='__test')
is_magic = _ast.FunctionDef(name='__test__')
decorated = _ast.FunctionDef(name='test', decorator_list=['any_decorator'])
is_static = _ast.FunctionDef(
name='test',
decorator_list=[_ast.FunctionDef(id='staticmethod')],
)
def test_empty_initialization(self):
ast = self.ast
def com(node):
return compile(node, "<test>", "exec")
mod = ast.Module()
raises(AttributeError, getattr, mod, "body")
exc = raises(TypeError, com, mod).value
assert str(exc) == "required field \"body\" missing from Module"
expr = ast.Name()
expr.id = "hi"
expr.ctx = ast.Load()
expr.lineno = 4
exc = raises(TypeError, com, ast.Module([ast.Expr(expr)])).value
assert (str(exc) == "required field \"lineno\" missing from stmt" or # cpython
str(exc) == "required field \"lineno\" missing from Expr") # pypy, better
def POP_TOP(self, instr):
node = self.ast_stack.pop()
node = self.process_ifexpr(node)
if isinstance(node, _ast.Import):
return
if isinstance(node, _ast_Print):
_ = self.ast_stack.pop()
self.ast_stack.append(node)
return
discard = _ast.Expr(value=node, lineno=instr.lineno, col_offset=0)
self.ast_stack.append(discard)
def make_module(code):
from meta.decompiler.disassemble import disassemble
instructions = Instructions(disassemble(code))
stmnts = instructions.stmnt()
doc = pop_doc(stmnts)
pop_return(stmnts)
# stmnt = ast.Stmt(stmnts, 0)
if doc is not None:
stmnts = [_ast.Expr(value=doc, lineno=doc.lineno, col_offset=0)] + stmnts
ast_obj = _ast.Module(body=stmnts, lineno=0, col_offset=0)
return ast_obj
def test_functiondef(self):
import ast
fAst = ast.FunctionDef(
name="foo",
args=ast.arguments(
args=[], vararg=None, kwarg=None, defaults=[],
kwonlyargs=[], kw_defaults=[]),
body=[ast.Expr(ast.Str('docstring'))],
decorator_list=[], lineno=5, col_offset=0)
exprAst = ast.Interactive(body=[fAst])
ast.fix_missing_locations(exprAst)
compiled = compile(exprAst, "<foo>", "single")
#
d = {}
eval(compiled, d, d)
assert type(d['foo']) is type(lambda: 42)
assert d['foo']() is None
def Expr(value):
"""Creates an _ast.Expr node.
Note that this node is mostly used to wrap other nodes so they're treated
as whole-line statements.
Args:
value: The value stored in the node.
Raises:
ValueError: If value is an _ast.stmt node.
Returns:
An _ast.Expr node.
"""
if isinstance(value, _ast.stmt):
raise ValueError(
'value must not be an _ast.stmt node, because those nodes don\'t need '
'to be wrapped in an Expr node. Value passed: {}'.format(value))
return _ast.Expr(value)
def MAKE_FUNCTION(self, instr):
code = self.ast_stack.pop()
ndefaults = instr.oparg
defaults = []
for i in range(ndefaults):
defaults.insert(0, self.ast_stack.pop())
function = make_function(code, defaults, lineno=instr.lineno)
doc = code.co_consts[0] if code.co_consts else None
if isinstance(doc, str):
function.body.insert(0, _ast.Expr(value=_ast.Str(s=doc, lineno=instr.lineno, col_offset=0),
lineno=instr.lineno, col_offset=0))
self.ast_stack.append(function)
def test_invalid_identitifer(self):
import ast
m = ast.Module([ast.Expr(ast.Name(b"x", ast.Load()))])
ast.fix_missing_locations(m)
exc = raises(TypeError, compile, m, "<test>", "exec")
def test_invalid_string(self):
import ast
m = ast.Module([ast.Expr(ast.Str(43))])
ast.fix_missing_locations(m)
exc = raises(TypeError, compile, m, "<test>", "exec")
def test_empty_set(self):
import ast
m = ast.Module(body=[ast.Expr(value=ast.Set(elts=[]))])
ast.fix_missing_locations(m)
compile(m, "<test>", "exec")
def test_invalid_sum(self):
import _ast as ast
pos = dict(lineno=2, col_offset=3)
m = ast.Module([ast.Expr(ast.expr(**pos), **pos)])
exc = raises(TypeError, compile, m, "<test>", "exec")
def function_call_expr(self, items):
(items, _) = items
return _ast.Expr(value=items)
def expr_stmt(value: _ast.expr) -> _ast.Expr:
return _ast.Expr(value=value)
def _expr(node, msg=None, exc=ValueError):
mod = ast.Module([ast.Expr(node)])
_mod(mod, msg, exc=exc)
def test_ast_types(self):
ast = self.ast
expr = ast.Expr()
expr.value = ast.Lt()
def _cfa(body, state, on_gen):
assert isinstance(on_gen, list)
for c in on_gen:
assert callable(c)
cfg = state.cfg
def make_connect(cur_bid):
assert isinstance(cur_bid, int)
def inner(new_bid):
assert isinstance(new_bid, int)
state.connect([cur_bid], [new_bid])
return inner
def push_block(block):
block_id = len(cfg.blocks)
assert block_id not in cfg.blocks
cfg.blocks[block_id] = block
for c in on_gen:
c(block_id)
return block_id
cur_block = []
for b in body:
if REDUCE_FORS_TO_WHILES and isinstance(b, _ast.For):
if isinstance(b.iter, _ast.Call) and isinstance(
b.iter.func,
_ast.Name) and b.iter.func.id in ("range", "xrange"):
if not b.iter.keywords and not b.iter.starargs and not b.iter.kwargs:
end_var = "__wfend_%d_%d_" % (b.lineno, b.col_offset)
iter_var = "__wfiter_%d_%d_" % (b.lineno, b.col_offset)
if len(b.iter.args) in (1, 2):
if len(b.iter.args) == 1:
start = _ast.Num(0)
end = b.iter.args[0]
elif len(b.iter.args) == 2:
start = b.iter.args[0]
end = b.iter.args[1]
else:
start = b.iter.args[0]
end = b.iter.args[1]
cur_block.append(
_ast.Assign([
_ast.Name(
iter_var, _ast.Store(), not_real=True)
],
start,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True))
cur_block.append(
_ast.Assign([
_ast.Name(end_var, _ast.Store(), not_real=True)
],
end,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True))
body = [
_ast.Assign([b.target],
_ast.Name(iter_var,
_ast.Load(),
not_real=True),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True),
_ast.Assign([
_ast.Name(
iter_var, _ast.Store(), not_real=True)
],
_ast.BinOp(
_ast.Name(iter_var,
_ast.Load(),
not_real=True),
_ast.Add(), _ast.Num(1)),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
] + b.body
b = _ast.While(_ast.Compare(
_ast.Name(iter_var, _ast.Load(),
not_real=True), [_ast.Lt()],
[_ast.Name(end_var, _ast.Load(), not_real=True)],
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True),
body,
b.orelse,
not_real=True)
if isinstance(b, (
_ast.Assign,
_ast.AugAssign,
_ast.ClassDef,
_ast.Delete,
_ast.Exec,
_ast.Expr,
_ast.FunctionDef,
_ast.Global,
_ast.Import,
_ast.ImportFrom,
_ast.Print,
_ast.Pass,
)):
cur_block.append(b)
elif isinstance(b, _ast.Assert):
cur_block.append(b)
if isinstance(b.test, _ast.Call) and isinstance(
b.test.func, _ast.Name
) and b.test.func.id == "isinstance" and isinstance(
b.test.args[0], _ast.Name) and isinstance(
b.test.args[1], _ast.Name):
varname = b.test.args[0].id
cast = _ast.Call(_ast.Name(
"__cast__", _ast.Load(), not_real=True, **pos(b)), [
_ast.Name(
varname, _ast.Store(), not_real=True, **pos(b)),
b.test.args[1]
], [],
None,
None,
not_real=True,
**pos(b))
assign = _ast.Assign([
_ast.Name(varname, _ast.Store(), not_real=True, **pos(b))
],
cast,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
cur_block.append(assign)
elif isinstance(b, (_ast.Break, _ast.Continue)):
f = state.add_break if isinstance(
b, _ast.Break) else state.add_continue
if cur_block:
j = Jump()
cur_block.append(j)
block_id = push_block(cur_block)
f(j.set_dest)
f(make_connect(block_id))
else:
for c in on_gen:
f(c)
return []
elif isinstance(b, _ast.If):
br = Branch(b.test, lineno=b.lineno)
cur_block.append(br)
next_block = push_block(cur_block)
on_gen = None # make sure this doesn't get used
cur_block = []
gen_true = [br.set_true, make_connect(next_block)]
gen_false = [br.set_false, make_connect(next_block)]
if ENFORCE_NO_MULTIMULTI:
on_gen = gen_true
j1 = Jump()
gen_true = [make_connect(push_block([j1])), j1.set_dest]
on_gen = gen_false
j2 = Jump()
gen_false = [make_connect(push_block([j2])), j2.set_dest]
on_gen = None
assert b.body
body = b.body
if isinstance(b.test, _ast.Call) and isinstance(
b.test.func, _ast.Name
) and b.test.func.id == "isinstance" and isinstance(
b.test.args[0], _ast.Name) and isinstance(
b.test.args[1], _ast.Name):
varname = b.test.args[0].id
cast = _ast.Call(_ast.Name(
"__cast__", _ast.Load(), not_real=True, **pos(b)), [
_ast.Name(
varname, _ast.Store(), not_real=True, **pos(b)),
b.test.args[1]
], [],
None,
None,
not_real=True,
**pos(b))
assign = _ast.Assign([
_ast.Name(varname, _ast.Store(), not_real=True, **pos(b))
],
cast,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
body = [assign] + body
if ADD_IF_ASSERTS:
body = [_ast.Assert(b.test, None, not_real=True, **pos(b))
] + body
ending_gen = _cfa(body, state, gen_true)
if b.orelse:
ending_gen += _cfa(b.orelse, state, gen_false)
else:
ending_gen += gen_false
on_gen = ending_gen
if not on_gen and PRUNE_UNREACHABLE_BLOCKS:
return []
elif isinstance(b, _ast.TryExcept):
j = Jump()
cur_block.append(j)
next_block = push_block(cur_block)
on_gen = [j.set_dest, make_connect(next_block)]
cur_block = []
on_gen = _cfa(b.body, state, on_gen)
# Set this to evaluate a string to try to defeat simple flow analysis
br = Branch(_ast.Str("nonzero"))
next_block = push_block([br])
on_except = [br.set_false, make_connect(next_block)]
on_fine = [br.set_true, make_connect(next_block)]
assert len(b.handlers) >= 1
# for handler in b.handlers:
# on_except = _cfa(b.handlers[0].body, state, on_except)
if b.orelse:
on_fine = _cfa(b.orelse, state, on_fine)
if ENFORCE_NO_MULTIMULTI:
j = Jump()
on_gen = on_fine
next_block = push_block([j])
on_fine = [j.set_dest, make_connect(next_block)]
j = Jump()
on_gen = on_except
next_block = push_block([j])
on_except = [j.set_dest, make_connect(next_block)]
on_gen = on_fine + on_except
cur_block = []
elif isinstance(b, _ast.TryFinally):
j = Jump()
cur_block.append(j)
next_block = push_block(cur_block)
on_gen = [j.set_dest, make_connect(next_block)]
cur_block = []
on_gen = _cfa(b.body, state, on_gen)
on_gen = _cfa(b.finalbody, state, on_gen)
elif isinstance(b, _ast.While):
# This could also be architected as having no extra block and having two jump statements, but I don't like that
if cur_block:
j = Jump()
cur_block.append(j)
on_gen = [make_connect(push_block(cur_block)), j.set_dest]
cur_block = []
always_true = False
always_false = False
if isinstance(b.test, _ast.Name):
if b.test.id == "True":
always_true = True
elif b.test.id == "False":
always_false = True
elif isinstance(b.test, _ast.Num):
if b.test.n:
always_true = True
else:
always_false = True
if always_true:
br = Jump()
on_true = br.set_dest
elif always_false:
br = Jump()
on_false = br.set_dest
else:
br = Branch(b.test)
on_true = br.set_true
on_false = br.set_false
init_id = push_block([br])
on_gen = None
assert cur_block == [] # just checking
if not always_false:
gen_true = [on_true, make_connect(init_id)]
if not always_true:
gen_false = [on_false, make_connect(init_id)]
if ENFORCE_NO_MULTIMULTI:
if not always_false:
on_gen = gen_true
j1 = Jump()
gen_true = [make_connect(push_block([j1])), j1.set_dest]
if not always_true:
on_gen = gen_false
j2 = Jump()
gen_false = [make_connect(push_block([j2])), j2.set_dest]
on_gen = None
ending_gen = []
if not always_false:
state.push_loop()
assert b.body
loop_ending_gen = _cfa(b.body, state, gen_true)
loop_ending_gen += state.get_continues()
for c in loop_ending_gen:
c(init_id)
ending_gen = state.get_breaks()
state.pop_loop()
if not always_true:
if b.orelse:
ending_gen += _cfa(b.orelse, state, gen_false)
else:
ending_gen += gen_false
on_gen = ending_gen
if not on_gen and PRUNE_UNREACHABLE_BLOCKS:
return []
elif isinstance(b, _ast.For):
iter_func = _ast.Attribute(b.iter,
"__iter__",
_ast.Load(),
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
iter_call = _ast.Call(iter_func, [], [],
None,
None,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
# iter_var = _make_temp_name()
iter_var = "__foriter_%d_%d_" % (b.lineno, b.col_offset)
iter_assign = _ast.Assign(
[_ast.Name(iter_var, _ast.Store(), not_real=True, **pos(b))],
iter_call,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
cur_block.append(iter_assign)
j = Jump()
cur_block.append(j)
on_gen = [make_connect(push_block(cur_block)), j.set_dest]
cur_block = []
br = Branch(
HasNext(
_ast.Name(iter_var, _ast.Load(), not_real=True, **pos(b)),
**pos(b)), **pos(b))
init_id = push_block([br])
on_gen = None
assert cur_block == [] # just checking
gen_true = [br.set_true, make_connect(init_id)]
gen_false = [br.set_false, make_connect(init_id)]
if ENFORCE_NO_MULTIMULTI:
on_gen = gen_true
j1 = Jump()
gen_true = [make_connect(push_block([j1])), j1.set_dest]
on_gen = gen_false
j2 = Jump()
gen_false = [make_connect(push_block([j2])), j2.set_dest]
on_gen = None
ending_gen = []
state.push_loop()
next_func = _ast.Attribute(_ast.Name(iter_var,
_ast.Load(),
not_real=True,
**pos(b)),
"next",
_ast.Load(),
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
next = _ast.Call(next_func, [], [],
None,
None,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
next_assign = _ast.Assign([b.target],
next,
not_real=True,
lineno=b.lineno,
col_offset=b.col_offset)
next_iter_gen = _cfa([next_assign] + b.body, state, gen_true)
next_iter_gen += state.get_continues()
for c in next_iter_gen:
c(init_id)
loop_done_gen = list(state.get_breaks())
state.pop_loop()
if b.orelse:
# if b.orelse and loop_ending_blocks:
loop_done_gen += _cfa(b.orelse, state, gen_false)
else:
loop_done_gen += gen_false
on_gen = loop_done_gen
if not on_gen and PRUNE_UNREACHABLE_BLOCKS:
return []
elif isinstance(b, (_ast.Return, _ast.Raise)):
cur_block.append(b)
block_id = push_block(cur_block)
state.returns.append(make_connect(block_id))
return []
elif isinstance(b, _ast.With):
# XXX totally ignores the functionality of with statements
# Have to save the context manager because the expression might not be valid later
mgr_name = "__mgr_%s_%s_" % (b.lineno, b.col_offset)
save_mgr = _ast.Assign([
_ast.Name(mgr_name,
_ast.Store(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
],
b.context_expr,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
enter_func = _ast.Attribute(_ast.Name(mgr_name,
_ast.Load(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True),
"__enter__",
_ast.Load(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
bind = _ast.Call(enter_func, [], [],
None,
None,
lineno=b.lineno,
col_offset="__enter__()",
not_real=True)
if b.optional_vars:
assert isinstance(b.optional_vars, _ast.AST)
init = _ast.Assign([b.optional_vars],
bind,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
else:
init = _ast.Expr(bind,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
exit_func = _ast.Attribute(_ast.Name(mgr_name,
_ast.Load(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True),
"__exit__",
_ast.Load(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
if SIMPLE_WITH_EXIT:
exit_call = _ast.Call(exit_func, [], [],
None,
None,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
else:
none_ = _ast.Name("None",
_ast.Load(),
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
exit_call = _ast.Call(exit_func, [none_, none_, none_], [],
None,
None,
lineno=b.lineno,
col_offset=b.col_offset,
not_real=True)
exit = _ast.Expr(exit_call,
lineno=b.lineno,
col_offset="__exit__()",
not_real=True)
cur_block.extend([save_mgr, init])
j = Jump()
cur_block.append(j)
next_block = push_block(cur_block)
on_gen = [j.set_dest, make_connect(next_block)]
cur_block = []
body = b.body + [exit]
next_gen = _cfa(body, state, on_gen)
on_gen = next_gen
else:
raise Exception(b)
if cur_block:
j = Jump()
cur_block.append(j)
next = push_block(cur_block)
return [j.set_dest, make_connect(next)]
return on_gen
return on_gen
