def DUP2():
return Instr(instr_names.DUP_TOP_TWO)
def STORE_FAST(name: str, *, lineno=None):
return Instr('STORE_FAST', name, lineno=lineno)
def MAP_ADD(n: int, *, lineno=None):
return Instr("MAP_ADD", n, lineno=lineno)
def test_invalid_stacksize(self):
code = Bytecode()
code.extend([Instr("STORE_NAME", "x")])
with self.assertRaises(RuntimeError):
code.compute_stacksize()
def test_copy(self):
block = BasicBlock([Instr("NOP")])
next_block = BasicBlock()
block.next_block = next_block
self.assertEqual(block, block.copy())
self.assertIs(next_block, block.copy().next_block)
def test_to_bytecode(self):
# if test:
# x = 2
# x = 5
blocks = ControlFlowGraph()
blocks.add_block()
blocks.add_block()
blocks[0].extend([
Instr("LOAD_NAME", "test", lineno=1),
Instr("POP_JUMP_IF_FALSE", blocks[2], lineno=1),
])
blocks[1].extend([
Instr("LOAD_CONST", 5, lineno=2),
Instr("STORE_NAME", "x", lineno=2),
Instr("JUMP_FORWARD", blocks[2], lineno=2),
])
blocks[2].extend([
Instr("LOAD_CONST", 7, lineno=3),
Instr("STORE_NAME", "x", lineno=3),
Instr("LOAD_CONST", None, lineno=3),
Instr("RETURN_VALUE", lineno=3),
])
bytecode = blocks.to_bytecode()
label = Label()
self.assertEqual(
bytecode,
[
Instr("LOAD_NAME", "test", lineno=1),
Instr("POP_JUMP_IF_FALSE", label, lineno=1),
Instr("LOAD_CONST", 5, lineno=2),
Instr("STORE_NAME", "x", lineno=2),
Instr("JUMP_FORWARD", label, lineno=2),
label,
Instr("LOAD_CONST", 7, lineno=3),
Instr("STORE_NAME", "x", lineno=3),
Instr("LOAD_CONST", None, lineno=3),
Instr("RETURN_VALUE", lineno=3),
],
)
def test_from_bytecode_loop(self):
# for x in (1, 2, 3):
# if x == 2:
# break
# continue
if sys.version_info < (3, 8):
label_loop_start = Label()
label_loop_exit = Label()
label_loop_end = Label()
code = Bytecode()
code.extend((
Instr("SETUP_LOOP", label_loop_end, lineno=1),
Instr("LOAD_CONST", (1, 2, 3), lineno=1),
Instr("GET_ITER", lineno=1),
label_loop_start,
Instr("FOR_ITER", label_loop_exit, lineno=1),
Instr("STORE_NAME", "x", lineno=1),
Instr("LOAD_NAME", "x", lineno=2),
Instr("LOAD_CONST", 2, lineno=2),
Instr("COMPARE_OP", Compare.EQ, lineno=2),
Instr("POP_JUMP_IF_FALSE", label_loop_start, lineno=2),
Instr("BREAK_LOOP", lineno=3),
Instr("JUMP_ABSOLUTE", label_loop_start, lineno=4),
Instr("JUMP_ABSOLUTE", label_loop_start, lineno=4),
label_loop_exit,
Instr("POP_BLOCK", lineno=4),
label_loop_end,
Instr("LOAD_CONST", None, lineno=4),
Instr("RETURN_VALUE", lineno=4),
))
blocks = ControlFlowGraph.from_bytecode(code)
expected = [
[Instr("SETUP_LOOP", blocks[8], lineno=1)],
[
Instr("LOAD_CONST", (1, 2, 3), lineno=1),
Instr("GET_ITER", lineno=1)
],
[Instr("FOR_ITER", blocks[7], lineno=1)],
[
Instr("STORE_NAME", "x", lineno=1),
Instr("LOAD_NAME", "x", lineno=2),
Instr("LOAD_CONST", 2, lineno=2),
Instr("COMPARE_OP", Compare.EQ, lineno=2),
Instr("POP_JUMP_IF_FALSE", blocks[2], lineno=2),
],
[Instr("BREAK_LOOP", lineno=3)],
[Instr("JUMP_ABSOLUTE", blocks[2], lineno=4)],
[Instr("JUMP_ABSOLUTE", blocks[2], lineno=4)],
[Instr("POP_BLOCK", lineno=4)],
[
Instr("LOAD_CONST", None, lineno=4),
Instr("RETURN_VALUE", lineno=4)
],
]
self.assertBlocksEqual(blocks, *expected)
else:
label_loop_start = Label()
label_loop_exit = Label()
code = Bytecode()
code.extend((
Instr("LOAD_CONST", (1, 2, 3), lineno=1),
Instr("GET_ITER", lineno=1),
label_loop_start,
Instr("FOR_ITER", label_loop_exit, lineno=1),
Instr("STORE_NAME", "x", lineno=1),
Instr("LOAD_NAME", "x", lineno=2),
Instr("LOAD_CONST", 2, lineno=2),
Instr("COMPARE_OP", Compare.EQ, lineno=2),
Instr("POP_JUMP_IF_FALSE", label_loop_start, lineno=2),
Instr("JUMP_ABSOLUTE", label_loop_exit, lineno=3),
Instr("JUMP_ABSOLUTE", label_loop_start, lineno=4),
Instr("JUMP_ABSOLUTE", label_loop_start, lineno=4),
label_loop_exit,
Instr("LOAD_CONST", None, lineno=4),
Instr("RETURN_VALUE", lineno=4),
))
blocks = ControlFlowGraph.from_bytecode(code)
expected = [
[
Instr("LOAD_CONST", (1, 2, 3), lineno=1),
Instr("GET_ITER", lineno=1)
],
[Instr("FOR_ITER", blocks[6], lineno=1)],
[
Instr("STORE_NAME", "x", lineno=1),
Instr("LOAD_NAME", "x", lineno=2),
Instr("LOAD_CONST", 2, lineno=2),
Instr("COMPARE_OP", Compare.EQ, lineno=2),
Instr("POP_JUMP_IF_FALSE", blocks[1], lineno=2),
],
[Instr("JUMP_ABSOLUTE", blocks[6], lineno=3)],
[Instr("JUMP_ABSOLUTE", blocks[1], lineno=4)],
[Instr("JUMP_ABSOLUTE", blocks[1], lineno=4)],
[
Instr("LOAD_CONST", None, lineno=4),
Instr("RETURN_VALUE", lineno=4)
],
]
self.assertBlocksEqual(blocks, *expected)
def UNARY(u_op):
return Instr('UNARY_' + u_op.name)
def UNPACK_SEQUENCE(n: int):
return Instr(instr_names.UNPACK_SEQUENCE, n)
def BINARY(bin_op):
return Instr('BINARY_' + bin_op.name)
def INPLACE_BINARY(bin_op):
return Instr('INPLACE_' + bin_op.name)
def JUMP_ABSOLUTE(i):
return Instr(instr_names.JUMP_ABSOLUTE, i)
def POP_JUMP_IF_FALSE(i):
return Instr(instr_names.POP_JUMP_IF_FALSE, i)
def POP_JUMP_IF_TRUE(i):
return Instr(instr_names.POP_JUMP_IF_TRUE, i)
def test_setlineno(self):
# x = 7
# y = 8
# z = 9
code = Bytecode()
code.first_lineno = 3
code.extend([
Instr("LOAD_CONST", 7),
Instr("STORE_NAME", "x"),
SetLineno(4),
Instr("LOAD_CONST", 8),
Instr("STORE_NAME", "y"),
SetLineno(5),
Instr("LOAD_CONST", 9),
Instr("STORE_NAME", "z"),
])
blocks = ControlFlowGraph.from_bytecode(code)
self.assertBlocksEqual(
blocks,
[
Instr("LOAD_CONST", 7),
Instr("STORE_NAME", "x"),
SetLineno(4),
Instr("LOAD_CONST", 8),
Instr("STORE_NAME", "y"),
SetLineno(5),
Instr("LOAD_CONST", 9),
Instr("STORE_NAME", "z"),
],
)
def number(self, n):
return [Instr('LOAD_CONST', int(n))]
def test_legalize(self):
code = Bytecode()
code.first_lineno = 3
code.extend([
Instr("LOAD_CONST", 7),
Instr("STORE_NAME", "x"),
Instr("LOAD_CONST", 8, lineno=4),
Instr("STORE_NAME", "y"),
SetLineno(5),
Instr("LOAD_CONST", 9, lineno=6),
Instr("STORE_NAME", "z"),
])
blocks = ControlFlowGraph.from_bytecode(code)
blocks.legalize()
self.assertBlocksEqual(
blocks,
[
Instr("LOAD_CONST", 7, lineno=3),
Instr("STORE_NAME", "x", lineno=3),
Instr("LOAD_CONST", 8, lineno=4),
Instr("STORE_NAME", "y", lineno=4),
Instr("LOAD_CONST", 9, lineno=5),
Instr("STORE_NAME", "z", lineno=5),
],
)
def string(self, s):
return [Instr('LOAD_CONST', s[1:-1])]
def test_from_bytecode(self):
bytecode = Bytecode()
label = Label()
bytecode.extend([
Instr("LOAD_NAME", "test", lineno=1),
Instr("POP_JUMP_IF_FALSE", label, lineno=1),
Instr("LOAD_CONST", 5, lineno=2),
Instr("STORE_NAME", "x", lineno=2),
Instr("JUMP_FORWARD", label, lineno=2),
# dead code!
Instr("LOAD_CONST", 7, lineno=4),
Instr("STORE_NAME", "x", lineno=4),
Label(), # unused label
label,
Label(), # unused label
Instr("LOAD_CONST", None, lineno=4),
Instr("RETURN_VALUE", lineno=4),
])
blocks = ControlFlowGraph.from_bytecode(bytecode)
label2 = blocks[3]
self.assertBlocksEqual(
blocks,
[
Instr("LOAD_NAME", "test", lineno=1),
Instr("POP_JUMP_IF_FALSE", label2, lineno=1),
],
[
Instr("LOAD_CONST", 5, lineno=2),
Instr("STORE_NAME", "x", lineno=2),
Instr("JUMP_FORWARD", label2, lineno=2),
],
[
Instr("LOAD_CONST", 7, lineno=4),
Instr("STORE_NAME", "x", lineno=4)
],
[
Instr("LOAD_CONST", None, lineno=4),
Instr("RETURN_VALUE", lineno=4)
],
)
def var(self, n):
return [Instr('LOAD_NAME', n)]
def test_to_code(self):
# test resolution of jump labels
bytecode = ControlFlowGraph()
bytecode.first_lineno = 3
bytecode.argcount = 3
if sys.version_info > (3, 8):
bytecode.posonlyargcount = 0
bytecode.kwonlyargcount = 2
bytecode.name = "func"
bytecode.filename = "hello.py"
bytecode.flags = 0x43
bytecode.argnames = ("arg", "arg2", "arg3", "kwonly", "kwonly2")
bytecode.docstring = None
block0 = bytecode[0]
block1 = bytecode.add_block()
block2 = bytecode.add_block()
block0.extend([
Instr("LOAD_FAST", "x", lineno=4),
Instr("POP_JUMP_IF_FALSE", block2, lineno=4),
])
block1.extend([
Instr("LOAD_FAST", "arg", lineno=5),
Instr("STORE_FAST", "x", lineno=5)
])
block2.extend([
Instr("LOAD_CONST", 3, lineno=6),
Instr("STORE_FAST", "x", lineno=6),
Instr("LOAD_FAST", "x", lineno=7),
Instr("RETURN_VALUE", lineno=7),
])
expected = (b"|\x05"
b"r\x08"
b"|\x00"
b"}\x05"
b"d\x01"
b"}\x05"
b"|\x05"
b"S\x00")
code = bytecode.to_code()
self.assertEqual(code.co_consts, (None, 3))
self.assertEqual(code.co_argcount, 3)
if sys.version_info > (3, 8):
self.assertEqual(code.co_posonlyargcount, 0)
self.assertEqual(code.co_kwonlyargcount, 2)
self.assertEqual(code.co_nlocals, 6)
self.assertEqual(code.co_stacksize, 1)
# FIXME: don't use hardcoded constants
self.assertEqual(code.co_flags, 0x43)
self.assertEqual(code.co_code, expected)
self.assertEqual(code.co_names, ())
self.assertEqual(code.co_varnames,
("arg", "arg2", "arg3", "kwonly", "kwonly2", "x"))
self.assertEqual(code.co_filename, "hello.py")
self.assertEqual(code.co_name, "func")
self.assertEqual(code.co_firstlineno, 3)
# verify stacksize argument is honored
explicit_stacksize = code.co_stacksize + 42
code = bytecode.to_code(stacksize=explicit_stacksize)
self.assertEqual(code.co_stacksize, explicit_stacksize)
def arith_expr(self, a, op, b):
# TODO support chain arithmetic
assert op == '+'
return a + b + [Instr('BINARY_ADD')]
def test_slice(self):
block = BasicBlock([Instr("NOP")])
next_block = BasicBlock()
block.next_block = next_block
self.assertEqual(block, block[:])
self.assertIs(next_block, block[:].next_block)
def funccall(self, name, args):
return name + args + [Instr('CALL_FUNCTION', 1)]
def _instrument_for_loop(
self,
cfg: CFG,
dominator_tree: DominatorTree,
node: ProgramGraphNode,
code_object_id: int,
) -> int:
"""Transform the for loop whose header is defined in the given node.
We only transform the underlying bytecode cfg, by partially unrolling the first
iteration. For this, we add three basic blocks after the loop header:
The first block is called, if the iterator on which the loop is based
yields at least one element, in which case we report the boolean value True
to the tracer, leave the yielded value of the iterator on top of the stack and
jump to the the regular body of the loop.
The second block is called, if the iterator on which the loop is based
does not yield an element, in which case we report the boolean value False
to the tracer and jump to the exit instruction of the loop.
The third block acts as the new internal header of the for loop. It consists
of a copy of the original "FOR_ITER" instruction of the loop.
The original loop header is changed such that it either falls through to the
first block or jumps to the second, if no element is yielded.
Since Python is a structured programming language, there can be no jumps
directly into the loop that bypass the loop header (e.g., GOTO).
Jumps which reach the loop header from outside the loop will still target
the original loop header, so they don't need to be modified.
Jumps which originate from within the loop (e.g., break or continue) need
to be redirected to the new internal header (3rd new block).
We use a dominator tree to find and redirect the jumps of such instructions.
Args:
cfg: The CFG that contains the loop
dominator_tree: The dominator tree of the given CFG.
node: The node which contains the header of the for loop.
code_object_id: The id of the containing Code Object.
Returns:
The ID of the instrumented predicate
"""
assert node.basic_block is not None, "Basic block of for loop cannot be None."
for_instr = node.basic_block[self._JUMP_OP_POS]
assert for_instr.name == "FOR_ITER"
lineno = for_instr.lineno
predicate_id = self._tracer.register_predicate(
PredicateMetaData(line_no=lineno, code_object_id=code_object_id)
)
for_instr_copy = for_instr.copy()
for_loop_exit = for_instr.arg
for_loop_body = node.basic_block.next_block
# pylint:disable=unbalanced-tuple-unpacking
entered, not_entered, new_header = self._create_consecutive_blocks(
cfg.bytecode_cfg(), node.basic_block, 3
)
for_instr.arg = not_entered
entered.extend(
[
Instr("LOAD_CONST", self._tracer, lineno=lineno),
Instr(
"LOAD_METHOD",
ExecutionTracer.executed_bool_predicate.__name__,
lineno=lineno,
),
Instr("LOAD_CONST", True, lineno=lineno),
Instr("LOAD_CONST", predicate_id, lineno=lineno),
Instr("CALL_METHOD", 2, lineno=lineno),
Instr("POP_TOP", lineno=lineno),
Instr("JUMP_ABSOLUTE", for_loop_body, lineno=lineno),
]
)
not_entered.extend(
[
Instr("LOAD_CONST", self._tracer, lineno=lineno),
Instr(
"LOAD_METHOD",
ExecutionTracer.executed_bool_predicate.__name__,
lineno=lineno,
),
Instr("LOAD_CONST", False, lineno=lineno),
Instr("LOAD_CONST", predicate_id, lineno=lineno),
Instr("CALL_METHOD", 2, lineno=lineno),
Instr("POP_TOP", lineno=lineno),
Instr("JUMP_ABSOLUTE", for_loop_exit, lineno=lineno),
]
)
new_header.append(for_instr_copy)
# Redirect internal jumps to the new loop header
for successor in dominator_tree.get_transitive_successors(node):
if (
successor.basic_block is not None
and successor.basic_block[self._JUMP_OP_POS].arg is node.basic_block
):
successor.basic_block[self._JUMP_OP_POS].arg = new_header
return predicate_id
def file_input(self, stmts):
return sum(stmts, []) + [Instr("RETURN_VALUE")]
def BUILD_STRING(n: int, *, lineno=None):
return Instr('BUILD_STRING', n, lineno=lineno)
def expr_stmt(self, lval, rval):
# TODO more complicated than that
name, = lval
assert name.name == 'LOAD_NAME' # XXX avoid with another layer of abstraction
return rval + [Instr("STORE_NAME", name.arg)]
def UNPACK_SEQUENCE(n: int, *, lineno=None):
return Instr('UNPACK_SEQUENCE', n, lineno=lineno)
def DUP():
return Instr(instr_names.DUP_TOP)
