def test_jumps(self):
# if test:
# x = 12
# else:
# x = 37
code = Bytecode()
label_else = Label()
label_return = Label()
code.extend([Instr('LOAD_NAME', 'test', lineno=1),
Instr('POP_JUMP_IF_FALSE', label_else),
Instr('LOAD_CONST', 12, lineno=2),
Instr('STORE_NAME', 'x'),
Instr('JUMP_FORWARD', label_return),
label_else,
Instr('LOAD_CONST', 37, lineno=4),
Instr('STORE_NAME', 'x'),
label_return,
Instr('LOAD_CONST', None, lineno=4),
Instr('RETURN_VALUE')])
code = code.to_concrete_bytecode()
expected = [ConcreteInstr('LOAD_NAME', 0, lineno=1),
ConcreteInstr('POP_JUMP_IF_FALSE', 15, lineno=1),
ConcreteInstr('LOAD_CONST', 0, lineno=2),
ConcreteInstr('STORE_NAME', 1, lineno=2),
ConcreteInstr('JUMP_FORWARD', 6, lineno=2),
ConcreteInstr('LOAD_CONST', 1, lineno=4),
ConcreteInstr('STORE_NAME', 1, lineno=4),
ConcreteInstr('LOAD_CONST', 2, lineno=4),
ConcreteInstr('RETURN_VALUE', lineno=4)]
self.assertListEqual(list(code), expected)
self.assertListEqual(code.consts, [12, 37, None])
self.assertListEqual(code.names, ['test', 'x'])
self.assertListEqual(code.varnames, [])
def test_invalid_types(self):
code = Bytecode()
code.append(123)
with self.assertRaises(ValueError):
list(code)
with self.assertRaises(ValueError):
Bytecode([123])
def test_optimize_code_obj(self):
# Test optimize() method with a code object
#
# x = 3 + 5 => x = 8
noopt = Bytecode([
Instr("LOAD_CONST", 3),
Instr("LOAD_CONST", 5),
Instr("BINARY_ADD"),
Instr("STORE_NAME", "x"),
Instr("LOAD_CONST", None),
Instr("RETURN_VALUE"),
])
noopt = noopt.to_code()
optimizer = peephole_opt.PeepholeOptimizer()
optim = optimizer.optimize(noopt)
code = Bytecode.from_code(optim)
self.assertEqual(
code,
[
Instr("LOAD_CONST", 8, lineno=1),
Instr("STORE_NAME", "x", lineno=1),
Instr("LOAD_CONST", None, lineno=1),
Instr("RETURN_VALUE", lineno=1),
],
)
def test_extended_jump(self):
NOP = bytes((opcode.opmap["NOP"], ))
class BigInstr(ConcreteInstr):
def __init__(self, size):
super().__init__("NOP")
self._size = size
def copy(self):
return self
def assemble(self):
return NOP * self._size
# (invalid) code using jumps > 0xffff to test extended arg
label = Label()
nb_nop = 2**16
code = Bytecode([
Instr("JUMP_ABSOLUTE", label),
BigInstr(nb_nop),
label,
Instr("LOAD_CONST", None),
Instr("RETURN_VALUE"),
])
code_obj = code.to_code()
if OFFSET_AS_INSTRUCTION:
expected = b"\x90\x80q\x02" + NOP * nb_nop + b"d\x00S\x00"
else:
expected = b"\x90\x01\x90\x00q\x06" + NOP * nb_nop + b"d\x00S\x00"
self.assertEqual(code_obj.co_code, expected)
def test_legalize(self):
code = Bytecode()
code.first_lineno = 3
code.extend([
Instr("LOAD_CONST", 7),
Instr("STORE_NAME", "x"),
Instr("LOAD_CONST", 8, lineno=4),
Instr("STORE_NAME", "y"),
SetLineno(5),
Instr("LOAD_CONST", 9, lineno=6),
Instr("STORE_NAME", "z"),
])
blocks = ControlFlowGraph.from_bytecode(code)
blocks.legalize()
self.assertBlocksEqual(
blocks,
[
Instr("LOAD_CONST", 7, lineno=3),
Instr("STORE_NAME", "x", lineno=3),
Instr("LOAD_CONST", 8, lineno=4),
Instr("STORE_NAME", "y", lineno=4),
Instr("LOAD_CONST", 9, lineno=5),
Instr("STORE_NAME", "z", lineno=5),
],
)
def test_setlineno(self):
# x = 7
# y = 8
# z = 9
code = Bytecode()
code.first_lineno = 3
code.extend([
Instr("LOAD_CONST", 7),
Instr("STORE_NAME", "x"),
SetLineno(4),
Instr("LOAD_CONST", 8),
Instr("STORE_NAME", "y"),
SetLineno(5),
Instr("LOAD_CONST", 9),
Instr("STORE_NAME", "z"),
])
blocks = ControlFlowGraph.from_bytecode(code)
self.assertBlocksEqual(
blocks,
[
Instr("LOAD_CONST", 7),
Instr("STORE_NAME", "x"),
SetLineno(4),
Instr("LOAD_CONST", 8),
Instr("STORE_NAME", "y"),
SetLineno(5),
Instr("LOAD_CONST", 9),
Instr("STORE_NAME", "z"),
],
)
def test_max_size(self):
max_size = 3
with mock.patch.object(peephole_opt, 'MAX_SIZE', max_size):
# optimized binary operation: size <= maximum size
#
# (9,) * size
size = max_size
result = (9, ) * size
code = Bytecode([
Instr('LOAD_CONST', 9),
Instr('BUILD_TUPLE', 1),
Instr('LOAD_CONST', size),
Instr('BINARY_MULTIPLY'),
Instr('STORE_NAME', 'x')
])
self.check(code, Instr('LOAD_CONST', result),
Instr('STORE_NAME', 'x'))
# don't optimize binary operation: size > maximum size
#
# x = (9,) * size
size = (max_size + 1)
code = Bytecode([
Instr('LOAD_CONST', 9),
Instr('BUILD_TUPLE', 1),
Instr('LOAD_CONST', size),
Instr('BINARY_MULTIPLY'),
Instr('STORE_NAME', 'x')
])
self.check(code, Instr('LOAD_CONST', (9, )),
Instr('LOAD_CONST', size), Instr('BINARY_MULTIPLY'),
Instr('STORE_NAME', 'x'))
def test_label2(self):
bytecode = Bytecode()
label = Label()
bytecode.extend([
Instr('LOAD_NAME', 'test', lineno=1),
Instr('POP_JUMP_IF_FALSE', label),
Instr('LOAD_CONST', 5, lineno=2),
Instr('STORE_NAME', 'x'),
Instr('JUMP_FORWARD', label),
Instr('LOAD_CONST', 7, lineno=4),
Instr('STORE_NAME', 'x'), label,
Instr('LOAD_CONST', None),
Instr('RETURN_VALUE')
])
concrete = bytecode.to_concrete_bytecode()
expected = [
ConcreteInstr('LOAD_NAME', 0, lineno=1),
ConcreteInstr('POP_JUMP_IF_FALSE',
14 if WORDCODE else 21,
lineno=1),
ConcreteInstr('LOAD_CONST', 0, lineno=2),
ConcreteInstr('STORE_NAME', 1, lineno=2),
ConcreteInstr('JUMP_FORWARD', 4 if WORDCODE else 6, lineno=2),
ConcreteInstr('LOAD_CONST', 1, lineno=4),
ConcreteInstr('STORE_NAME', 1, lineno=4),
ConcreteInstr('LOAD_CONST', 2, lineno=4),
ConcreteInstr('RETURN_VALUE', lineno=4)
]
self.assertListEqual(list(concrete), expected)
self.assertListEqual(concrete.consts, [5, 7, None])
self.assertListEqual(concrete.names, ['test', 'x'])
self.assertListEqual(concrete.varnames, [])
def test_extended_jump(self):
NOP = bytes((opcode.opmap['NOP'], ))
class BigInstr(ConcreteInstr):
def __init__(self, size):
super().__init__('NOP')
self._size = size
def copy(self):
return self
def assemble(self):
return NOP * self._size
# (invalid) code using jumps > 0xffff to test extended arg
label = Label()
nb_nop = 2**16
code = Bytecode([
Instr("JUMP_ABSOLUTE", label),
BigInstr(nb_nop), label,
Instr('LOAD_CONST', None),
Instr('RETURN_VALUE')
])
code_obj = code.to_code()
if WORDCODE:
expected = b'\x90\x01\x90\x00q\x06' + NOP * nb_nop + b'd\x00S\x00'
else:
expected = b'\x90\x01\x00q\x06\x00' + NOP * nb_nop + b'd\x00\x00S'
self.assertEqual(code_obj.co_code, expected)
def test_from_bytecode(self):
bytecode = Bytecode()
label = Label()
bytecode.extend([
Instr('LOAD_NAME', 'test', lineno=1),
Instr('POP_JUMP_IF_FALSE', label, lineno=1),
Instr('LOAD_CONST', 5, lineno=2),
Instr('STORE_NAME', 'x', lineno=2),
Instr('JUMP_FORWARD', label, lineno=2),
# dead code!
Instr('LOAD_CONST', 7, lineno=4),
Instr('STORE_NAME', 'x', lineno=4),
Label(), # unused label
label,
Label(), # unused label
Instr('LOAD_CONST', None, lineno=4),
Instr('RETURN_VALUE', lineno=4)
])
blocks = ControlFlowGraph.from_bytecode(bytecode)
label2 = blocks[3]
self.assertBlocksEqual(blocks, [
Instr('LOAD_NAME', 'test', lineno=1),
Instr('POP_JUMP_IF_FALSE', label2, lineno=1)
], [
Instr('LOAD_CONST', 5, lineno=2),
Instr('STORE_NAME', 'x', lineno=2),
Instr('JUMP_FORWARD', label2, lineno=2)
], [
Instr('LOAD_CONST', 7, lineno=4),
Instr('STORE_NAME', 'x', lineno=4)
], [
Instr('LOAD_CONST', None, lineno=4),
Instr('RETURN_VALUE', lineno=4)
])
def test_code_object_containing_mutable_data(self):
from types import CodeType
from bytecode import Bytecode, Instr
def f():
def g():
return "value"
return g
f_code = Bytecode.from_code(f.__code__)
instr_load_code = None
mutable_datum = [4, 2]
for each in f_code:
if (isinstance(each, Instr) and each.name == "LOAD_CONST"
and isinstance(each.arg, CodeType)):
instr_load_code = each
break
self.assertIsNotNone(instr_load_code)
g_code = Bytecode.from_code(instr_load_code.arg)
g_code[0].arg = mutable_datum
instr_load_code.arg = g_code.to_code()
f.__code__ = f_code.to_code()
self.assertIs(f()(), mutable_datum)
def case(code, ctx, debug=False, cpython_compat=True):
stmt = parse(code).result
code_obj = py_compile(stmt, is_entrypoint=False)
if debug:
with open('out_yapypy_bc.log',
'w') as yapypy_bc, open('out_yapypy_info.log',
'w') as yapypy_info:
dis_code(code_obj, yapypy_bc)
show_code(code_obj, yapypy_info)
if cpython_compat:
code_obj2 = compile(code, "", "exec")
with open('out_cpy_bc.log',
'w') as cpy_bc, open('out_cpy_info.log',
'w') as cpy_info:
dis_code(code_obj2, cpy_bc)
show_code(code_obj2, cpy_info)
print('python:')
exec(Bytecode.from_code(code_obj2).to_code(), ctx or {})
print('yapypy')
exec(Bytecode.from_code(code_obj).to_code(), ctx or {})
else:
exec(code_obj, ctx)
def test_label2(self):
bytecode = Bytecode()
label = Label()
bytecode.extend([Instr('LOAD_NAME', 'test', lineno=1),
Instr('POP_JUMP_IF_FALSE', label),
Instr('LOAD_CONST', 5, lineno=2),
Instr('STORE_NAME', 'x'),
Instr('JUMP_FORWARD', label),
Instr('LOAD_CONST', 7, lineno=4),
Instr('STORE_NAME', 'x'),
label,
Instr('LOAD_CONST', None),
Instr('RETURN_VALUE')])
concrete = bytecode.to_concrete_bytecode()
expected = [ConcreteInstr('LOAD_NAME', 0, lineno=1),
ConcreteInstr('POP_JUMP_IF_FALSE',
14 if WORDCODE else 21, lineno=1),
ConcreteInstr('LOAD_CONST', 0, lineno=2),
ConcreteInstr('STORE_NAME', 1, lineno=2),
ConcreteInstr('JUMP_FORWARD',
4 if WORDCODE else 6, lineno=2),
ConcreteInstr('LOAD_CONST', 1, lineno=4),
ConcreteInstr('STORE_NAME', 1, lineno=4),
ConcreteInstr('LOAD_CONST', 2, lineno=4),
ConcreteInstr('RETURN_VALUE', lineno=4)]
self.assertListEqual(list(concrete), expected)
self.assertListEqual(concrete.consts, [5, 7, None])
self.assertListEqual(concrete.names, ['test', 'x'])
self.assertListEqual(concrete.varnames, [])
def test_dont_optimize(self):
# x = 3 < 5
code = Bytecode(
[
Instr("LOAD_CONST", 3),
Instr("LOAD_CONST", 5),
Instr("COMPARE_OP", Compare.LT),
Instr("STORE_NAME", "x"),
Instr("LOAD_CONST", None),
Instr("RETURN_VALUE"),
]
)
self.check_dont_optimize(code)
# x = (10, 20, 30)[1:]
code = Bytecode(
[
Instr("LOAD_CONST", (10, 20, 30)),
Instr("LOAD_CONST", 1),
Instr("LOAD_CONST", None),
Instr("BUILD_SLICE", 2),
Instr("BINARY_SUBSCR"),
Instr("STORE_NAME", "x"),
]
)
self.check_dont_optimize(code)
def test_general_constants(self):
"""Test if general object could be linked as constants.
"""
class CustomObject:
pass
class UnHashableCustomObject:
__hash__ = None
obj1 = [1, 2, 3]
obj2 = {1, 2, 3}
obj3 = CustomObject()
obj4 = UnHashableCustomObject()
code = Bytecode([Instr('LOAD_CONST', obj1, lineno=1),
Instr('LOAD_CONST', obj2, lineno=1),
Instr('LOAD_CONST', obj3, lineno=1),
Instr('LOAD_CONST', obj4, lineno=1),
Instr('BUILD_TUPLE', 4, lineno=1),
Instr('RETURN_VALUE', lineno=1)])
self.assertEqual(code.to_code().co_consts,
(obj1, obj2, obj3, obj4))
def f():
return # pragma: no cover
f.__code__ = code.to_code()
self.assertEqual(f(), (obj1, obj2, obj3, obj4))
def test_setlineno(self):
# x = 7
# y = 8
# z = 9
code = Bytecode()
code.first_lineno = 3
code.extend([Instr("LOAD_CONST", 7),
Instr("STORE_NAME", 'x'),
SetLineno(4),
Instr("LOAD_CONST", 8),
Instr("STORE_NAME", 'y'),
SetLineno(5),
Instr("LOAD_CONST", 9),
Instr("STORE_NAME", 'z')])
blocks = ControlFlowGraph.from_bytecode(code)
self.assertBlocksEqual(blocks,
[Instr("LOAD_CONST", 7),
Instr("STORE_NAME", 'x'),
SetLineno(4),
Instr("LOAD_CONST", 8),
Instr("STORE_NAME", 'y'),
SetLineno(5),
Instr("LOAD_CONST", 9),
Instr("STORE_NAME", 'z')])
def test_slice(self):
code = Bytecode()
code.first_lineno = 3
code.extend([
Instr("LOAD_CONST", 7),
Instr("STORE_NAME", "x"),
SetLineno(4),
Instr("LOAD_CONST", 8),
Instr("STORE_NAME", "y"),
SetLineno(5),
Instr("LOAD_CONST", 9),
Instr("STORE_NAME", "z"),
])
sliced_code = code[:]
self.assertEqual(code, sliced_code)
for name in (
"argcount",
"posonlyargcount",
"kwonlyargcount",
"first_lineno",
"name",
"filename",
"docstring",
"cellvars",
"freevars",
"argnames",
):
self.assertEqual(getattr(code, name, None),
getattr(sliced_code, name, None))
def test_from_bytecode(self):
bytecode = Bytecode()
label = Label()
bytecode.extend([Instr('LOAD_NAME', 'test', lineno=1),
Instr('POP_JUMP_IF_FALSE', label, lineno=1),
Instr('LOAD_CONST', 5, lineno=2),
Instr('STORE_NAME', 'x', lineno=2),
Instr('JUMP_FORWARD', label, lineno=2),
# dead code!
Instr('LOAD_CONST', 7, lineno=4),
Instr('STORE_NAME', 'x', lineno=4),
Label(), # unused label
label,
Label(), # unused label
Instr('LOAD_CONST', None, lineno=4),
Instr('RETURN_VALUE', lineno=4)])
blocks = ControlFlowGraph.from_bytecode(bytecode)
label2 = blocks[3]
self.assertBlocksEqual(blocks,
[Instr('LOAD_NAME', 'test', lineno=1),
Instr('POP_JUMP_IF_FALSE', label2, lineno=1)],
[Instr('LOAD_CONST', 5, lineno=2),
Instr('STORE_NAME', 'x', lineno=2),
Instr('JUMP_FORWARD', label2, lineno=2)],
[Instr('LOAD_CONST', 7, lineno=4),
Instr('STORE_NAME', 'x', lineno=4)],
[Instr('LOAD_CONST', None, lineno=4),
Instr('RETURN_VALUE', lineno=4)])
def test_extended_jump(self):
NOP = bytes((opcode.opmap['NOP'],))
class BigInstr(ConcreteInstr):
def __init__(self, size):
super().__init__('NOP')
self._size = size
def copy(self):
return self
def assemble(self):
return NOP * self._size
# (invalid) code using jumps > 0xffff to test extended arg
label = Label()
nb_nop = 2**16
code = Bytecode([Instr("JUMP_ABSOLUTE", label),
BigInstr(nb_nop),
label,
Instr('LOAD_CONST', None),
Instr('RETURN_VALUE')])
code_obj = code.to_code()
if WORDCODE:
expected = b'\x90\x01\x90\x00q\x06' + NOP * nb_nop + b'd\x00S\x00'
else:
expected = b'\x90\x01\x00q\x06\x00' + NOP * nb_nop + b'd\x00\x00S'
self.assertEqual(code_obj.co_code, expected)
def test_from_bytecode_loop(self):
# for x in (1, 2, 3):
# if x == 2:
# break
# continue
label_loop_start = Label()
label_loop_exit = Label()
label_loop_end = Label()
code = Bytecode()
code.extend((Instr('SETUP_LOOP', label_loop_end, lineno=1),
Instr('LOAD_CONST', (1, 2, 3), lineno=1),
Instr('GET_ITER', lineno=1),
label_loop_start,
Instr('FOR_ITER', label_loop_exit, lineno=1),
Instr('STORE_NAME', 'x', lineno=1),
Instr('LOAD_NAME', 'x', lineno=2),
Instr('LOAD_CONST', 2, lineno=2),
Instr('COMPARE_OP', Compare.EQ, lineno=2),
Instr('POP_JUMP_IF_FALSE', label_loop_start, lineno=2),
Instr('BREAK_LOOP', lineno=3),
Instr('JUMP_ABSOLUTE', label_loop_start, lineno=4),
Instr('JUMP_ABSOLUTE', label_loop_start, lineno=4),
label_loop_exit,
Instr('POP_BLOCK', lineno=4),
label_loop_end,
Instr('LOAD_CONST', None, lineno=4),
Instr('RETURN_VALUE', lineno=4),
))
blocks = ControlFlowGraph.from_bytecode(code)
expected = [[Instr('SETUP_LOOP', blocks[8], lineno=1)],
[Instr('LOAD_CONST', (1, 2, 3), lineno=1),
Instr('GET_ITER', lineno=1)],
[Instr('FOR_ITER', blocks[7], lineno=1)],
[Instr('STORE_NAME', 'x', lineno=1),
Instr('LOAD_NAME', 'x', lineno=2),
Instr('LOAD_CONST', 2, lineno=2),
Instr('COMPARE_OP', Compare.EQ, lineno=2),
Instr('POP_JUMP_IF_FALSE', blocks[2], lineno=2)],
[Instr('BREAK_LOOP', lineno=3)],
[Instr('JUMP_ABSOLUTE', blocks[2], lineno=4)],
[Instr('JUMP_ABSOLUTE', blocks[2], lineno=4)],
[Instr('POP_BLOCK', lineno=4)],
[Instr('LOAD_CONST', None, lineno=4),
Instr('RETURN_VALUE', lineno=4)]]
self.assertBlocksEqual(blocks, *expected)
def test_compare_op_unary_not(self):
for op, not_op in (
(Compare.IN, Compare.NOT_IN), # in => not in
(Compare.NOT_IN, Compare.IN), # not in => in
(Compare.IS, Compare.IS_NOT), # is => is not
(Compare.IS_NOT, Compare.IS), # is not => is
):
code = Bytecode([
Instr('LOAD_NAME', 'a'),
Instr('LOAD_NAME', 'b'),
Instr('COMPARE_OP', op),
Instr('UNARY_NOT'),
Instr('STORE_NAME', 'x')
])
self.check(code, Instr('LOAD_NAME', 'a'), Instr('LOAD_NAME', 'b'),
Instr('COMPARE_OP', not_op), Instr('STORE_NAME', 'x'))
# don't optimize:
# x = not (a and b is True)
label_instr5 = Label()
code = Bytecode([
Instr('LOAD_NAME', 'a'),
Instr('JUMP_IF_FALSE_OR_POP', label_instr5),
Instr('LOAD_NAME', 'b'),
Instr('LOAD_CONST', True),
Instr('COMPARE_OP', Compare.IS), label_instr5,
Instr('UNARY_NOT'),
Instr('STORE_NAME', 'x'),
Instr('LOAD_CONST', None),
Instr('RETURN_VALUE')
])
self.check_dont_optimize(code)
def test_label2(self):
bytecode = Bytecode()
label = Label()
bytecode.extend([
Instr("LOAD_NAME", "test", lineno=1),
Instr("POP_JUMP_IF_FALSE", label),
Instr("LOAD_CONST", 5, lineno=2),
Instr("STORE_NAME", "x"),
Instr("JUMP_FORWARD", label),
Instr("LOAD_CONST", 7, lineno=4),
Instr("STORE_NAME", "x"),
label,
Instr("LOAD_CONST", None),
Instr("RETURN_VALUE"),
])
concrete = bytecode.to_concrete_bytecode()
expected = [
ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("POP_JUMP_IF_FALSE",
14 if WORDCODE else 21,
lineno=1),
ConcreteInstr("LOAD_CONST", 0, lineno=2),
ConcreteInstr("STORE_NAME", 1, lineno=2),
ConcreteInstr("JUMP_FORWARD", 4 if WORDCODE else 6, lineno=2),
ConcreteInstr("LOAD_CONST", 1, lineno=4),
ConcreteInstr("STORE_NAME", 1, lineno=4),
ConcreteInstr("LOAD_CONST", 2, lineno=4),
ConcreteInstr("RETURN_VALUE", lineno=4),
]
self.assertListEqual(list(concrete), expected)
self.assertListEqual(concrete.consts, [5, 7, None])
self.assertListEqual(concrete.names, ["test", "x"])
self.assertListEqual(concrete.varnames, [])
def test_return_value(self):
# return+return: remove second return
#
# def func():
# return 4
# return 5
code = Bytecode([
Instr('LOAD_CONST', 4, lineno=2),
Instr('RETURN_VALUE', lineno=2),
Instr('LOAD_CONST', 5, lineno=3),
Instr('RETURN_VALUE', lineno=3)
])
code = ControlFlowGraph.from_bytecode(code)
self.check(code, Instr('LOAD_CONST', 4, lineno=2),
Instr('RETURN_VALUE', lineno=2))
# return+return + return+return: remove second and fourth return
#
# def func():
# return 4
# return 5
# return 6
# return 7
code = Bytecode([
Instr('LOAD_CONST', 4, lineno=2),
Instr('RETURN_VALUE', lineno=2),
Instr('LOAD_CONST', 5, lineno=3),
Instr('RETURN_VALUE', lineno=3),
Instr('LOAD_CONST', 6, lineno=4),
Instr('RETURN_VALUE', lineno=4),
Instr('LOAD_CONST', 7, lineno=5),
Instr('RETURN_VALUE', lineno=5)
])
code = ControlFlowGraph.from_bytecode(code)
self.check(code, Instr('LOAD_CONST', 4, lineno=2),
Instr('RETURN_VALUE', lineno=2))
# return + JUMP_ABSOLUTE: remove JUMP_ABSOLUTE
# while 1:
# return 7
setup_loop = Label()
return_label = Label()
code = Bytecode([
setup_loop,
Instr('SETUP_LOOP', return_label, lineno=2),
Instr('LOAD_CONST', 7, lineno=3),
Instr('RETURN_VALUE', lineno=3),
Instr('JUMP_ABSOLUTE', setup_loop, lineno=3),
Instr('POP_BLOCK', lineno=3), return_label,
Instr('LOAD_CONST', None, lineno=3),
Instr('RETURN_VALUE', lineno=3)
])
code = ControlFlowGraph.from_bytecode(code)
end_loop = Label()
self.check(code, Instr('SETUP_LOOP', end_loop, lineno=2),
Instr('LOAD_CONST', 7, lineno=3),
Instr('RETURN_VALUE', lineno=3), end_loop,
Instr('LOAD_CONST', None, lineno=3),
Instr('RETURN_VALUE', lineno=3))
def test_general_constants(self):
"""Test if general object could be linked as constants.
"""
class CustomObject:
pass
class UnHashableCustomObject:
__hash__ = None
obj1 = [1, 2, 3]
obj2 = {1, 2, 3}
obj3 = CustomObject()
obj4 = UnHashableCustomObject()
code = Bytecode([
Instr('LOAD_CONST', obj1, lineno=1),
Instr('LOAD_CONST', obj2, lineno=1),
Instr('LOAD_CONST', obj3, lineno=1),
Instr('LOAD_CONST', obj4, lineno=1),
Instr('BUILD_TUPLE', 4, lineno=1),
Instr('RETURN_VALUE', lineno=1)
])
self.assertEqual(code.to_code().co_consts, (obj1, obj2, obj3, obj4))
def f():
return # pragma: no cover
f.__code__ = code.to_code()
self.assertEqual(f(), (obj1, obj2, obj3, obj4))
def test_combined_unary_bin_ops(self):
# x = 1 + 3 + 7
code = Bytecode([
Instr("LOAD_CONST", 1),
Instr("LOAD_CONST", 3),
Instr("BINARY_ADD"),
Instr("LOAD_CONST", 7),
Instr("BINARY_ADD"),
Instr("STORE_NAME", "x"),
])
self.check(code, Instr("LOAD_CONST", 11), Instr("STORE_NAME", "x"))
# x = ~(~(5))
code = Bytecode([
Instr("LOAD_CONST", 5),
Instr("UNARY_INVERT"),
Instr("UNARY_INVERT"),
Instr("STORE_NAME", "x"),
])
self.check(code, Instr("LOAD_CONST", 5), Instr("STORE_NAME", "x"))
# "events = [(0, 'call'), (1, 'line'), (-(3), 'call')]"
code = Bytecode([
Instr("LOAD_CONST", 0),
Instr("LOAD_CONST", "call"),
Instr("BUILD_TUPLE", 2),
Instr("LOAD_CONST", 1),
Instr("LOAD_CONST", "line"),
Instr("BUILD_TUPLE", 2),
Instr("LOAD_CONST", 3),
Instr("UNARY_NEGATIVE"),
Instr("LOAD_CONST", "call"),
Instr("BUILD_TUPLE", 2),
Instr("BUILD_LIST", 3),
Instr("STORE_NAME", "events"),
])
self.check(
code,
Instr("LOAD_CONST", (0, "call")),
Instr("LOAD_CONST", (1, "line")),
Instr("LOAD_CONST", (-3, "call")),
Instr("BUILD_LIST", 3),
Instr("STORE_NAME", "events"),
)
# 'x = (1,) + (0,) * 8'
code = Bytecode([
Instr("LOAD_CONST", 1),
Instr("BUILD_TUPLE", 1),
Instr("LOAD_CONST", 0),
Instr("BUILD_TUPLE", 1),
Instr("LOAD_CONST", 8),
Instr("BINARY_MULTIPLY"),
Instr("BINARY_ADD"),
Instr("STORE_NAME", "x"),
])
zeros = (0, ) * 8
result = (1, ) + zeros
self.check(code, Instr("LOAD_CONST", result), Instr("STORE_NAME", "x"))
def __init__(self):
self.co_code = Bytecode()
self.co_consts = [None]
self.co_names = []
self.co_varnames = []
self.co_lnotab = array('B')
self.blocks = []
self.stack_history = []
def test_all(self):
for each in filter(lambda p: p[-1].endswith('.py'), yapypy.collect()):
filename = each.__str__()
if each.parent().exists():
pass
else:
each.parent().mkdir()
with each.open('r') as fr:
collector = DocStringsCollector()
mod = ast.parse(fr.read())
collector.visit(mod)
mod_name, _ = splitext(each.relative())
for idx, [fn_name, lineno, title, prepare_code,
test_code] in enumerate(collector.docs):
print(f'tests of {mod_name}.{title or fn_name} started...')
context = {'self': self}
prepare_code = dedent_all(prepare_code)
test_code = dedent_all(test_code)
fixer = FixLineno(lineno)
try:
node = ast.parse(prepare_code, filename, mode='exec')
fixer.visit(node)
code = compile(node, filename, "exec")
except SyntaxError as exc:
exc.lineno = lineno
exc.filename = filename
raise exc
bc = Bytecode.from_code(code)
bc.filename = filename
bc.first_lineno = lineno
exec(bc.to_code(), context)
# not correct but as a workaround
fixer = FixLineno(lineno + test_code.count('\n'))
try:
node = parse(test_code, filename).result
# pprint(node)
fixer.visit(node)
code = py_compile(node, filename, is_entrypoint=True)
except SyntaxError as exc:
exc.lineno = lineno
exc.filename = filename
raise exc
bc = Bytecode.from_code(code)
bc.filename = filename
bc.first_lineno = lineno
code_obj = bc.to_code()
exec(code_obj, context)
print(f'tests of {mod_name}.{title or fn_name} passed.')
def test_build_list_unpack_seq(self):
for build_list in ("BUILD_TUPLE", "BUILD_LIST"):
# x, = [a]
code = Bytecode(
[
Instr("LOAD_NAME", "a"),
Instr(build_list, 1),
Instr("UNPACK_SEQUENCE", 1),
Instr("STORE_NAME", "x"),
]
)
self.check(code, Instr("LOAD_NAME", "a"), Instr("STORE_NAME", "x"))
# x, y = [a, b]
code = Bytecode(
[
Instr("LOAD_NAME", "a"),
Instr("LOAD_NAME", "b"),
Instr(build_list, 2),
Instr("UNPACK_SEQUENCE", 2),
Instr("STORE_NAME", "x"),
Instr("STORE_NAME", "y"),
]
)
self.check(
code,
Instr("LOAD_NAME", "a"),
Instr("LOAD_NAME", "b"),
Instr("ROT_TWO"),
Instr("STORE_NAME", "x"),
Instr("STORE_NAME", "y"),
)
# x, y, z = [a, b, c]
code = Bytecode(
[
Instr("LOAD_NAME", "a"),
Instr("LOAD_NAME", "b"),
Instr("LOAD_NAME", "c"),
Instr(build_list, 3),
Instr("UNPACK_SEQUENCE", 3),
Instr("STORE_NAME", "x"),
Instr("STORE_NAME", "y"),
Instr("STORE_NAME", "z"),
]
)
self.check(
code,
Instr("LOAD_NAME", "a"),
Instr("LOAD_NAME", "b"),
Instr("LOAD_NAME", "c"),
Instr("ROT_THREE"),
Instr("ROT_TWO"),
Instr("STORE_NAME", "x"),
Instr("STORE_NAME", "y"),
Instr("STORE_NAME", "z"),
)
def _get_predefine_code(self, predefine):
bytecode_ = Bytecode.from_code(predefine.__code__)
for bc in bytecode_:
# stubstitute *_FAST to *_NAME such as STORE_FAST to STORE_NAME
# so that variable will be stored in given namespace
bc.name = bc.name.replace('_FAST', '_NAME')
# there is a mysterious bug with 'return bytecode_.to_code()'
return Bytecode(bytecode_).to_code()
def test_from_bytecode_loop(self):
# for x in (1, 2, 3):
# if x == 2:
# break
# continue
label_loop_start = Label()
label_loop_exit = Label()
label_loop_end = Label()
code = Bytecode()
code.extend((
Instr("SETUP_LOOP", label_loop_end, lineno=1),
Instr("LOAD_CONST", (1, 2, 3), lineno=1),
Instr("GET_ITER", lineno=1),
label_loop_start,
Instr("FOR_ITER", label_loop_exit, lineno=1),
Instr("STORE_NAME", "x", lineno=1),
Instr("LOAD_NAME", "x", lineno=2),
Instr("LOAD_CONST", 2, lineno=2),
Instr("COMPARE_OP", Compare.EQ, lineno=2),
Instr("POP_JUMP_IF_FALSE", label_loop_start, lineno=2),
Instr("BREAK_LOOP", lineno=3),
Instr("JUMP_ABSOLUTE", label_loop_start, lineno=4),
Instr("JUMP_ABSOLUTE", label_loop_start, lineno=4),
label_loop_exit,
Instr("POP_BLOCK", lineno=4),
label_loop_end,
Instr("LOAD_CONST", None, lineno=4),
Instr("RETURN_VALUE", lineno=4),
))
blocks = ControlFlowGraph.from_bytecode(code)
expected = [
[Instr("SETUP_LOOP", blocks[8], lineno=1)],
[
Instr("LOAD_CONST", (1, 2, 3), lineno=1),
Instr("GET_ITER", lineno=1)
],
[Instr("FOR_ITER", blocks[7], lineno=1)],
[
Instr("STORE_NAME", "x", lineno=1),
Instr("LOAD_NAME", "x", lineno=2),
Instr("LOAD_CONST", 2, lineno=2),
Instr("COMPARE_OP", Compare.EQ, lineno=2),
Instr("POP_JUMP_IF_FALSE", blocks[2], lineno=2),
],
[Instr("BREAK_LOOP", lineno=3)],
[Instr("JUMP_ABSOLUTE", blocks[2], lineno=4)],
[Instr("JUMP_ABSOLUTE", blocks[2], lineno=4)],
[Instr("POP_BLOCK", lineno=4)],
[
Instr("LOAD_CONST", None, lineno=4),
Instr("RETURN_VALUE", lineno=4)
],
]
self.assertBlocksEqual(blocks, *expected)
def test_combined_unary_bin_ops(self):
# x = 1 + 3 + 7
code = Bytecode([
Instr('LOAD_CONST', 1),
Instr('LOAD_CONST', 3),
Instr('BINARY_ADD'),
Instr('LOAD_CONST', 7),
Instr('BINARY_ADD'),
Instr('STORE_NAME', 'x')
])
self.check(code, Instr('LOAD_CONST', 11), Instr('STORE_NAME', 'x'))
# x = ~(~(5))
code = Bytecode([
Instr('LOAD_CONST', 5),
Instr('UNARY_INVERT'),
Instr('UNARY_INVERT'),
Instr('STORE_NAME', 'x')
])
self.check(code, Instr('LOAD_CONST', 5), Instr('STORE_NAME', 'x'))
# "events = [(0, 'call'), (1, 'line'), (-(3), 'call')]"
code = Bytecode([
Instr('LOAD_CONST', 0),
Instr('LOAD_CONST', 'call'),
Instr('BUILD_TUPLE', 2),
Instr('LOAD_CONST', 1),
Instr('LOAD_CONST', 'line'),
Instr('BUILD_TUPLE', 2),
Instr('LOAD_CONST', 3),
Instr('UNARY_NEGATIVE'),
Instr('LOAD_CONST', 'call'),
Instr('BUILD_TUPLE', 2),
Instr('BUILD_LIST', 3),
Instr('STORE_NAME', 'events')
])
self.check(code, Instr('LOAD_CONST', (0, 'call')),
Instr('LOAD_CONST', (1, 'line')),
Instr('LOAD_CONST', (-3, 'call')), Instr('BUILD_LIST', 3),
Instr('STORE_NAME', 'events'))
# 'x = (1,) + (0,) * 8'
code = Bytecode([
Instr('LOAD_CONST', 1),
Instr('BUILD_TUPLE', 1),
Instr('LOAD_CONST', 0),
Instr('BUILD_TUPLE', 1),
Instr('LOAD_CONST', 8),
Instr('BINARY_MULTIPLY'),
Instr('BINARY_ADD'),
Instr('STORE_NAME', 'x')
])
zeros = (0, ) * 8
result = (1, ) + zeros
self.check(code, Instr('LOAD_CONST', result), Instr('STORE_NAME', 'x'))
def test_from_bytecode_loop(self):
# for x in (1, 2, 3):
# if x == 2:
# break
# continue
label_loop_start = Label()
label_loop_exit = Label()
label_loop_end = Label()
code = Bytecode()
code.extend((
Instr('SETUP_LOOP', label_loop_end, lineno=1),
Instr('LOAD_CONST', (1, 2, 3), lineno=1),
Instr('GET_ITER', lineno=1),
label_loop_start,
Instr('FOR_ITER', label_loop_exit, lineno=1),
Instr('STORE_NAME', 'x', lineno=1),
Instr('LOAD_NAME', 'x', lineno=2),
Instr('LOAD_CONST', 2, lineno=2),
Instr('COMPARE_OP', Compare.EQ, lineno=2),
Instr('POP_JUMP_IF_FALSE', label_loop_start, lineno=2),
Instr('BREAK_LOOP', lineno=3),
Instr('JUMP_ABSOLUTE', label_loop_start, lineno=4),
Instr('JUMP_ABSOLUTE', label_loop_start, lineno=4),
label_loop_exit,
Instr('POP_BLOCK', lineno=4),
label_loop_end,
Instr('LOAD_CONST', None, lineno=4),
Instr('RETURN_VALUE', lineno=4),
))
blocks = ControlFlowGraph.from_bytecode(code)
expected = [[Instr('SETUP_LOOP', blocks[8], lineno=1)],
[
Instr('LOAD_CONST', (1, 2, 3), lineno=1),
Instr('GET_ITER', lineno=1)
], [Instr('FOR_ITER', blocks[7], lineno=1)],
[
Instr('STORE_NAME', 'x', lineno=1),
Instr('LOAD_NAME', 'x', lineno=2),
Instr('LOAD_CONST', 2, lineno=2),
Instr('COMPARE_OP', Compare.EQ, lineno=2),
Instr('POP_JUMP_IF_FALSE', blocks[2], lineno=2)
], [Instr('BREAK_LOOP', lineno=3)],
[Instr('JUMP_ABSOLUTE', blocks[2], lineno=4)],
[Instr('JUMP_ABSOLUTE', blocks[2], lineno=4)],
[Instr('POP_BLOCK', lineno=4)],
[
Instr('LOAD_CONST', None, lineno=4),
Instr('RETURN_VALUE', lineno=4)
]]
self.assertBlocksEqual(blocks, *expected)
def check(self, source, function=False):
ref_code = get_code(source, function=function)
code = ConcreteBytecode.from_code(ref_code).to_code()
self.assertEqual(code, ref_code)
code = Bytecode.from_code(ref_code).to_code()
self.assertEqual(code, ref_code)
bytecode = Bytecode.from_code(ref_code)
blocks = ControlFlowGraph.from_bytecode(bytecode)
code = blocks.to_bytecode().to_code()
self.assertEqual(code, ref_code)
def check(self, source, function=False):
ref_code = get_code(source, function=function)
code = ConcreteBytecode.from_code(ref_code).to_code()
self.assertEqual(code, ref_code)
code = Bytecode.from_code(ref_code).to_code()
self.assertEqual(code, ref_code)
bytecode = Bytecode.from_code(ref_code)
blocks = ControlFlowGraph.from_bytecode(bytecode)
code = blocks.to_bytecode().to_code()
self.assertEqual(code, ref_code)
def test_label(self):
code = Bytecode()
label = Label()
code.extend([Instr('LOAD_CONST', 'hello', lineno=1),
Instr('JUMP_FORWARD', label, lineno=1),
label,
Instr('POP_TOP', lineno=1)])
code = code.to_concrete_bytecode()
expected = [ConcreteInstr('LOAD_CONST', 0, lineno=1),
ConcreteInstr('JUMP_FORWARD', 0, lineno=1),
ConcreteInstr('POP_TOP', lineno=1)]
self.assertListEqual(list(code), expected)
self.assertListEqual(code.consts, ['hello'])
def _make_bytecode(self, source, template_locator):
instructions = []
symbol_table = {"write_func": io.StringIO.write}
parser_obj = parser.Parser(self._template_locator)
sequence = parser_obj.parse(self._get_chunks(source))
for item in sequence.elements:
instructions += item.make_bytecode(symbol_table)
bytecode = Bytecode(instructions +
[Instr("LOAD_CONST", None),
Instr("RETURN_VALUE")])
return bytecode.to_code()
def test_setlineno(self):
# x = 7
# y = 8
# z = 9
code = Bytecode()
code.first_lineno = 3
code.extend([
Instr("LOAD_CONST", 7),
Instr("STORE_NAME", 'x'),
SetLineno(4),
Instr("LOAD_CONST", 8),
Instr("STORE_NAME", 'y'),
SetLineno(5),
Instr("LOAD_CONST", 9),
Instr("STORE_NAME", 'z')
])
concrete = code.to_concrete_bytecode()
self.assertEqual(concrete.consts, [7, 8, 9])
self.assertEqual(concrete.names, ['x', 'y', 'z'])
code.extend([
ConcreteInstr("LOAD_CONST", 0, lineno=3),
ConcreteInstr("STORE_NAME", 0, lineno=3),
ConcreteInstr("LOAD_CONST", 1, lineno=4),
ConcreteInstr("STORE_NAME", 1, lineno=4),
ConcreteInstr("LOAD_CONST", 2, lineno=5),
ConcreteInstr("STORE_NAME", 2, lineno=5)
])
def test_label(self):
code = Bytecode()
label = Label()
code.extend([Instr('LOAD_CONST', 'hello', lineno=1),
Instr('JUMP_FORWARD', label, lineno=1),
label,
Instr('POP_TOP', lineno=1)])
code = code.to_concrete_bytecode()
expected = [ConcreteInstr('LOAD_CONST', 0, lineno=1),
ConcreteInstr('JUMP_FORWARD', 0, lineno=1),
ConcreteInstr('POP_TOP', lineno=1)]
self.assertListEqual(list(code), expected)
self.assertListEqual(code.consts, ['hello'])
def test_cellvars(self):
code = Bytecode()
code.cellvars = ['x']
code.freevars = ['y']
code.extend([Instr('LOAD_DEREF', CellVar('x'), lineno=1),
Instr('LOAD_DEREF', FreeVar('y'), lineno=1)])
concrete = code.to_concrete_bytecode()
self.assertEqual(concrete.cellvars, ['x'])
self.assertEqual(concrete.freevars, ['y'])
code.extend([ConcreteInstr("LOAD_DEREF", 0, lineno=1),
ConcreteInstr("LOAD_DEREF", 1, lineno=1)])
def test_dont_merge_constants(self):
# test two constants which are equal but have a different type
code = Bytecode()
code.extend([Instr('LOAD_CONST', 5, lineno=1),
Instr('LOAD_CONST', 5.0, lineno=1),
Instr('LOAD_CONST', -0.0, lineno=1),
Instr('LOAD_CONST', +0.0, lineno=1)])
code = code.to_concrete_bytecode()
expected = [ConcreteInstr('LOAD_CONST', 0, lineno=1),
ConcreteInstr('LOAD_CONST', 1, lineno=1),
ConcreteInstr('LOAD_CONST', 2, lineno=1),
ConcreteInstr('LOAD_CONST', 3, lineno=1)]
self.assertListEqual(list(code), expected)
self.assertListEqual(code.consts, [5, 5.0, -0.0, +0.0])
def test_dont_merge_constants(self):
# test two constants which are equal but have a different type
code = Bytecode()
code.extend([Instr('LOAD_CONST', 5, lineno=1),
Instr('LOAD_CONST', 5.0, lineno=1),
Instr('LOAD_CONST', -0.0, lineno=1),
Instr('LOAD_CONST', +0.0, lineno=1)])
code = code.to_concrete_bytecode()
expected = [ConcreteInstr('LOAD_CONST', 0, lineno=1),
ConcreteInstr('LOAD_CONST', 1, lineno=1),
ConcreteInstr('LOAD_CONST', 2, lineno=1),
ConcreteInstr('LOAD_CONST', 3, lineno=1)]
self.assertListEqual(list(code), expected)
self.assertListEqual(code.consts, [5, 5.0, -0.0, +0.0])
def optimize(self, code_obj):
bytecode = Bytecode.from_code(code_obj)
cfg = ControlFlowGraph.from_bytecode(bytecode)
self.optimize_cfg(cfg)
bytecode = cfg.to_bytecode()
code = bytecode.to_code()
return code
def test_cellvars(self):
code = Bytecode()
code.cellvars = ['x']
code.freevars = ['y']
code.extend([Instr('LOAD_DEREF', CellVar('x'), lineno=1),
Instr('LOAD_DEREF', FreeVar('y'), lineno=1)])
concrete = code.to_concrete_bytecode()
self.assertEqual(concrete.cellvars, ['x'])
self.assertEqual(concrete.freevars, ['y'])
code.extend([ConcreteInstr("LOAD_DEREF", 0, lineno=1),
ConcreteInstr("LOAD_DEREF", 1, lineno=1)])
def test_optimize_code_obj(self):
# Test optimize() method with a code object
#
# x = 3 + 5 => x = 8
noopt = Bytecode([Instr('LOAD_CONST', 3),
Instr('LOAD_CONST', 5),
Instr('BINARY_ADD'),
Instr('STORE_NAME', 'x'),
Instr('LOAD_CONST', None),
Instr('RETURN_VALUE')])
noopt = noopt.to_code()
optimizer = peephole_opt.PeepholeOptimizer()
optim = optimizer.optimize(noopt)
code = Bytecode.from_code(optim)
self.assertEqual(code,
[Instr('LOAD_CONST', 8, lineno=1),
Instr('STORE_NAME', 'x', lineno=1),
Instr('LOAD_CONST', None, lineno=1),
Instr('RETURN_VALUE', lineno=1)])
def test_from_code_load_fast(self):
code = get_code("""
def func():
x = 33
y = x
""", function=True)
code = Bytecode.from_code(code)
self.assertEqual(code,
[Instr('LOAD_CONST', 33, lineno=2),
Instr('STORE_FAST', 'x', lineno=2),
Instr('LOAD_FAST', 'x', lineno=3),
Instr('STORE_FAST', 'y', lineno=3),
Instr('LOAD_CONST', None, lineno=3),
Instr('RETURN_VALUE', lineno=3)])
def test_extended_jump(self):
NOP = bytes((opcode.opmap['NOP'],))
class BigInstr(ConcreteInstr):
def __init__(self, size):
super().__init__('NOP')
self._size = size
def copy(self):
return self
def assemble(self):
return NOP * self._size
# (invalid) code using jumps > 0xffff to test extended arg
label = Label()
nb_nop = 2**16
code = Bytecode([Instr("JUMP_ABSOLUTE", label),
BigInstr(nb_nop),
label])
code_obj = code.to_code()
expected = (b'\x90\x01\x00q\x06\x00' + NOP * nb_nop)
self.assertEqual(code_obj.co_code, expected)
def test_from_code_freevars(self):
ns = {}
exec(textwrap.dedent('''
def create_func():
x = 1
def func():
return x
return func
func = create_func()
'''), ns, ns)
code = ns['func'].__code__
bytecode = Bytecode.from_code(code)
self.assertEqual(bytecode,
[Instr('LOAD_DEREF', FreeVar('x'), lineno=5),
Instr('RETURN_VALUE', lineno=5)])
def test_to_code(self):
code = Bytecode()
code.first_lineno = 50
code.extend([Instr("LOAD_NAME", "print"),
Instr("LOAD_CONST", "%s"),
Instr("LOAD_GLOBAL", "a"),
Instr("BINARY_MODULO"),
Instr("CALL_FUNCTION", 1),
Instr("RETURN_VALUE")])
co = code.to_code()
# hopefully this is obvious from inspection? :-)
self.assertEqual(co.co_stacksize, 3)
co = code.to_code(stacksize=42)
self.assertEqual(co.co_stacksize, 42)
def test_from_code(self):
code = get_code("""
if test:
x = 1
else:
x = 2
""")
bytecode = Bytecode.from_code(code)
label_else = Label()
label_exit = Label()
self.assertEqual(bytecode,
[Instr('LOAD_NAME', 'test', lineno=1),
Instr('POP_JUMP_IF_FALSE', label_else, lineno=1),
Instr('LOAD_CONST', 1, lineno=2),
Instr('STORE_NAME', 'x', lineno=2),
Instr('JUMP_FORWARD', label_exit, lineno=2),
label_else,
Instr('LOAD_CONST', 2, lineno=4),
Instr('STORE_NAME', 'x', lineno=4),
label_exit,
Instr('LOAD_CONST', None, lineno=4),
Instr('RETURN_VALUE', lineno=4)])
def test_setlineno(self):
# x = 7
# y = 8
# z = 9
code = Bytecode()
code.first_lineno = 3
code.extend([Instr("LOAD_CONST", 7),
Instr("STORE_NAME", 'x'),
SetLineno(4),
Instr("LOAD_CONST", 8),
Instr("STORE_NAME", 'y'),
SetLineno(5),
Instr("LOAD_CONST", 9),
Instr("STORE_NAME", 'z')])
concrete = code.to_concrete_bytecode()
self.assertEqual(concrete.consts, [7, 8, 9])
self.assertEqual(concrete.names, ['x', 'y', 'z'])
code.extend([ConcreteInstr("LOAD_CONST", 0, lineno=3),
ConcreteInstr("STORE_NAME", 0, lineno=3),
ConcreteInstr("LOAD_CONST", 1, lineno=4),
ConcreteInstr("STORE_NAME", 1, lineno=4),
ConcreteInstr("LOAD_CONST", 2, lineno=5),
ConcreteInstr("STORE_NAME", 2, lineno=5)])
def test_label3(self):
"""
CPython generates useless EXTENDED_ARG 0 in some cases. We need to
properly track them as otherwise we can end up with broken offset for
jumps.
"""
source = """
def func(x):
if x == 1:
return x + 0
elif x == 2:
return x + 1
elif x == 3:
return x + 2
elif x == 4:
return x + 3
elif x == 5:
return x + 4
elif x == 6:
return x + 5
elif x == 7:
return x + 6
elif x == 8:
return x + 7
elif x == 9:
return x + 8
elif x == 10:
return x + 9
elif x == 11:
return x + 10
elif x == 12:
return x + 11
elif x == 13:
return x + 12
elif x == 14:
return x + 13
elif x == 15:
return x + 14
elif x == 16:
return x + 15
elif x == 17:
return x + 16
return -1
"""
code = get_code(source, function=True)
bcode = Bytecode.from_code(code)
concrete = bcode.to_concrete_bytecode()
self.assertIsInstance(concrete, ConcreteBytecode)
# Ensure that we do not generate broken code
loc = {}
exec(textwrap.dedent(source), loc)
func = loc['func']
func.__code__ = bcode.to_code()
for i, x in enumerate(range(1, 18)):
self.assertEqual(func(x), x + i)
self.assertEqual(func(18), -1)
# Ensure that we properly round trip in such cases
self.assertEqual(ConcreteBytecode.from_code(code).to_code().co_code,
code.co_code)
def test_extreme_compute_jumps_convergence(self):
"""Test of compute_jumps() requiring absurd number of passes.
NOTE: This test also serves to demonstrate that there is no worst
case: the number of passes can be unlimited (or, actually, limited by
the size of the provided code).
This is an extension of test_compute_jumps_convergence. Instead of
two jumps, where the earlier gets extended after the latter, we
instead generate a series of many jumps. Each pass of compute_jumps()
extends one more instruction, which in turn causes the one behind it
to be extended on the next pass.
"""
if not WORDCODE:
return
# N: the number of unextended instructions that can be squeezed into a
# set of bytes adressable by the arg of an unextended instruction.
# The answer is "128", but here's how we arrive at it (and it also
# hints at how to make this work for pre-WORDCODE).
max_unextended_offset = 1 << 8
unextended_branch_instr_size = 2
N = max_unextended_offset // unextended_branch_instr_size
nop = 'UNARY_POSITIVE' # don't use NOP, dis.stack_effect will raise
# The number of jumps will be equal to the number of labels. The
# number of passes of compute_jumps() required will be one greater
# than this.
labels = [Label() for x in range(0, 3 * N)]
code = Bytecode()
code.extend(Instr('JUMP_FORWARD', labels[len(labels) - x - 1])
for x in range(0, len(labels)))
end_of_jumps = len(code)
code.extend(Instr(nop) for x in range(0, N))
# Now insert the labels. The first is N instructions (i.e. 256
# bytes) after the last jump. Then they proceed to earlier positions
# 4 bytes at a time. While the targets are in the range of the nop
# instructions, 4 bytes is two instructions. When the targets are in
# the range of JUMP_FORWARD instructions we have to allow for the fact
# that the instructions will have been extended to four bytes each, so
# working backwards 4 bytes per label means just one instruction per
# label.
offset = end_of_jumps + N
for l in range(0, len(labels)):
code.insert(offset, labels[l])
if offset <= end_of_jumps:
offset -= 1
else:
offset -= 2
code.insert(0, Instr("LOAD_CONST", 0))
del end_of_jumps
code.append(Instr('RETURN_VALUE'))
code.to_code(compute_jumps_passes=(len(labels) + 1))
def test_compute_jumps_convergence(self):
# Consider the following sequence of instructions:
#
# JUMP_ABSOLUTE Label1
# JUMP_ABSOLUTE Label2
# ...126 instructions...
# Label1: Offset 254 on first pass, 256 second pass
# NOP
# ... many more instructions ...
# Label2: Offset > 256 on first pass
#
# On first pass of compute_jumps(), Label2 will be at address 254, so
# that value encodes into the single byte arg of JUMP_ABSOLUTE.
#
# On second pass compute_jumps() the instr at Label1 will have offset
# of 256 so will also be given an EXTENDED_ARG.
#
# Thus we need to make an additional pass. This test only verifies
# case where 2 passes is insufficient but three is enough.
if not WORDCODE:
# Could be done pre-WORDCODE, but that requires 2**16 bytes of
# code.
return
# Create code from comment above.
code = Bytecode()
label1 = Label()
label2 = Label()
nop = 'UNARY_POSITIVE' # don't use NOP, dis.stack_effect will raise
code.append(Instr('JUMP_ABSOLUTE', label1))
code.append(Instr('JUMP_ABSOLUTE', label2))
for x in range(4, 254, 2):
code.append(Instr(nop))
code.append(label1)
code.append(Instr(nop))
for x in range(256, 300, 2):
code.append(Instr(nop))
code.append(label2)
code.append(Instr(nop))
# This should pass by default.
code.to_code()
# Try with max of two passes: it should raise
with self.assertRaises(RuntimeError):
code.to_code(compute_jumps_passes=2)
