def test_eq(self):
code = ConcreteBytecode()
self.assertFalse(code == 1)
for name, val in (
("names", ["a"]),
("varnames", ["a"]),
("consts", [1]),
("argcount", 1),
("flags", CompilerFlags(CompilerFlags.GENERATOR)),
("first_lineno", 10),
("filename", "xxxx.py"),
("name", "__x"),
("docstring", "x-x-x"),
("cellvars", [CellVar("x")]),
("freevars", [FreeVar("x")]),
):
c = ConcreteBytecode()
setattr(c, name, val)
# For obscure reasons using assertNotEqual here fail
self.assertFalse(code == c)
c = ConcreteBytecode()
c.consts = [1]
code.consts = [1]
c.append(ConcreteInstr("LOAD_CONST", 0))
self.assertFalse(code == c)
def test_invalid_types(self):
code = ConcreteBytecode()
code.append(Label())
with self.assertRaises(ValueError):
list(code)
with self.assertRaises(ValueError):
ConcreteBytecode([Label()])
def test_extended_arg(self):
# Create a code object from arbitrary bytecode
co_code = (b'\x90\x12\x904\x90\xabd\xcd'
if WORDCODE else b'\x904\x12d\xcd\xab')
code = get_code('x=1')
code = types.CodeType(code.co_argcount, code.co_kwonlyargcount,
code.co_nlocals, code.co_stacksize,
code.co_flags, co_code, code.co_consts,
code.co_names, code.co_varnames,
code.co_filename, code.co_name,
code.co_firstlineno, code.co_lnotab,
code.co_freevars, code.co_cellvars)
# without EXTENDED_ARG opcode
bytecode = ConcreteBytecode.from_code(code)
self.assertListEqual(
list(bytecode),
[ConcreteInstr("LOAD_CONST", 0x1234abcd, lineno=1)])
# with EXTENDED_ARG opcode
bytecode = ConcreteBytecode.from_code(code, extended_arg=True)
if WORDCODE:
expected = [
ConcreteInstr('EXTENDED_ARG', 0x12, lineno=1),
ConcreteInstr('EXTENDED_ARG', 0x34, lineno=1),
ConcreteInstr('EXTENDED_ARG', 0xab, lineno=1),
ConcreteInstr('LOAD_CONST', 0xcd, lineno=1)
]
else:
expected = [
ConcreteInstr('EXTENDED_ARG', 0x1234, lineno=1),
ConcreteInstr('LOAD_CONST', 0xabcd, lineno=1)
]
self.assertListEqual(list(bytecode), expected)
def test_extended_arg(self):
# Create a code object from arbitrary bytecode
co_code = b'\x904\x12d\xcd\xab'
code = get_code('x=1')
code = types.CodeType(code.co_argcount,
code.co_kwonlyargcount,
code.co_nlocals,
code.co_stacksize,
code.co_flags,
co_code,
code.co_consts,
code.co_names,
code.co_varnames,
code.co_filename,
code.co_name,
code.co_firstlineno,
code.co_lnotab,
code.co_freevars,
code.co_cellvars)
# without EXTENDED_ARG opcode
bytecode = ConcreteBytecode.from_code(code)
self.assertListEqual(list(bytecode),
[ConcreteInstr("LOAD_CONST", 0x1234abcd, lineno=1)])
# with EXTENDED_ARG opcode
bytecode = ConcreteBytecode.from_code(code, extended_arg=True)
self.assertListEqual(list(bytecode),
[ConcreteInstr('EXTENDED_ARG', 0x1234, lineno=1),
ConcreteInstr('LOAD_CONST', 0xabcd, lineno=1)])
def test_invalid_types(self):
code = ConcreteBytecode()
code.append(Label())
with self.assertRaises(ValueError):
list(code)
with self.assertRaises(ValueError):
ConcreteBytecode([Label()])
def test_extended_arg_make_function(self):
code_obj = get_code('''
def foo(x: int, y: int):
pass
''')
# without EXTENDED_ARG
concrete = ConcreteBytecode.from_code(code_obj)
func_code = concrete.consts[1]
self.assertEqual(concrete.names, ['int', 'foo'])
self.assertEqual(concrete.consts, [('x', 'y'), func_code, 'foo', None])
if WORDCODE:
expected = [
ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 0, lineno=1),
ConcreteInstr("BUILD_CONST_KEY_MAP", 2, lineno=1),
ConcreteInstr("LOAD_CONST", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 2, lineno=1),
ConcreteInstr("MAKE_FUNCTION", 4, lineno=1),
ConcreteInstr("STORE_NAME", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 3, lineno=1),
ConcreteInstr("RETURN_VALUE", lineno=1)
]
else:
expected = [
ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 2, lineno=1),
ConcreteInstr("MAKE_FUNCTION", 3 << 16, lineno=1),
ConcreteInstr("STORE_NAME", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 3, lineno=1),
ConcreteInstr("RETURN_VALUE", lineno=1)
]
self.assertListEqual(list(concrete), expected)
# with EXTENDED_ARG
concrete = ConcreteBytecode.from_code(code_obj, extended_arg=True)
func_code = concrete.consts[1]
self.assertEqual(concrete.names, ['int', 'foo'])
self.assertEqual(concrete.consts, [('x', 'y'), func_code, 'foo', None])
if not WORDCODE:
expected = [
ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 2, lineno=1),
ConcreteInstr("EXTENDED_ARG", 3, lineno=1),
ConcreteInstr("MAKE_FUNCTION", 0, lineno=1),
ConcreteInstr("STORE_NAME", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 3, lineno=1),
ConcreteInstr("RETURN_VALUE", lineno=1)
]
self.assertListEqual(list(concrete), expected)
def test_negative_lnotab(self):
# x = 7
# y = 8
concrete = ConcreteBytecode()
concrete.consts = [7, 8]
concrete.names = ['x', 'y']
concrete.first_lineno = 5
concrete.extend([ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("STORE_NAME", 0),
# line number goes backward!
SetLineno(2),
ConcreteInstr("LOAD_CONST", 1),
ConcreteInstr("STORE_NAME", 1)])
if sys.version_info >= (3, 6):
code = concrete.to_code()
expected = b'd\x00Z\x00d\x01Z\x01'
self.assertEqual(code.co_code, expected)
self.assertEqual(code.co_firstlineno, 5)
self.assertEqual(code.co_lnotab, b'\x04\xfd')
else:
with self.assertRaises(ValueError) as cm:
code = concrete.to_code()
self.assertEqual(str(cm.exception),
"negative line number delta is not supported "
"on Python < 3.6")
def test_cellvar_freevar(self):
concrete = ConcreteBytecode()
concrete.cellvars = ["cell"]
concrete.freevars = ["free"]
concrete.append(ConcreteInstr("LOAD_DEREF", 0))
concrete.append(ConcreteInstr("LOAD_DEREF", 1))
code = concrete.to_code()
concrete = ConcreteBytecode.from_code(code)
self.assertEqual(concrete.cellvars, ["cell"])
self.assertEqual(concrete.freevars, ["free"])
self.assertEqual(
list(concrete),
[
ConcreteInstr("LOAD_DEREF", 0, lineno=1),
ConcreteInstr("LOAD_DEREF", 1, lineno=1),
],
)
bytecode = concrete.to_bytecode()
self.assertEqual(bytecode.cellvars, ["cell"])
self.assertEqual(
list(bytecode),
[
Instr("LOAD_DEREF", CellVar("cell"), lineno=1),
Instr("LOAD_DEREF", FreeVar("free"), lineno=1),
],
)
def test_legalize(self):
concrete = ConcreteBytecode()
concrete.first_lineno = 3
concrete.consts = [7, 8, 9]
concrete.names = ["x", "y", "z"]
concrete.extend(
[
ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("STORE_NAME", 0),
ConcreteInstr("LOAD_CONST", 1, lineno=4),
ConcreteInstr("STORE_NAME", 1),
SetLineno(5),
ConcreteInstr("LOAD_CONST", 2, lineno=6),
ConcreteInstr("STORE_NAME", 2),
]
)
concrete.legalize()
self.assertListEqual(
list(concrete),
[
ConcreteInstr("LOAD_CONST", 0, lineno=3),
ConcreteInstr("STORE_NAME", 0, lineno=3),
ConcreteInstr("LOAD_CONST", 1, lineno=4),
ConcreteInstr("STORE_NAME", 1, lineno=4),
ConcreteInstr("LOAD_CONST", 2, lineno=5),
ConcreteInstr("STORE_NAME", 2, lineno=5),
],
)
def test_flag_inference(self):
# Check no loss of non-infered flags
code = ControlFlowGraph()
code.flags |= (CompilerFlags.NEWLOCALS | CompilerFlags.VARARGS
| CompilerFlags.VARKEYWORDS | CompilerFlags.NESTED
| CompilerFlags.FUTURE_GENERATOR_STOP)
code.update_flags()
for f in (CompilerFlags.NEWLOCALS, CompilerFlags.VARARGS,
CompilerFlags.VARKEYWORDS, CompilerFlags.NESTED,
CompilerFlags.NOFREE, CompilerFlags.OPTIMIZED,
CompilerFlags.FUTURE_GENERATOR_STOP):
self.assertTrue(bool(code.flags & f))
# Infer optimized and nofree
code = Bytecode()
flags = infer_flags(code)
self.assertTrue(bool(flags & CompilerFlags.OPTIMIZED))
self.assertTrue(bool(flags & CompilerFlags.NOFREE))
code.append(ConcreteInstr('STORE_NAME', 1))
flags = infer_flags(code)
self.assertFalse(bool(flags & CompilerFlags.OPTIMIZED))
self.assertTrue(bool(flags & CompilerFlags.NOFREE))
code.append(ConcreteInstr('STORE_DEREF', 2))
code.update_flags()
self.assertFalse(bool(code.flags & CompilerFlags.OPTIMIZED))
self.assertFalse(bool(code.flags & CompilerFlags.NOFREE))
# Infer generator
code = ConcreteBytecode()
code.append(ConcreteInstr('YIELD_VALUE'))
for is_async, expected in ((False, CompilerFlags.GENERATOR),
(True, CompilerFlags.ASYNC_GENERATOR)):
self.assertTrue(bool(infer_flags(code, is_async) & expected))
# Infer coroutine
code = ConcreteBytecode()
code.append(ConcreteInstr('GET_AWAITABLE'))
iter_flags = CompilerFlags(CompilerFlags.ITERABLE_COROUTINE)
for f, expected in ((CompilerFlags(0), True), (iter_flags, False)):
code.flags = f
self.assertEqual(bool(infer_flags(code) & CompilerFlags.COROUTINE),
expected)
# Test check flag sanity
code.append(ConcreteInstr('YIELD_VALUE'))
code.flags = CompilerFlags(CompilerFlags.GENERATOR
| CompilerFlags.COROUTINE)
infer_flags(code, is_async=True) # Just want to be sure it pases
with self.assertRaises(ValueError):
code.update_flags()
with self.assertRaises(ValueError):
infer_flags(None)
def test_extended_arg_make_function(self):
code_obj = get_code('''
def foo(x: int, y: int):
pass
''')
# without EXTENDED_ARG
concrete = ConcreteBytecode.from_code(code_obj)
func_code = concrete.consts[1]
self.assertEqual(concrete.names, ['int', 'foo'])
self.assertEqual(concrete.consts, [('x', 'y'), func_code, 'foo', None])
if WORDCODE:
expected = [ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 0, lineno=1),
ConcreteInstr("BUILD_CONST_KEY_MAP", 2, lineno=1),
ConcreteInstr("LOAD_CONST", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 2, lineno=1),
ConcreteInstr("MAKE_FUNCTION", 4, lineno=1),
ConcreteInstr("STORE_NAME", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 3, lineno=1),
ConcreteInstr("RETURN_VALUE", lineno=1)]
else:
expected = [ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 2, lineno=1),
ConcreteInstr("MAKE_FUNCTION", 3 << 16, lineno=1),
ConcreteInstr("STORE_NAME", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 3, lineno=1),
ConcreteInstr("RETURN_VALUE", lineno=1)]
self.assertListEqual(list(concrete), expected)
# with EXTENDED_ARG
concrete = ConcreteBytecode.from_code(code_obj, extended_arg=True)
func_code = concrete.consts[1]
self.assertEqual(concrete.names, ['int', 'foo'])
self.assertEqual(concrete.consts, [('x', 'y'), func_code, 'foo', None])
if not WORDCODE:
expected = [ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_NAME", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 2, lineno=1),
ConcreteInstr("EXTENDED_ARG", 3, lineno=1),
ConcreteInstr("MAKE_FUNCTION", 0, lineno=1),
ConcreteInstr("STORE_NAME", 1, lineno=1),
ConcreteInstr("LOAD_CONST", 3, lineno=1),
ConcreteInstr("RETURN_VALUE", lineno=1)]
self.assertListEqual(list(concrete), expected)
def test_load_classderef(self):
concrete = ConcreteBytecode()
concrete.cellvars = ['__class__']
concrete.freevars = ['__class__']
concrete.extend([
ConcreteInstr('LOAD_CLASSDEREF', 1),
ConcreteInstr('STORE_DEREF', 1)
])
bytecode = concrete.to_bytecode()
self.assertEqual(bytecode.freevars, ['__class__'])
self.assertEqual(bytecode.cellvars, ['__class__'])
self.assertEqual(list(bytecode), [
Instr('LOAD_CLASSDEREF', FreeVar('__class__'), lineno=1),
Instr('STORE_DEREF', FreeVar('__class__'), lineno=1)
])
concrete = bytecode.to_concrete_bytecode()
self.assertEqual(concrete.freevars, ['__class__'])
self.assertEqual(concrete.cellvars, ['__class__'])
self.assertEqual(list(concrete), [
ConcreteInstr('LOAD_CLASSDEREF', 1, lineno=1),
ConcreteInstr('STORE_DEREF', 1, lineno=1)
])
code = concrete.to_code()
self.assertEqual(code.co_freevars, ('__class__', ))
self.assertEqual(code.co_cellvars, ('__class__', ))
self.assertEqual(
code.co_code,
b'\x94\x01\x89\x01' if WORDCODE else b'\x94\x01\x00\x89\x01\x00')
def test_to_bytecode_consts(self):
# x = -0.0
# x = +0.0
#
# code optimized by the CPython 3.6 peephole optimizer which emits
# duplicated constants (0.0 is twice in consts).
code = ConcreteBytecode()
code.consts = [0.0, None, -0.0, 0.0]
code.names = ["x", "y"]
code.extend([
ConcreteInstr("LOAD_CONST", 2, lineno=1),
ConcreteInstr("STORE_NAME", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 3, lineno=2),
ConcreteInstr("STORE_NAME", 1, lineno=2),
ConcreteInstr("LOAD_CONST", 1, lineno=2),
ConcreteInstr("RETURN_VALUE", lineno=2),
])
code = code.to_bytecode().to_concrete_bytecode()
# the conversion changes the constant order: the order comes from
# the order of LOAD_CONST instructions
self.assertEqual(code.consts, [-0.0, 0.0, None])
code.names = ["x", "y"]
self.assertListEqual(
list(code),
[
ConcreteInstr("LOAD_CONST", 0, lineno=1),
ConcreteInstr("STORE_NAME", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 1, lineno=2),
ConcreteInstr("STORE_NAME", 1, lineno=2),
ConcreteInstr("LOAD_CONST", 2, lineno=2),
ConcreteInstr("RETURN_VALUE", lineno=2),
],
)
def test_setlineno(self):
# x = 7
# y = 8
# z = 9
concrete = ConcreteBytecode()
concrete.consts = [7, 8, 9]
concrete.names = ["x", "y", "z"]
concrete.first_lineno = 3
concrete.extend(
[
ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("STORE_NAME", 0),
SetLineno(4),
ConcreteInstr("LOAD_CONST", 1),
ConcreteInstr("STORE_NAME", 1),
SetLineno(5),
ConcreteInstr("LOAD_CONST", 2),
ConcreteInstr("STORE_NAME", 2),
]
)
code = concrete.to_bytecode()
self.assertEqual(
code,
[
Instr("LOAD_CONST", 7, lineno=3),
Instr("STORE_NAME", "x", lineno=3),
Instr("LOAD_CONST", 8, lineno=4),
Instr("STORE_NAME", "y", lineno=4),
Instr("LOAD_CONST", 9, lineno=5),
Instr("STORE_NAME", "z", lineno=5),
],
)
def test_negative_lnotab(self):
# x = 7
# y = 8
concrete = ConcreteBytecode()
concrete.consts = [7, 8]
concrete.names = ['x', 'y']
concrete.first_lineno = 5
concrete.extend([ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("STORE_NAME", 0),
# line number goes backward!
SetLineno(2),
ConcreteInstr("LOAD_CONST", 1),
ConcreteInstr("STORE_NAME", 1)])
if sys.version_info >= (3, 6):
code = concrete.to_code()
expected = (b'd\x00\x00'
b'Z\x00\x00'
b'd\x01\x00'
b'Z\x01\x00')
self.assertEqual(code.co_code, expected)
self.assertEqual(code.co_firstlineno, 5)
self.assertEqual(code.co_lnotab, b'\x06\xfd')
else:
with self.assertRaises(ValueError) as cm:
code = concrete.to_code()
self.assertEqual(str(cm.exception),
"negative line number delta is not supported "
"on Python < 3.6")
def test_to_code_lnotab(self):
# We use an actual function for the simple case to
# ensure we get lnotab right
def f():
#
#
x = 7 # noqa
y = 8 # noqa
z = 9 # noqa
fl = f.__code__.co_firstlineno
concrete = ConcreteBytecode()
concrete.consts = [None, 7, 8, 9]
concrete.varnames = ["x", "y", "z"]
concrete.first_lineno = fl
concrete.extend([
SetLineno(fl + 3),
ConcreteInstr("LOAD_CONST", 1),
ConcreteInstr("STORE_FAST", 0),
SetLineno(fl + 4),
ConcreteInstr("LOAD_CONST", 2),
ConcreteInstr("STORE_FAST", 1),
SetLineno(fl + 5),
ConcreteInstr("LOAD_CONST", 3),
ConcreteInstr("STORE_FAST", 2),
ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("RETURN_VALUE"),
])
code = concrete.to_code()
self.assertEqual(code.co_code, f.__code__.co_code)
self.assertEqual(code.co_lnotab, f.__code__.co_lnotab)
if sys.version_info >= (3, 10):
self.assertEqual(code.co_linetable, f.__code__.co_linetable)
def test_to_code_lnotab(self):
# x = 7
# y = 8
# z = 9
concrete = ConcreteBytecode()
concrete.consts = [7, 8, 9]
concrete.names = ['x', 'y', 'z']
concrete.first_lineno = 3
concrete.extend([
ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("STORE_NAME", 0),
SetLineno(4),
ConcreteInstr("LOAD_CONST", 1),
ConcreteInstr("STORE_NAME", 1),
SetLineno(5),
ConcreteInstr("LOAD_CONST", 2),
ConcreteInstr("STORE_NAME", 2)
])
code = concrete.to_code()
if WORDCODE:
expected = b'd\x00Z\x00d\x01Z\x01d\x02Z\x02'
else:
expected = (b'd\x00\x00'
b'Z\x00\x00'
b'd\x01\x00'
b'Z\x01\x00'
b'd\x02\x00'
b'Z\x02\x00')
self.assertEqual(code.co_code, expected)
self.assertEqual(code.co_firstlineno, 3)
self.assertEqual(
code.co_lnotab,
b'\x04\x01\x04\x01' if WORDCODE else b'\x06\x01\x06\x01')
def test_to_code_lnotab(self):
# x = 7
# y = 8
# z = 9
concrete = ConcreteBytecode()
concrete.consts = [7, 8, 9]
concrete.names = ["x", "y", "z"]
concrete.first_lineno = 3
concrete.extend([
ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("STORE_NAME", 0),
SetLineno(4),
ConcreteInstr("LOAD_CONST", 1),
ConcreteInstr("STORE_NAME", 1),
SetLineno(5),
ConcreteInstr("LOAD_CONST", 2),
ConcreteInstr("STORE_NAME", 2),
])
code = concrete.to_code()
expected = (b"d\x00\x00"
b"Z\x00\x00"
b"d\x01\x00"
b"Z\x01\x00"
b"d\x02\x00"
b"Z\x02\x00")
self.assertEqual(code.co_code, expected)
self.assertEqual(code.co_firstlineno, 3)
self.assertEqual(code.co_lnotab, b"\x06\x01\x06\x01")
def instrument_code_recursive(
self,
code: CodeType,
parent_code_object_id: Optional[int] = None) -> CodeType:
"""Instrument the given Code Object recursively.
:param code: The code object to be instrumented.
:param parent_code_object_id: Internal id of the code object to which this code object belongs
(can be None if `code` is the highest node, i.e. the module node).
:return The instrumented code object.
"""
# The original bytecode should match the disassembly, so EXTENDED_ARG is included
# original_cfg = CFG.from_bytecode(ConcreteBytecode.from_code(code, extended_arg=False).to_bytecode())
original_cfg = CFG.from_bytecode(
ConcreteBytecode.from_code(code, extended_arg=False).to_bytecode())
original_cdg = ControlDependenceGraph.compute(original_cfg)
cfg = CFG.from_bytecode(Bytecode.from_code(code))
code_object_id = self._tracer.register_code_object(
CodeObjectMetaData(code.co_filename, code, parent_code_object_id,
original_cfg, cfg, original_cdg))
assert cfg.entry_node is not None, "Entry node cannot be None."
module_entry = False
if not parent_code_object_id:
# This is the module entry, in which we want to import the ExecutionTracer
self._instrument_import(cfg)
module_entry = True
self._instrument_cfg(cfg, original_cfg, code_object_id, module_entry)
return self._instrument_inner_code_objects(
cfg.bytecode_cfg().to_code(), code_object_id)
def make_bytecode(self, symbol_table):
code = [
Instr("LOAD_CONST", symbol_table["write_func"]),
Instr("LOAD_NAME", "_output"),
Instr("LOAD_NAME", "str")
]
compiled_expr = compile(self.variable_name,
filename="<none>",
mode="eval")
concrete_bytecode = ConcreteBytecode.from_code(compiled_expr)
inner = concrete_bytecode.to_bytecode()
# The compiler drops a return statement at the end of the
# expression, which we want to strip off so that we can use
# the result
inner.pop()
code += inner
code += [
Instr("CALL_FUNCTION", 1),
Instr("CALL_FUNCTION", 2),
Instr("POP_TOP")
]
return code
def test_async_gen_no_flag_is_async_True(self):
# Test inference when we request an async function
# Force coroutine
code = ConcreteBytecode()
code.update_flags(is_async=True)
self.assertTrue(bool(code.flags & CompilerFlags.COROUTINE))
# Infer coroutine or async generator
for i, expected in (
("YIELD_VALUE", CompilerFlags.ASYNC_GENERATOR),
("YIELD_FROM", CompilerFlags.COROUTINE),
):
code = ConcreteBytecode()
code.append(ConcreteInstr(i))
code.update_flags(is_async=True)
self.assertTrue(bool(code.flags & expected))
def test_extended_arg(self):
# Create a code object from arbitrary bytecode
co_code = b"\x90\x12\x904\x90\xabd\xcd" if WORDCODE else b"\x904\x12d\xcd\xab"
code = get_code("x=1")
args = ((code.co_argcount, ) if sys.version_info < (3, 8) else
(code.co_argcount, code.co_posonlyargcount))
args += (
code.co_kwonlyargcount,
code.co_nlocals,
code.co_stacksize,
code.co_flags,
co_code,
code.co_consts,
code.co_names,
code.co_varnames,
code.co_filename,
code.co_name,
code.co_firstlineno,
code.co_lnotab,
code.co_freevars,
code.co_cellvars,
)
code = types.CodeType(*args)
# without EXTENDED_ARG opcode
bytecode = ConcreteBytecode.from_code(code)
self.assertListEqual(
list(bytecode),
[ConcreteInstr("LOAD_CONST", 0x1234ABCD, lineno=1)])
# with EXTENDED_ARG opcode
bytecode = ConcreteBytecode.from_code(code, extended_arg=True)
if WORDCODE:
expected = [
ConcreteInstr("EXTENDED_ARG", 0x12, lineno=1),
ConcreteInstr("EXTENDED_ARG", 0x34, lineno=1),
ConcreteInstr("EXTENDED_ARG", 0xAB, lineno=1),
ConcreteInstr("LOAD_CONST", 0xCD, lineno=1),
]
else:
expected = [
ConcreteInstr("EXTENDED_ARG", 0x1234, lineno=1),
ConcreteInstr("LOAD_CONST", 0xABCD, lineno=1),
]
self.assertListEqual(list(bytecode), expected)
def test_to_code_lnotab(self):
# x = 7
# y = 8
# z = 9
concrete = ConcreteBytecode()
concrete.consts = [7, 8, 9]
concrete.names = ['x', 'y', 'z']
concrete.first_lineno = 3
concrete.extend([ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("STORE_NAME", 0),
SetLineno(4),
ConcreteInstr("LOAD_CONST", 1),
ConcreteInstr("STORE_NAME", 1),
SetLineno(5),
ConcreteInstr("LOAD_CONST", 2),
ConcreteInstr("STORE_NAME", 2)])
code = concrete.to_code()
if WORDCODE:
expected = b'd\x00Z\x00d\x01Z\x01d\x02Z\x02'
else:
expected = (b'd\x00\x00'
b'Z\x00\x00'
b'd\x01\x00'
b'Z\x01\x00'
b'd\x02\x00'
b'Z\x02\x00')
self.assertEqual(code.co_code, expected)
self.assertEqual(code.co_firstlineno, 3)
self.assertEqual(
code.co_lnotab,
b'\x04\x01\x04\x01' if WORDCODE else b'\x06\x01\x06\x01')
def test_to_bytecode_consts(self):
# x = -0.0
# x = +0.0
#
# code optimized by the CPython 3.6 peephole optimizer which emits
# duplicated constants (0.0 is twice in consts).
code = ConcreteBytecode()
code.consts = [0.0, None, -0.0, 0.0]
code.names = ['x', 'y']
code.extend([ConcreteInstr('LOAD_CONST', 2, lineno=1),
ConcreteInstr('STORE_NAME', 0, lineno=1),
ConcreteInstr('LOAD_CONST', 3, lineno=2),
ConcreteInstr('STORE_NAME', 1, lineno=2),
ConcreteInstr('LOAD_CONST', 1, lineno=2),
ConcreteInstr('RETURN_VALUE', lineno=2)])
code = code.to_bytecode().to_concrete_bytecode()
# the conversion changes the constant order: the order comes from
# the order of LOAD_CONST instructions
self.assertEqual(code.consts, [-0.0, 0.0, None])
code.names = ['x', 'y']
self.assertListEqual(list(code),
[ConcreteInstr('LOAD_CONST', 0, lineno=1),
ConcreteInstr('STORE_NAME', 0, lineno=1),
ConcreteInstr('LOAD_CONST', 1, lineno=2),
ConcreteInstr('STORE_NAME', 1, lineno=2),
ConcreteInstr('LOAD_CONST', 2, lineno=2),
ConcreteInstr('RETURN_VALUE', lineno=2)])
def test_load_classderef(self):
concrete = ConcreteBytecode()
concrete.cellvars = ['__class__']
concrete.freevars = ['__class__']
concrete.extend([ConcreteInstr('LOAD_CLASSDEREF', 1),
ConcreteInstr('STORE_DEREF', 1)])
bytecode = concrete.to_bytecode()
self.assertEqual(bytecode.freevars, ['__class__'])
self.assertEqual(bytecode.cellvars, ['__class__'])
self.assertEqual(list(bytecode),
[Instr('LOAD_CLASSDEREF', FreeVar('__class__'),
lineno=1),
Instr('STORE_DEREF', FreeVar('__class__'), lineno=1)]
)
concrete = bytecode.to_concrete_bytecode()
self.assertEqual(concrete.freevars, ['__class__'])
self.assertEqual(concrete.cellvars, ['__class__'])
self.assertEqual(list(concrete),
[ConcreteInstr('LOAD_CLASSDEREF', 1, lineno=1),
ConcreteInstr('STORE_DEREF', 1, lineno=1)])
code = concrete.to_code()
self.assertEqual(code.co_freevars, ('__class__',))
self.assertEqual(code.co_cellvars, ('__class__',))
self.assertEqual(
code.co_code,
b'\x94\x01\x89\x01' if WORDCODE else b'\x94\x01\x00\x89\x01\x00')
def test_setlineno(self):
# x = 7
# y = 8
# z = 9
concrete = ConcreteBytecode()
concrete.consts = [7, 8, 9]
concrete.names = ['x', 'y', 'z']
concrete.first_lineno = 3
concrete.extend([ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("STORE_NAME", 0),
SetLineno(4),
ConcreteInstr("LOAD_CONST", 1),
ConcreteInstr("STORE_NAME", 1),
SetLineno(5),
ConcreteInstr("LOAD_CONST", 2),
ConcreteInstr("STORE_NAME", 2)])
code = concrete.to_bytecode()
self.assertEqual(code,
[Instr("LOAD_CONST", 7, lineno=3),
Instr("STORE_NAME", 'x', lineno=3),
Instr("LOAD_CONST", 8, lineno=4),
Instr("STORE_NAME", 'y', lineno=4),
Instr("LOAD_CONST", 9, lineno=5),
Instr("STORE_NAME", 'z', lineno=5)])
def test_extended_lnotab(self):
# x = 7
# y = 8
concrete = ConcreteBytecode([
ConcreteInstr("LOAD_CONST", 0),
SetLineno(1 + 128),
ConcreteInstr("STORE_NAME", 0),
# line number goes backward!
SetLineno(1 + 129),
ConcreteInstr("LOAD_CONST", 1),
SetLineno(1),
ConcreteInstr("STORE_NAME", 1),
])
concrete.consts = [7, 8]
concrete.names = ["x", "y"]
concrete.first_lineno = 1
if sys.version_info >= (3, 6):
code = concrete.to_code()
expected = b"d\x00Z\x00d\x01Z\x01"
self.assertEqual(code.co_code, expected)
self.assertEqual(code.co_firstlineno, 1)
self.assertEqual(code.co_lnotab,
b"\x00\x7f\x02\x01\x02\x01\x00\x80\x02\xff")
else:
with self.assertRaises(ValueError) as cm:
code = concrete.to_code()
self.assertEqual(
str(cm.exception),
"negative line number delta is not supported "
"on Python < 3.6",
)
def test_extended_arg_unpack_ex(self):
def test():
p = [1, 2, 3, 4, 5, 6]
q, r, *s, t = p
return q, r, s, t
test.__code__ = ConcreteBytecode.from_code(test.__code__, extended_arg=True).to_code()
self.assertEqual(test.__code__.co_stacksize, 6)
self.assertEqual(test(), (1, 2, [3, 4, 5], 6))
def test_packing_lines(self):
from bytecode.tests.long_lines_example import long_lines
import dis
line_starts = list(dis.findlinestarts(long_lines.__code__))
concrete = ConcreteBytecode.from_code(long_lines.__code__)
as_code = concrete.to_code()
self.assertEqual(line_starts, list(dis.findlinestarts(as_code)))
def test_attr(self):
code_obj = get_code("x = 5")
code = ConcreteBytecode.from_code(code_obj)
self.assertEqual(code.consts, [5, None])
self.assertEqual(code.names, ['x'])
self.assertEqual(code.varnames, [])
self.assertEqual(code.freevars, [])
self.assertListEqual(list(code),
[ConcreteInstr('LOAD_CONST', 0, lineno=1),
ConcreteInstr('STORE_NAME', 0, lineno=1),
ConcreteInstr('LOAD_CONST', 1, lineno=1),
ConcreteInstr('RETURN_VALUE', lineno=1)])
def test_attr(self):
code_obj = get_code("x = 5")
code = ConcreteBytecode.from_code(code_obj)
self.assertEqual(code.consts, [5, None])
self.assertEqual(code.names, ['x'])
self.assertEqual(code.varnames, [])
self.assertEqual(code.freevars, [])
self.assertListEqual(list(code),
[ConcreteInstr('LOAD_CONST', 0, lineno=1),
ConcreteInstr('STORE_NAME', 0, lineno=1),
ConcreteInstr('LOAD_CONST', 1, lineno=1),
ConcreteInstr('RETURN_VALUE', lineno=1)])
def test_freevar(self):
concrete = ConcreteBytecode()
concrete.freevars = ["x"]
concrete.append(ConcreteInstr("LOAD_DEREF", 0))
code = concrete.to_code()
concrete = ConcreteBytecode.from_code(code)
self.assertEqual(concrete.cellvars, [])
self.assertEqual(concrete.freevars, ["x"])
self.assertEqual(list(concrete), [ConcreteInstr("LOAD_DEREF", 0, lineno=1)])
bytecode = concrete.to_bytecode()
self.assertEqual(bytecode.cellvars, [])
self.assertEqual(list(bytecode), [Instr("LOAD_DEREF", FreeVar("x"), lineno=1)])
def test_extended_lnotab2(self):
# x = 7
# 200 blank lines
# y = 8
base_code = compile("x = 7" + "\n" * 200 + "y = 8", "", "exec")
concrete = ConcreteBytecode(
[
ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("STORE_NAME", 0),
SetLineno(201),
ConcreteInstr("LOAD_CONST", 1),
ConcreteInstr("STORE_NAME", 1),
ConcreteInstr("LOAD_CONST", 2),
ConcreteInstr("RETURN_VALUE"),
]
)
concrete.consts = [None, 7, 8]
concrete.names = ["x", "y"]
concrete.first_lineno = 1
code = concrete.to_code()
self.assertEqual(code.co_code, base_code.co_code)
self.assertEqual(code.co_firstlineno, base_code.co_firstlineno)
self.assertEqual(code.co_lnotab, base_code.co_lnotab)
if sys.version_info >= (3, 10):
self.assertEqual(code.co_linetable, base_code.co_linetable)
def test_extended_lnotab(self):
# x = 7
# 200 blank lines
# y = 8
concrete = ConcreteBytecode(
[
ConcreteInstr("LOAD_CONST", 0),
SetLineno(1 + 128),
ConcreteInstr("STORE_NAME", 0),
# line number goes backward!
SetLineno(1 + 129),
ConcreteInstr("LOAD_CONST", 1),
SetLineno(1),
ConcreteInstr("STORE_NAME", 1),
]
)
concrete.consts = [7, 8]
concrete.names = ["x", "y"]
concrete.first_lineno = 1
code = concrete.to_code()
expected = b"d\x00Z\x00d\x01Z\x01"
self.assertEqual(code.co_code, expected)
self.assertEqual(code.co_firstlineno, 1)
self.assertEqual(code.co_lnotab, b"\x02\x7f\x00\x01\x02\x01\x02\x80\x00\xff")
def check(self, source, function=False):
ref_code = get_code(source, function=function)
code = ConcreteBytecode.from_code(ref_code).to_code()
self.assertEqual(code, ref_code)
code = Bytecode.from_code(ref_code).to_code()
self.assertEqual(code, ref_code)
bytecode = Bytecode.from_code(ref_code)
blocks = ControlFlowGraph.from_bytecode(bytecode)
code = blocks.to_bytecode().to_code()
self.assertEqual(code, ref_code)
def check(self, source, function=False):
ref_code = get_code(source, function=function)
code = ConcreteBytecode.from_code(ref_code).to_code()
self.assertEqual(code, ref_code)
code = Bytecode.from_code(ref_code).to_code()
self.assertEqual(code, ref_code)
bytecode = Bytecode.from_code(ref_code)
blocks = ControlFlowGraph.from_bytecode(bytecode)
code = blocks.to_bytecode().to_code()
self.assertEqual(code, ref_code)
def test_copy(self):
concrete = ConcreteBytecode()
concrete.first_lineno = 3
concrete.consts = [7, 8, 9]
concrete.names = ["x", "y", "z"]
concrete.extend([
ConcreteInstr("LOAD_CONST", 0),
ConcreteInstr("STORE_NAME", 0),
SetLineno(4),
ConcreteInstr("LOAD_CONST", 1),
ConcreteInstr("STORE_NAME", 1),
SetLineno(5),
ConcreteInstr("LOAD_CONST", 2),
ConcreteInstr("STORE_NAME", 2),
])
self.assertEqual(concrete, concrete.copy())
def test_cellvar(self):
concrete = ConcreteBytecode()
concrete.cellvars = ['x']
concrete.append(ConcreteInstr('LOAD_DEREF', 0))
code = concrete.to_code()
concrete = ConcreteBytecode.from_code(code)
self.assertEqual(concrete.cellvars, ['x'])
self.assertEqual(concrete.freevars, [])
self.assertEqual(list(concrete),
[ConcreteInstr('LOAD_DEREF', 0, lineno=1)])
bytecode = concrete.to_bytecode()
self.assertEqual(bytecode.cellvars, ['x'])
self.assertEqual(list(bytecode),
[Instr('LOAD_DEREF', CellVar('x'), lineno=1)])
def test_extended_arg(self):
# Create a code object from arbitrary bytecode
co_code = b"%c4\x12d\xcd\xab" % EXTENDED_ARG
code = get_code("x=1")
args = (
code.co_argcount,
code.co_nlocals,
code.co_stacksize,
code.co_flags,
co_code,
code.co_consts,
code.co_names,
code.co_varnames,
code.co_filename,
code.co_name,
code.co_firstlineno,
code.co_lnotab,
code.co_freevars,
code.co_cellvars,
)
code = types.CodeType(*args)
# without EXTENDED_ARG opcode
bytecode = ConcreteBytecode.from_code(code)
self.assertListEqual(
list(bytecode),
[ConcreteInstr("LOAD_CONST", 0x1234ABCD, lineno=1)])
# with EXTENDED_ARG opcode
bytecode = ConcreteBytecode.from_code(code, extended_arg=True)
expected = [
ConcreteInstr("EXTENDED_ARG", 0x1234, lineno=1),
ConcreteInstr("LOAD_CONST", 0xABCD, lineno=1),
]
self.assertListEqual(list(bytecode), expected)
def test_attr(self):
code_obj = get_code("x = 5")
code = ConcreteBytecode.from_code(code_obj)
self.assertEqual(code.consts, [5, None])
self.assertEqual(code.names, ["x"])
self.assertEqual(code.varnames, [])
self.assertEqual(code.freevars, [])
self.assertListEqual(
list(code),
[
ConcreteInstr("LOAD_CONST", 0, lineno=1),
ConcreteInstr("STORE_NAME", 0, lineno=1),
ConcreteInstr("LOAD_CONST", 1, lineno=1),
ConcreteInstr("RETURN_VALUE", lineno=1),
],
)
def test_freevar(self):
concrete = ConcreteBytecode()
concrete.freevars = ['x']
concrete.append(ConcreteInstr('LOAD_DEREF', 0))
code = concrete.to_code()
concrete = ConcreteBytecode.from_code(code)
self.assertEqual(concrete.cellvars, [])
self.assertEqual(concrete.freevars, ['x'])
self.assertEqual(list(concrete),
[ConcreteInstr('LOAD_DEREF', 0, lineno=1)])
bytecode = concrete.to_bytecode()
self.assertEqual(bytecode.cellvars, [])
self.assertEqual(list(bytecode),
[Instr('LOAD_DEREF', FreeVar('x'), lineno=1)])
def test_explicit_stacksize(self):
# Passing stacksize=... to ConcreteBytecode.to_code should result in a
# code object with the specified stacksize. We pass some silly values
# and assert that they are honored.
code_obj = get_code("print('%s' % (a,b,c))")
original_stacksize = code_obj.co_stacksize
concrete = ConcreteBytecode.from_code(code_obj)
# First with something bigger than necessary.
explicit_stacksize = original_stacksize + 42
new_code_obj = concrete.to_code(stacksize=explicit_stacksize)
self.assertEqual(new_code_obj.co_stacksize, explicit_stacksize)
# Then with something bogus. We probably don't want to advertise this
# in the documentation. If this fails then decide if it's for good
# reason, and remove if so.
explicit_stacksize = 0
new_code_obj = concrete.to_code(stacksize=explicit_stacksize)
self.assertEqual(new_code_obj.co_stacksize, explicit_stacksize)
def test_label3(self):
"""
CPython generates useless EXTENDED_ARG 0 in some cases. We need to
properly track them as otherwise we can end up with broken offset for
jumps.
"""
source = """
def func(x):
if x == 1:
return x + 0
elif x == 2:
return x + 1
elif x == 3:
return x + 2
elif x == 4:
return x + 3
elif x == 5:
return x + 4
elif x == 6:
return x + 5
elif x == 7:
return x + 6
elif x == 8:
return x + 7
elif x == 9:
return x + 8
elif x == 10:
return x + 9
elif x == 11:
return x + 10
elif x == 12:
return x + 11
elif x == 13:
return x + 12
elif x == 14:
return x + 13
elif x == 15:
return x + 14
elif x == 16:
return x + 15
elif x == 17:
return x + 16
return -1
"""
code = get_code(source, function=True)
bcode = Bytecode.from_code(code)
concrete = bcode.to_concrete_bytecode()
self.assertIsInstance(concrete, ConcreteBytecode)
# Ensure that we do not generate broken code
loc = {}
exec(textwrap.dedent(source), loc)
func = loc['func']
func.__code__ = bcode.to_code()
for i, x in enumerate(range(1, 18)):
self.assertEqual(func(x), x + i)
self.assertEqual(func(18), -1)
# Ensure that we properly round trip in such cases
self.assertEqual(ConcreteBytecode.from_code(code).to_code().co_code,
code.co_code)
