def test_eq(self): code = ConcreteBytecode() self.assertFalse(code == 1) for name, val in ( ("names", ["a"]), ("varnames", ["a"]), ("consts", [1]), ("argcount", 1), ("flags", CompilerFlags(CompilerFlags.GENERATOR)), ("first_lineno", 10), ("filename", "xxxx.py"), ("name", "__x"), ("docstring", "x-x-x"), ("cellvars", [CellVar("x")]), ("freevars", [FreeVar("x")]), ): c = ConcreteBytecode() setattr(c, name, val) # For obscure reasons using assertNotEqual here fail self.assertFalse(code == c) c = ConcreteBytecode() c.consts = [1] code.consts = [1] c.append(ConcreteInstr("LOAD_CONST", 0)) self.assertFalse(code == c)
def test_invalid_types(self): code = ConcreteBytecode() code.append(Label()) with self.assertRaises(ValueError): list(code) with self.assertRaises(ValueError): ConcreteBytecode([Label()])
def test_extended_arg(self): # Create a code object from arbitrary bytecode co_code = (b'\x90\x12\x904\x90\xabd\xcd' if WORDCODE else b'\x904\x12d\xcd\xab') code = get_code('x=1') code = types.CodeType(code.co_argcount, code.co_kwonlyargcount, code.co_nlocals, code.co_stacksize, code.co_flags, co_code, code.co_consts, code.co_names, code.co_varnames, code.co_filename, code.co_name, code.co_firstlineno, code.co_lnotab, code.co_freevars, code.co_cellvars) # without EXTENDED_ARG opcode bytecode = ConcreteBytecode.from_code(code) self.assertListEqual( list(bytecode), [ConcreteInstr("LOAD_CONST", 0x1234abcd, lineno=1)]) # with EXTENDED_ARG opcode bytecode = ConcreteBytecode.from_code(code, extended_arg=True) if WORDCODE: expected = [ ConcreteInstr('EXTENDED_ARG', 0x12, lineno=1), ConcreteInstr('EXTENDED_ARG', 0x34, lineno=1), ConcreteInstr('EXTENDED_ARG', 0xab, lineno=1), ConcreteInstr('LOAD_CONST', 0xcd, lineno=1) ] else: expected = [ ConcreteInstr('EXTENDED_ARG', 0x1234, lineno=1), ConcreteInstr('LOAD_CONST', 0xabcd, lineno=1) ] self.assertListEqual(list(bytecode), expected)
def test_extended_arg(self): # Create a code object from arbitrary bytecode co_code = b'\x904\x12d\xcd\xab' code = get_code('x=1') code = types.CodeType(code.co_argcount, code.co_kwonlyargcount, code.co_nlocals, code.co_stacksize, code.co_flags, co_code, code.co_consts, code.co_names, code.co_varnames, code.co_filename, code.co_name, code.co_firstlineno, code.co_lnotab, code.co_freevars, code.co_cellvars) # without EXTENDED_ARG opcode bytecode = ConcreteBytecode.from_code(code) self.assertListEqual(list(bytecode), [ConcreteInstr("LOAD_CONST", 0x1234abcd, lineno=1)]) # with EXTENDED_ARG opcode bytecode = ConcreteBytecode.from_code(code, extended_arg=True) self.assertListEqual(list(bytecode), [ConcreteInstr('EXTENDED_ARG', 0x1234, lineno=1), ConcreteInstr('LOAD_CONST', 0xabcd, lineno=1)])
def test_extended_arg_make_function(self): code_obj = get_code(''' def foo(x: int, y: int): pass ''') # without EXTENDED_ARG concrete = ConcreteBytecode.from_code(code_obj) func_code = concrete.consts[1] self.assertEqual(concrete.names, ['int', 'foo']) self.assertEqual(concrete.consts, [('x', 'y'), func_code, 'foo', None]) if WORDCODE: expected = [ ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 0, lineno=1), ConcreteInstr("BUILD_CONST_KEY_MAP", 2, lineno=1), ConcreteInstr("LOAD_CONST", 1, lineno=1), ConcreteInstr("LOAD_CONST", 2, lineno=1), ConcreteInstr("MAKE_FUNCTION", 4, lineno=1), ConcreteInstr("STORE_NAME", 1, lineno=1), ConcreteInstr("LOAD_CONST", 3, lineno=1), ConcreteInstr("RETURN_VALUE", lineno=1) ] else: expected = [ ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 0, lineno=1), ConcreteInstr("LOAD_CONST", 1, lineno=1), ConcreteInstr("LOAD_CONST", 2, lineno=1), ConcreteInstr("MAKE_FUNCTION", 3 << 16, lineno=1), ConcreteInstr("STORE_NAME", 1, lineno=1), ConcreteInstr("LOAD_CONST", 3, lineno=1), ConcreteInstr("RETURN_VALUE", lineno=1) ] self.assertListEqual(list(concrete), expected) # with EXTENDED_ARG concrete = ConcreteBytecode.from_code(code_obj, extended_arg=True) func_code = concrete.consts[1] self.assertEqual(concrete.names, ['int', 'foo']) self.assertEqual(concrete.consts, [('x', 'y'), func_code, 'foo', None]) if not WORDCODE: expected = [ ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 0, lineno=1), ConcreteInstr("LOAD_CONST", 1, lineno=1), ConcreteInstr("LOAD_CONST", 2, lineno=1), ConcreteInstr("EXTENDED_ARG", 3, lineno=1), ConcreteInstr("MAKE_FUNCTION", 0, lineno=1), ConcreteInstr("STORE_NAME", 1, lineno=1), ConcreteInstr("LOAD_CONST", 3, lineno=1), ConcreteInstr("RETURN_VALUE", lineno=1) ] self.assertListEqual(list(concrete), expected)
def test_negative_lnotab(self): # x = 7 # y = 8 concrete = ConcreteBytecode() concrete.consts = [7, 8] concrete.names = ['x', 'y'] concrete.first_lineno = 5 concrete.extend([ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), # line number goes backward! SetLineno(2), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1)]) if sys.version_info >= (3, 6): code = concrete.to_code() expected = b'd\x00Z\x00d\x01Z\x01' self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 5) self.assertEqual(code.co_lnotab, b'\x04\xfd') else: with self.assertRaises(ValueError) as cm: code = concrete.to_code() self.assertEqual(str(cm.exception), "negative line number delta is not supported " "on Python < 3.6")
def test_cellvar_freevar(self): concrete = ConcreteBytecode() concrete.cellvars = ["cell"] concrete.freevars = ["free"] concrete.append(ConcreteInstr("LOAD_DEREF", 0)) concrete.append(ConcreteInstr("LOAD_DEREF", 1)) code = concrete.to_code() concrete = ConcreteBytecode.from_code(code) self.assertEqual(concrete.cellvars, ["cell"]) self.assertEqual(concrete.freevars, ["free"]) self.assertEqual( list(concrete), [ ConcreteInstr("LOAD_DEREF", 0, lineno=1), ConcreteInstr("LOAD_DEREF", 1, lineno=1), ], ) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.cellvars, ["cell"]) self.assertEqual( list(bytecode), [ Instr("LOAD_DEREF", CellVar("cell"), lineno=1), Instr("LOAD_DEREF", FreeVar("free"), lineno=1), ], )
def test_legalize(self): concrete = ConcreteBytecode() concrete.first_lineno = 3 concrete.consts = [7, 8, 9] concrete.names = ["x", "y", "z"] concrete.extend( [ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), ConcreteInstr("LOAD_CONST", 1, lineno=4), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2, lineno=6), ConcreteInstr("STORE_NAME", 2), ] ) concrete.legalize() self.assertListEqual( list(concrete), [ ConcreteInstr("LOAD_CONST", 0, lineno=3), ConcreteInstr("STORE_NAME", 0, lineno=3), ConcreteInstr("LOAD_CONST", 1, lineno=4), ConcreteInstr("STORE_NAME", 1, lineno=4), ConcreteInstr("LOAD_CONST", 2, lineno=5), ConcreteInstr("STORE_NAME", 2, lineno=5), ], )
def test_flag_inference(self): # Check no loss of non-infered flags code = ControlFlowGraph() code.flags |= (CompilerFlags.NEWLOCALS | CompilerFlags.VARARGS | CompilerFlags.VARKEYWORDS | CompilerFlags.NESTED | CompilerFlags.FUTURE_GENERATOR_STOP) code.update_flags() for f in (CompilerFlags.NEWLOCALS, CompilerFlags.VARARGS, CompilerFlags.VARKEYWORDS, CompilerFlags.NESTED, CompilerFlags.NOFREE, CompilerFlags.OPTIMIZED, CompilerFlags.FUTURE_GENERATOR_STOP): self.assertTrue(bool(code.flags & f)) # Infer optimized and nofree code = Bytecode() flags = infer_flags(code) self.assertTrue(bool(flags & CompilerFlags.OPTIMIZED)) self.assertTrue(bool(flags & CompilerFlags.NOFREE)) code.append(ConcreteInstr('STORE_NAME', 1)) flags = infer_flags(code) self.assertFalse(bool(flags & CompilerFlags.OPTIMIZED)) self.assertTrue(bool(flags & CompilerFlags.NOFREE)) code.append(ConcreteInstr('STORE_DEREF', 2)) code.update_flags() self.assertFalse(bool(code.flags & CompilerFlags.OPTIMIZED)) self.assertFalse(bool(code.flags & CompilerFlags.NOFREE)) # Infer generator code = ConcreteBytecode() code.append(ConcreteInstr('YIELD_VALUE')) for is_async, expected in ((False, CompilerFlags.GENERATOR), (True, CompilerFlags.ASYNC_GENERATOR)): self.assertTrue(bool(infer_flags(code, is_async) & expected)) # Infer coroutine code = ConcreteBytecode() code.append(ConcreteInstr('GET_AWAITABLE')) iter_flags = CompilerFlags(CompilerFlags.ITERABLE_COROUTINE) for f, expected in ((CompilerFlags(0), True), (iter_flags, False)): code.flags = f self.assertEqual(bool(infer_flags(code) & CompilerFlags.COROUTINE), expected) # Test check flag sanity code.append(ConcreteInstr('YIELD_VALUE')) code.flags = CompilerFlags(CompilerFlags.GENERATOR | CompilerFlags.COROUTINE) infer_flags(code, is_async=True) # Just want to be sure it pases with self.assertRaises(ValueError): code.update_flags() with self.assertRaises(ValueError): infer_flags(None)
def test_extended_arg_make_function(self): code_obj = get_code(''' def foo(x: int, y: int): pass ''') # without EXTENDED_ARG concrete = ConcreteBytecode.from_code(code_obj) func_code = concrete.consts[1] self.assertEqual(concrete.names, ['int', 'foo']) self.assertEqual(concrete.consts, [('x', 'y'), func_code, 'foo', None]) if WORDCODE: expected = [ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 0, lineno=1), ConcreteInstr("BUILD_CONST_KEY_MAP", 2, lineno=1), ConcreteInstr("LOAD_CONST", 1, lineno=1), ConcreteInstr("LOAD_CONST", 2, lineno=1), ConcreteInstr("MAKE_FUNCTION", 4, lineno=1), ConcreteInstr("STORE_NAME", 1, lineno=1), ConcreteInstr("LOAD_CONST", 3, lineno=1), ConcreteInstr("RETURN_VALUE", lineno=1)] else: expected = [ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 0, lineno=1), ConcreteInstr("LOAD_CONST", 1, lineno=1), ConcreteInstr("LOAD_CONST", 2, lineno=1), ConcreteInstr("MAKE_FUNCTION", 3 << 16, lineno=1), ConcreteInstr("STORE_NAME", 1, lineno=1), ConcreteInstr("LOAD_CONST", 3, lineno=1), ConcreteInstr("RETURN_VALUE", lineno=1)] self.assertListEqual(list(concrete), expected) # with EXTENDED_ARG concrete = ConcreteBytecode.from_code(code_obj, extended_arg=True) func_code = concrete.consts[1] self.assertEqual(concrete.names, ['int', 'foo']) self.assertEqual(concrete.consts, [('x', 'y'), func_code, 'foo', None]) if not WORDCODE: expected = [ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 0, lineno=1), ConcreteInstr("LOAD_CONST", 1, lineno=1), ConcreteInstr("LOAD_CONST", 2, lineno=1), ConcreteInstr("EXTENDED_ARG", 3, lineno=1), ConcreteInstr("MAKE_FUNCTION", 0, lineno=1), ConcreteInstr("STORE_NAME", 1, lineno=1), ConcreteInstr("LOAD_CONST", 3, lineno=1), ConcreteInstr("RETURN_VALUE", lineno=1)] self.assertListEqual(list(concrete), expected)
def test_load_classderef(self): concrete = ConcreteBytecode() concrete.cellvars = ['__class__'] concrete.freevars = ['__class__'] concrete.extend([ ConcreteInstr('LOAD_CLASSDEREF', 1), ConcreteInstr('STORE_DEREF', 1) ]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.freevars, ['__class__']) self.assertEqual(bytecode.cellvars, ['__class__']) self.assertEqual(list(bytecode), [ Instr('LOAD_CLASSDEREF', FreeVar('__class__'), lineno=1), Instr('STORE_DEREF', FreeVar('__class__'), lineno=1) ]) concrete = bytecode.to_concrete_bytecode() self.assertEqual(concrete.freevars, ['__class__']) self.assertEqual(concrete.cellvars, ['__class__']) self.assertEqual(list(concrete), [ ConcreteInstr('LOAD_CLASSDEREF', 1, lineno=1), ConcreteInstr('STORE_DEREF', 1, lineno=1) ]) code = concrete.to_code() self.assertEqual(code.co_freevars, ('__class__', )) self.assertEqual(code.co_cellvars, ('__class__', )) self.assertEqual( code.co_code, b'\x94\x01\x89\x01' if WORDCODE else b'\x94\x01\x00\x89\x01\x00')
def test_to_bytecode_consts(self): # x = -0.0 # x = +0.0 # # code optimized by the CPython 3.6 peephole optimizer which emits # duplicated constants (0.0 is twice in consts). code = ConcreteBytecode() code.consts = [0.0, None, -0.0, 0.0] code.names = ["x", "y"] code.extend([ ConcreteInstr("LOAD_CONST", 2, lineno=1), ConcreteInstr("STORE_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 3, lineno=2), ConcreteInstr("STORE_NAME", 1, lineno=2), ConcreteInstr("LOAD_CONST", 1, lineno=2), ConcreteInstr("RETURN_VALUE", lineno=2), ]) code = code.to_bytecode().to_concrete_bytecode() # the conversion changes the constant order: the order comes from # the order of LOAD_CONST instructions self.assertEqual(code.consts, [-0.0, 0.0, None]) code.names = ["x", "y"] self.assertListEqual( list(code), [ ConcreteInstr("LOAD_CONST", 0, lineno=1), ConcreteInstr("STORE_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 1, lineno=2), ConcreteInstr("STORE_NAME", 1, lineno=2), ConcreteInstr("LOAD_CONST", 2, lineno=2), ConcreteInstr("RETURN_VALUE", lineno=2), ], )
def test_setlineno(self): # x = 7 # y = 8 # z = 9 concrete = ConcreteBytecode() concrete.consts = [7, 8, 9] concrete.names = ["x", "y", "z"] concrete.first_lineno = 3 concrete.extend( [ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(4), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_NAME", 2), ] ) code = concrete.to_bytecode() self.assertEqual( code, [ Instr("LOAD_CONST", 7, lineno=3), Instr("STORE_NAME", "x", lineno=3), Instr("LOAD_CONST", 8, lineno=4), Instr("STORE_NAME", "y", lineno=4), Instr("LOAD_CONST", 9, lineno=5), Instr("STORE_NAME", "z", lineno=5), ], )
def test_negative_lnotab(self): # x = 7 # y = 8 concrete = ConcreteBytecode() concrete.consts = [7, 8] concrete.names = ['x', 'y'] concrete.first_lineno = 5 concrete.extend([ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), # line number goes backward! SetLineno(2), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1)]) if sys.version_info >= (3, 6): code = concrete.to_code() expected = (b'd\x00\x00' b'Z\x00\x00' b'd\x01\x00' b'Z\x01\x00') self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 5) self.assertEqual(code.co_lnotab, b'\x06\xfd') else: with self.assertRaises(ValueError) as cm: code = concrete.to_code() self.assertEqual(str(cm.exception), "negative line number delta is not supported " "on Python < 3.6")
def test_to_code_lnotab(self): # We use an actual function for the simple case to # ensure we get lnotab right def f(): # # x = 7 # noqa y = 8 # noqa z = 9 # noqa fl = f.__code__.co_firstlineno concrete = ConcreteBytecode() concrete.consts = [None, 7, 8, 9] concrete.varnames = ["x", "y", "z"] concrete.first_lineno = fl concrete.extend([ SetLineno(fl + 3), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_FAST", 0), SetLineno(fl + 4), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_FAST", 1), SetLineno(fl + 5), ConcreteInstr("LOAD_CONST", 3), ConcreteInstr("STORE_FAST", 2), ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("RETURN_VALUE"), ]) code = concrete.to_code() self.assertEqual(code.co_code, f.__code__.co_code) self.assertEqual(code.co_lnotab, f.__code__.co_lnotab) if sys.version_info >= (3, 10): self.assertEqual(code.co_linetable, f.__code__.co_linetable)
def test_to_code_lnotab(self): # x = 7 # y = 8 # z = 9 concrete = ConcreteBytecode() concrete.consts = [7, 8, 9] concrete.names = ['x', 'y', 'z'] concrete.first_lineno = 3 concrete.extend([ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(4), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_NAME", 2) ]) code = concrete.to_code() if WORDCODE: expected = b'd\x00Z\x00d\x01Z\x01d\x02Z\x02' else: expected = (b'd\x00\x00' b'Z\x00\x00' b'd\x01\x00' b'Z\x01\x00' b'd\x02\x00' b'Z\x02\x00') self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 3) self.assertEqual( code.co_lnotab, b'\x04\x01\x04\x01' if WORDCODE else b'\x06\x01\x06\x01')
def test_to_code_lnotab(self): # x = 7 # y = 8 # z = 9 concrete = ConcreteBytecode() concrete.consts = [7, 8, 9] concrete.names = ["x", "y", "z"] concrete.first_lineno = 3 concrete.extend([ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(4), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_NAME", 2), ]) code = concrete.to_code() expected = (b"d\x00\x00" b"Z\x00\x00" b"d\x01\x00" b"Z\x01\x00" b"d\x02\x00" b"Z\x02\x00") self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 3) self.assertEqual(code.co_lnotab, b"\x06\x01\x06\x01")
def instrument_code_recursive( self, code: CodeType, parent_code_object_id: Optional[int] = None) -> CodeType: """Instrument the given Code Object recursively. :param code: The code object to be instrumented. :param parent_code_object_id: Internal id of the code object to which this code object belongs (can be None if `code` is the highest node, i.e. the module node). :return The instrumented code object. """ # The original bytecode should match the disassembly, so EXTENDED_ARG is included # original_cfg = CFG.from_bytecode(ConcreteBytecode.from_code(code, extended_arg=False).to_bytecode()) original_cfg = CFG.from_bytecode( ConcreteBytecode.from_code(code, extended_arg=False).to_bytecode()) original_cdg = ControlDependenceGraph.compute(original_cfg) cfg = CFG.from_bytecode(Bytecode.from_code(code)) code_object_id = self._tracer.register_code_object( CodeObjectMetaData(code.co_filename, code, parent_code_object_id, original_cfg, cfg, original_cdg)) assert cfg.entry_node is not None, "Entry node cannot be None." module_entry = False if not parent_code_object_id: # This is the module entry, in which we want to import the ExecutionTracer self._instrument_import(cfg) module_entry = True self._instrument_cfg(cfg, original_cfg, code_object_id, module_entry) return self._instrument_inner_code_objects( cfg.bytecode_cfg().to_code(), code_object_id)
def make_bytecode(self, symbol_table): code = [ Instr("LOAD_CONST", symbol_table["write_func"]), Instr("LOAD_NAME", "_output"), Instr("LOAD_NAME", "str") ] compiled_expr = compile(self.variable_name, filename="<none>", mode="eval") concrete_bytecode = ConcreteBytecode.from_code(compiled_expr) inner = concrete_bytecode.to_bytecode() # The compiler drops a return statement at the end of the # expression, which we want to strip off so that we can use # the result inner.pop() code += inner code += [ Instr("CALL_FUNCTION", 1), Instr("CALL_FUNCTION", 2), Instr("POP_TOP") ] return code
def test_async_gen_no_flag_is_async_True(self): # Test inference when we request an async function # Force coroutine code = ConcreteBytecode() code.update_flags(is_async=True) self.assertTrue(bool(code.flags & CompilerFlags.COROUTINE)) # Infer coroutine or async generator for i, expected in ( ("YIELD_VALUE", CompilerFlags.ASYNC_GENERATOR), ("YIELD_FROM", CompilerFlags.COROUTINE), ): code = ConcreteBytecode() code.append(ConcreteInstr(i)) code.update_flags(is_async=True) self.assertTrue(bool(code.flags & expected))
def test_extended_arg(self): # Create a code object from arbitrary bytecode co_code = b"\x90\x12\x904\x90\xabd\xcd" if WORDCODE else b"\x904\x12d\xcd\xab" code = get_code("x=1") args = ((code.co_argcount, ) if sys.version_info < (3, 8) else (code.co_argcount, code.co_posonlyargcount)) args += ( code.co_kwonlyargcount, code.co_nlocals, code.co_stacksize, code.co_flags, co_code, code.co_consts, code.co_names, code.co_varnames, code.co_filename, code.co_name, code.co_firstlineno, code.co_lnotab, code.co_freevars, code.co_cellvars, ) code = types.CodeType(*args) # without EXTENDED_ARG opcode bytecode = ConcreteBytecode.from_code(code) self.assertListEqual( list(bytecode), [ConcreteInstr("LOAD_CONST", 0x1234ABCD, lineno=1)]) # with EXTENDED_ARG opcode bytecode = ConcreteBytecode.from_code(code, extended_arg=True) if WORDCODE: expected = [ ConcreteInstr("EXTENDED_ARG", 0x12, lineno=1), ConcreteInstr("EXTENDED_ARG", 0x34, lineno=1), ConcreteInstr("EXTENDED_ARG", 0xAB, lineno=1), ConcreteInstr("LOAD_CONST", 0xCD, lineno=1), ] else: expected = [ ConcreteInstr("EXTENDED_ARG", 0x1234, lineno=1), ConcreteInstr("LOAD_CONST", 0xABCD, lineno=1), ] self.assertListEqual(list(bytecode), expected)
def test_to_code_lnotab(self): # x = 7 # y = 8 # z = 9 concrete = ConcreteBytecode() concrete.consts = [7, 8, 9] concrete.names = ['x', 'y', 'z'] concrete.first_lineno = 3 concrete.extend([ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(4), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_NAME", 2)]) code = concrete.to_code() if WORDCODE: expected = b'd\x00Z\x00d\x01Z\x01d\x02Z\x02' else: expected = (b'd\x00\x00' b'Z\x00\x00' b'd\x01\x00' b'Z\x01\x00' b'd\x02\x00' b'Z\x02\x00') self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 3) self.assertEqual( code.co_lnotab, b'\x04\x01\x04\x01' if WORDCODE else b'\x06\x01\x06\x01')
def test_to_bytecode_consts(self): # x = -0.0 # x = +0.0 # # code optimized by the CPython 3.6 peephole optimizer which emits # duplicated constants (0.0 is twice in consts). code = ConcreteBytecode() code.consts = [0.0, None, -0.0, 0.0] code.names = ['x', 'y'] code.extend([ConcreteInstr('LOAD_CONST', 2, lineno=1), ConcreteInstr('STORE_NAME', 0, lineno=1), ConcreteInstr('LOAD_CONST', 3, lineno=2), ConcreteInstr('STORE_NAME', 1, lineno=2), ConcreteInstr('LOAD_CONST', 1, lineno=2), ConcreteInstr('RETURN_VALUE', lineno=2)]) code = code.to_bytecode().to_concrete_bytecode() # the conversion changes the constant order: the order comes from # the order of LOAD_CONST instructions self.assertEqual(code.consts, [-0.0, 0.0, None]) code.names = ['x', 'y'] self.assertListEqual(list(code), [ConcreteInstr('LOAD_CONST', 0, lineno=1), ConcreteInstr('STORE_NAME', 0, lineno=1), ConcreteInstr('LOAD_CONST', 1, lineno=2), ConcreteInstr('STORE_NAME', 1, lineno=2), ConcreteInstr('LOAD_CONST', 2, lineno=2), ConcreteInstr('RETURN_VALUE', lineno=2)])
def test_load_classderef(self): concrete = ConcreteBytecode() concrete.cellvars = ['__class__'] concrete.freevars = ['__class__'] concrete.extend([ConcreteInstr('LOAD_CLASSDEREF', 1), ConcreteInstr('STORE_DEREF', 1)]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.freevars, ['__class__']) self.assertEqual(bytecode.cellvars, ['__class__']) self.assertEqual(list(bytecode), [Instr('LOAD_CLASSDEREF', FreeVar('__class__'), lineno=1), Instr('STORE_DEREF', FreeVar('__class__'), lineno=1)] ) concrete = bytecode.to_concrete_bytecode() self.assertEqual(concrete.freevars, ['__class__']) self.assertEqual(concrete.cellvars, ['__class__']) self.assertEqual(list(concrete), [ConcreteInstr('LOAD_CLASSDEREF', 1, lineno=1), ConcreteInstr('STORE_DEREF', 1, lineno=1)]) code = concrete.to_code() self.assertEqual(code.co_freevars, ('__class__',)) self.assertEqual(code.co_cellvars, ('__class__',)) self.assertEqual( code.co_code, b'\x94\x01\x89\x01' if WORDCODE else b'\x94\x01\x00\x89\x01\x00')
def test_setlineno(self): # x = 7 # y = 8 # z = 9 concrete = ConcreteBytecode() concrete.consts = [7, 8, 9] concrete.names = ['x', 'y', 'z'] concrete.first_lineno = 3 concrete.extend([ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(4), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_NAME", 2)]) code = concrete.to_bytecode() self.assertEqual(code, [Instr("LOAD_CONST", 7, lineno=3), Instr("STORE_NAME", 'x', lineno=3), Instr("LOAD_CONST", 8, lineno=4), Instr("STORE_NAME", 'y', lineno=4), Instr("LOAD_CONST", 9, lineno=5), Instr("STORE_NAME", 'z', lineno=5)])
def test_extended_lnotab(self): # x = 7 # y = 8 concrete = ConcreteBytecode([ ConcreteInstr("LOAD_CONST", 0), SetLineno(1 + 128), ConcreteInstr("STORE_NAME", 0), # line number goes backward! SetLineno(1 + 129), ConcreteInstr("LOAD_CONST", 1), SetLineno(1), ConcreteInstr("STORE_NAME", 1), ]) concrete.consts = [7, 8] concrete.names = ["x", "y"] concrete.first_lineno = 1 if sys.version_info >= (3, 6): code = concrete.to_code() expected = b"d\x00Z\x00d\x01Z\x01" self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 1) self.assertEqual(code.co_lnotab, b"\x00\x7f\x02\x01\x02\x01\x00\x80\x02\xff") else: with self.assertRaises(ValueError) as cm: code = concrete.to_code() self.assertEqual( str(cm.exception), "negative line number delta is not supported " "on Python < 3.6", )
def test_extended_arg_unpack_ex(self): def test(): p = [1, 2, 3, 4, 5, 6] q, r, *s, t = p return q, r, s, t test.__code__ = ConcreteBytecode.from_code(test.__code__, extended_arg=True).to_code() self.assertEqual(test.__code__.co_stacksize, 6) self.assertEqual(test(), (1, 2, [3, 4, 5], 6))
def test_packing_lines(self): from bytecode.tests.long_lines_example import long_lines import dis line_starts = list(dis.findlinestarts(long_lines.__code__)) concrete = ConcreteBytecode.from_code(long_lines.__code__) as_code = concrete.to_code() self.assertEqual(line_starts, list(dis.findlinestarts(as_code)))
def test_attr(self): code_obj = get_code("x = 5") code = ConcreteBytecode.from_code(code_obj) self.assertEqual(code.consts, [5, None]) self.assertEqual(code.names, ['x']) self.assertEqual(code.varnames, []) self.assertEqual(code.freevars, []) self.assertListEqual(list(code), [ConcreteInstr('LOAD_CONST', 0, lineno=1), ConcreteInstr('STORE_NAME', 0, lineno=1), ConcreteInstr('LOAD_CONST', 1, lineno=1), ConcreteInstr('RETURN_VALUE', lineno=1)])
def test_freevar(self): concrete = ConcreteBytecode() concrete.freevars = ["x"] concrete.append(ConcreteInstr("LOAD_DEREF", 0)) code = concrete.to_code() concrete = ConcreteBytecode.from_code(code) self.assertEqual(concrete.cellvars, []) self.assertEqual(concrete.freevars, ["x"]) self.assertEqual(list(concrete), [ConcreteInstr("LOAD_DEREF", 0, lineno=1)]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.cellvars, []) self.assertEqual(list(bytecode), [Instr("LOAD_DEREF", FreeVar("x"), lineno=1)])
def test_extended_lnotab2(self): # x = 7 # 200 blank lines # y = 8 base_code = compile("x = 7" + "\n" * 200 + "y = 8", "", "exec") concrete = ConcreteBytecode( [ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(201), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("RETURN_VALUE"), ] ) concrete.consts = [None, 7, 8] concrete.names = ["x", "y"] concrete.first_lineno = 1 code = concrete.to_code() self.assertEqual(code.co_code, base_code.co_code) self.assertEqual(code.co_firstlineno, base_code.co_firstlineno) self.assertEqual(code.co_lnotab, base_code.co_lnotab) if sys.version_info >= (3, 10): self.assertEqual(code.co_linetable, base_code.co_linetable)
def test_extended_lnotab(self): # x = 7 # 200 blank lines # y = 8 concrete = ConcreteBytecode( [ ConcreteInstr("LOAD_CONST", 0), SetLineno(1 + 128), ConcreteInstr("STORE_NAME", 0), # line number goes backward! SetLineno(1 + 129), ConcreteInstr("LOAD_CONST", 1), SetLineno(1), ConcreteInstr("STORE_NAME", 1), ] ) concrete.consts = [7, 8] concrete.names = ["x", "y"] concrete.first_lineno = 1 code = concrete.to_code() expected = b"d\x00Z\x00d\x01Z\x01" self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 1) self.assertEqual(code.co_lnotab, b"\x02\x7f\x00\x01\x02\x01\x02\x80\x00\xff")
def check(self, source, function=False): ref_code = get_code(source, function=function) code = ConcreteBytecode.from_code(ref_code).to_code() self.assertEqual(code, ref_code) code = Bytecode.from_code(ref_code).to_code() self.assertEqual(code, ref_code) bytecode = Bytecode.from_code(ref_code) blocks = ControlFlowGraph.from_bytecode(bytecode) code = blocks.to_bytecode().to_code() self.assertEqual(code, ref_code)
def test_copy(self): concrete = ConcreteBytecode() concrete.first_lineno = 3 concrete.consts = [7, 8, 9] concrete.names = ["x", "y", "z"] concrete.extend([ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(4), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_NAME", 2), ]) self.assertEqual(concrete, concrete.copy())
def test_cellvar(self): concrete = ConcreteBytecode() concrete.cellvars = ['x'] concrete.append(ConcreteInstr('LOAD_DEREF', 0)) code = concrete.to_code() concrete = ConcreteBytecode.from_code(code) self.assertEqual(concrete.cellvars, ['x']) self.assertEqual(concrete.freevars, []) self.assertEqual(list(concrete), [ConcreteInstr('LOAD_DEREF', 0, lineno=1)]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.cellvars, ['x']) self.assertEqual(list(bytecode), [Instr('LOAD_DEREF', CellVar('x'), lineno=1)])
def test_extended_arg(self): # Create a code object from arbitrary bytecode co_code = b"%c4\x12d\xcd\xab" % EXTENDED_ARG code = get_code("x=1") args = ( code.co_argcount, code.co_nlocals, code.co_stacksize, code.co_flags, co_code, code.co_consts, code.co_names, code.co_varnames, code.co_filename, code.co_name, code.co_firstlineno, code.co_lnotab, code.co_freevars, code.co_cellvars, ) code = types.CodeType(*args) # without EXTENDED_ARG opcode bytecode = ConcreteBytecode.from_code(code) self.assertListEqual( list(bytecode), [ConcreteInstr("LOAD_CONST", 0x1234ABCD, lineno=1)]) # with EXTENDED_ARG opcode bytecode = ConcreteBytecode.from_code(code, extended_arg=True) expected = [ ConcreteInstr("EXTENDED_ARG", 0x1234, lineno=1), ConcreteInstr("LOAD_CONST", 0xABCD, lineno=1), ] self.assertListEqual(list(bytecode), expected)
def test_attr(self): code_obj = get_code("x = 5") code = ConcreteBytecode.from_code(code_obj) self.assertEqual(code.consts, [5, None]) self.assertEqual(code.names, ["x"]) self.assertEqual(code.varnames, []) self.assertEqual(code.freevars, []) self.assertListEqual( list(code), [ ConcreteInstr("LOAD_CONST", 0, lineno=1), ConcreteInstr("STORE_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 1, lineno=1), ConcreteInstr("RETURN_VALUE", lineno=1), ], )
def test_freevar(self): concrete = ConcreteBytecode() concrete.freevars = ['x'] concrete.append(ConcreteInstr('LOAD_DEREF', 0)) code = concrete.to_code() concrete = ConcreteBytecode.from_code(code) self.assertEqual(concrete.cellvars, []) self.assertEqual(concrete.freevars, ['x']) self.assertEqual(list(concrete), [ConcreteInstr('LOAD_DEREF', 0, lineno=1)]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.cellvars, []) self.assertEqual(list(bytecode), [Instr('LOAD_DEREF', FreeVar('x'), lineno=1)])
def test_explicit_stacksize(self): # Passing stacksize=... to ConcreteBytecode.to_code should result in a # code object with the specified stacksize. We pass some silly values # and assert that they are honored. code_obj = get_code("print('%s' % (a,b,c))") original_stacksize = code_obj.co_stacksize concrete = ConcreteBytecode.from_code(code_obj) # First with something bigger than necessary. explicit_stacksize = original_stacksize + 42 new_code_obj = concrete.to_code(stacksize=explicit_stacksize) self.assertEqual(new_code_obj.co_stacksize, explicit_stacksize) # Then with something bogus. We probably don't want to advertise this # in the documentation. If this fails then decide if it's for good # reason, and remove if so. explicit_stacksize = 0 new_code_obj = concrete.to_code(stacksize=explicit_stacksize) self.assertEqual(new_code_obj.co_stacksize, explicit_stacksize)
def test_label3(self): """ CPython generates useless EXTENDED_ARG 0 in some cases. We need to properly track them as otherwise we can end up with broken offset for jumps. """ source = """ def func(x): if x == 1: return x + 0 elif x == 2: return x + 1 elif x == 3: return x + 2 elif x == 4: return x + 3 elif x == 5: return x + 4 elif x == 6: return x + 5 elif x == 7: return x + 6 elif x == 8: return x + 7 elif x == 9: return x + 8 elif x == 10: return x + 9 elif x == 11: return x + 10 elif x == 12: return x + 11 elif x == 13: return x + 12 elif x == 14: return x + 13 elif x == 15: return x + 14 elif x == 16: return x + 15 elif x == 17: return x + 16 return -1 """ code = get_code(source, function=True) bcode = Bytecode.from_code(code) concrete = bcode.to_concrete_bytecode() self.assertIsInstance(concrete, ConcreteBytecode) # Ensure that we do not generate broken code loc = {} exec(textwrap.dedent(source), loc) func = loc['func'] func.__code__ = bcode.to_code() for i, x in enumerate(range(1, 18)): self.assertEqual(func(x), x + i) self.assertEqual(func(18), -1) # Ensure that we properly round trip in such cases self.assertEqual(ConcreteBytecode.from_code(code).to_code().co_code, code.co_code)