def test_async_gen_flags(self): # Test inference in the presence of pre-existing flags for is_async in (None, True): # Infer generator code = ConcreteBytecode() code.append(ConcreteInstr("YIELD_VALUE")) for f, expected in ( (CompilerFlags.COROUTINE, CompilerFlags.ASYNC_GENERATOR), (CompilerFlags.ASYNC_GENERATOR, CompilerFlags.ASYNC_GENERATOR), (CompilerFlags.ITERABLE_COROUTINE, CompilerFlags.ITERABLE_COROUTINE), ): code.flags = CompilerFlags(f) code.update_flags(is_async=is_async) self.assertTrue(bool(code.flags & expected)) # Infer coroutine code = ConcreteBytecode() code.append(ConcreteInstr("YIELD_FROM")) for f, expected in ( (CompilerFlags.COROUTINE, CompilerFlags.COROUTINE), (CompilerFlags.ASYNC_GENERATOR, CompilerFlags.COROUTINE), (CompilerFlags.ITERABLE_COROUTINE, CompilerFlags.ITERABLE_COROUTINE), ): code.flags = CompilerFlags(f) code.update_flags(is_async=is_async) self.assertTrue(bool(code.flags & expected)) # Crash on ITERABLE_COROUTINE with async bytecode code = ConcreteBytecode() code.append(ConcreteInstr("GET_AWAITABLE")) code.flags = CompilerFlags(CompilerFlags.ITERABLE_COROUTINE) with self.assertRaises(ValueError): code.update_flags(is_async=is_async)
def test_invalid_types(self): code = ConcreteBytecode() code.append(Label()) with self.assertRaises(ValueError): list(code) with self.assertRaises(ValueError): ConcreteBytecode([Label()])
def test_eq(self): code = ConcreteBytecode() self.assertFalse(code == 1) for name, val in ( ("names", ["a"]), ("varnames", ["a"]), ("consts", [1]), ("argcount", 1), ("kwonlyargcount", 2), ("flags", CompilerFlags(CompilerFlags.GENERATOR)), ("first_lineno", 10), ("filename", "xxxx.py"), ("name", "__x"), ("docstring", "x-x-x"), ("cellvars", [CellVar("x")]), ("freevars", [FreeVar("x")]), ): c = ConcreteBytecode() setattr(c, name, val) # For obscure reasons using assertNotEqual here fail self.assertFalse(code == c) if sys.version_info > (3, 8): c = ConcreteBytecode() c.posonlyargcount = 10 self.assertFalse(code == c) c = ConcreteBytecode() c.consts = [1] code.consts = [1] c.append(ConcreteInstr("LOAD_CONST", 0)) self.assertFalse(code == c)
def test_async_gen_no_flag_is_async_None(self): # Test inference in the absence of any flag set on the bytecode # Infer generator code = ConcreteBytecode() code.append(ConcreteInstr("YIELD_VALUE")) code.update_flags() self.assertTrue(bool(code.flags & CompilerFlags.GENERATOR)) # Infer coroutine code = ConcreteBytecode() code.append(ConcreteInstr("GET_AWAITABLE")) code.update_flags() self.assertTrue(bool(code.flags & CompilerFlags.COROUTINE)) # Infer coroutine or async generator for i, expected in ( ("YIELD_VALUE", CompilerFlags.ASYNC_GENERATOR), ("YIELD_FROM", CompilerFlags.COROUTINE), ): code = ConcreteBytecode() code.append(ConcreteInstr("GET_AWAITABLE")) code.append(ConcreteInstr(i)) code.update_flags() self.assertTrue(bool(code.flags & expected))
def test_flag_inference(self): # Check no loss of non-infered flags code = ControlFlowGraph() code.flags |= (CompilerFlags.NEWLOCALS | CompilerFlags.VARARGS | CompilerFlags.VARKEYWORDS | CompilerFlags.NESTED | CompilerFlags.FUTURE_GENERATOR_STOP) code.update_flags() for f in (CompilerFlags.NEWLOCALS, CompilerFlags.VARARGS, CompilerFlags.VARKEYWORDS, CompilerFlags.NESTED, CompilerFlags.NOFREE, CompilerFlags.OPTIMIZED, CompilerFlags.FUTURE_GENERATOR_STOP): self.assertTrue(bool(code.flags & f)) # Infer optimized and nofree code = Bytecode() flags = infer_flags(code) self.assertTrue(bool(flags & CompilerFlags.OPTIMIZED)) self.assertTrue(bool(flags & CompilerFlags.NOFREE)) code.append(ConcreteInstr('STORE_NAME', 1)) flags = infer_flags(code) self.assertFalse(bool(flags & CompilerFlags.OPTIMIZED)) self.assertTrue(bool(flags & CompilerFlags.NOFREE)) code.append(ConcreteInstr('STORE_DEREF', 2)) code.update_flags() self.assertFalse(bool(code.flags & CompilerFlags.OPTIMIZED)) self.assertFalse(bool(code.flags & CompilerFlags.NOFREE)) # Infer generator code = ConcreteBytecode() code.append(ConcreteInstr('YIELD_VALUE')) for is_async, expected in ((False, CompilerFlags.GENERATOR), (True, CompilerFlags.ASYNC_GENERATOR)): self.assertTrue(bool(infer_flags(code, is_async) & expected)) # Infer coroutine code = ConcreteBytecode() code.append(ConcreteInstr('GET_AWAITABLE')) iter_flags = CompilerFlags(CompilerFlags.ITERABLE_COROUTINE) for f, expected in ((CompilerFlags(0), True), (iter_flags, False)): code.flags = f self.assertEqual(bool(infer_flags(code) & CompilerFlags.COROUTINE), expected) # Test check flag sanity code.append(ConcreteInstr('YIELD_VALUE')) code.flags = CompilerFlags(CompilerFlags.GENERATOR | CompilerFlags.COROUTINE) infer_flags(code, is_async=True) # Just want to be sure it pases with self.assertRaises(ValueError): code.update_flags() with self.assertRaises(ValueError): infer_flags(None)
def test_load_classderef(self): concrete = ConcreteBytecode() concrete.cellvars = ['__class__'] concrete.freevars = ['__class__'] concrete.extend([ ConcreteInstr('LOAD_CLASSDEREF', 1), ConcreteInstr('STORE_DEREF', 1) ]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.freevars, ['__class__']) self.assertEqual(bytecode.cellvars, ['__class__']) self.assertEqual(list(bytecode), [ Instr('LOAD_CLASSDEREF', FreeVar('__class__'), lineno=1), Instr('STORE_DEREF', FreeVar('__class__'), lineno=1) ]) concrete = bytecode.to_concrete_bytecode() self.assertEqual(concrete.freevars, ['__class__']) self.assertEqual(concrete.cellvars, ['__class__']) self.assertEqual(list(concrete), [ ConcreteInstr('LOAD_CLASSDEREF', 1, lineno=1), ConcreteInstr('STORE_DEREF', 1, lineno=1) ]) code = concrete.to_code() self.assertEqual(code.co_freevars, ('__class__', )) self.assertEqual(code.co_cellvars, ('__class__', )) self.assertEqual( code.co_code, b'\x94\x01\x89\x01' if WORDCODE else b'\x94\x01\x00\x89\x01\x00')
def test_legalize(self): concrete = ConcreteBytecode() concrete.first_lineno = 3 concrete.consts = [7, 8, 9] concrete.names = ["x", "y", "z"] concrete.extend( [ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), ConcreteInstr("LOAD_CONST", 1, lineno=4), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2, lineno=6), ConcreteInstr("STORE_NAME", 2), ] ) concrete.legalize() self.assertListEqual( list(concrete), [ ConcreteInstr("LOAD_CONST", 0, lineno=3), ConcreteInstr("STORE_NAME", 0, lineno=3), ConcreteInstr("LOAD_CONST", 1, lineno=4), ConcreteInstr("STORE_NAME", 1, lineno=4), ConcreteInstr("LOAD_CONST", 2, lineno=5), ConcreteInstr("STORE_NAME", 2, lineno=5), ], )
def test_to_bytecode_consts(self): # x = -0.0 # x = +0.0 # # code optimized by the CPython 3.6 peephole optimizer which emits # duplicated constants (0.0 is twice in consts). code = ConcreteBytecode() code.consts = [0.0, None, -0.0, 0.0] code.names = ["x", "y"] code.extend([ ConcreteInstr("LOAD_CONST", 2, lineno=1), ConcreteInstr("STORE_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 3, lineno=2), ConcreteInstr("STORE_NAME", 1, lineno=2), ConcreteInstr("LOAD_CONST", 1, lineno=2), ConcreteInstr("RETURN_VALUE", lineno=2), ]) code = code.to_bytecode().to_concrete_bytecode() # the conversion changes the constant order: the order comes from # the order of LOAD_CONST instructions self.assertEqual(code.consts, [-0.0, 0.0, None]) code.names = ["x", "y"] self.assertListEqual( list(code), [ ConcreteInstr("LOAD_CONST", 0, lineno=1), ConcreteInstr("STORE_NAME", 0, lineno=1), ConcreteInstr("LOAD_CONST", 1, lineno=2), ConcreteInstr("STORE_NAME", 1, lineno=2), ConcreteInstr("LOAD_CONST", 2, lineno=2), ConcreteInstr("RETURN_VALUE", lineno=2), ], )
def test_extended_lnotab(self): # x = 7 # y = 8 concrete = ConcreteBytecode([ ConcreteInstr("LOAD_CONST", 0), SetLineno(1 + 128), ConcreteInstr("STORE_NAME", 0), # line number goes backward! SetLineno(1 + 129), ConcreteInstr("LOAD_CONST", 1), SetLineno(1), ConcreteInstr("STORE_NAME", 1), ]) concrete.consts = [7, 8] concrete.names = ["x", "y"] concrete.first_lineno = 1 if sys.version_info >= (3, 6): code = concrete.to_code() expected = b"d\x00Z\x00d\x01Z\x01" self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 1) self.assertEqual(code.co_lnotab, b"\x00\x7f\x02\x01\x02\x01\x00\x80\x02\xff") else: with self.assertRaises(ValueError) as cm: code = concrete.to_code() self.assertEqual( str(cm.exception), "negative line number delta is not supported " "on Python < 3.6", )
def test_extended_lnotab2(self): # x = 7 # 200 blank lines # y = 8 base_code = compile("x = 7" + "\n" * 200 + "y = 8", "", "exec") concrete = ConcreteBytecode( [ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(201), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("RETURN_VALUE"), ] ) concrete.consts = [None, 7, 8] concrete.names = ["x", "y"] concrete.first_lineno = 1 code = concrete.to_code() self.assertEqual(code.co_code, base_code.co_code) self.assertEqual(code.co_firstlineno, base_code.co_firstlineno) self.assertEqual(code.co_lnotab, base_code.co_lnotab) if sys.version_info >= (3, 10): self.assertEqual(code.co_linetable, base_code.co_linetable)
def test_setlineno(self): # x = 7 # y = 8 # z = 9 concrete = ConcreteBytecode() concrete.consts = [7, 8, 9] concrete.names = ["x", "y", "z"] concrete.first_lineno = 3 concrete.extend( [ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(4), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_NAME", 2), ] ) code = concrete.to_bytecode() self.assertEqual( code, [ Instr("LOAD_CONST", 7, lineno=3), Instr("STORE_NAME", "x", lineno=3), Instr("LOAD_CONST", 8, lineno=4), Instr("STORE_NAME", "y", lineno=4), Instr("LOAD_CONST", 9, lineno=5), Instr("STORE_NAME", "z", lineno=5), ], )
def test_async_gen_no_flag_is_async_False(self): # Test inference when we request a non-async function # Infer generator code = ConcreteBytecode() code.append(ConcreteInstr("YIELD_VALUE")) code.flags = CompilerFlags(CompilerFlags.COROUTINE) code.update_flags(is_async=False) self.assertTrue(bool(code.flags & CompilerFlags.GENERATOR)) # Abort on coroutine code = ConcreteBytecode() code.append(ConcreteInstr("GET_AWAITABLE")) code.flags = CompilerFlags(CompilerFlags.COROUTINE) with self.assertRaises(ValueError): code.update_flags(is_async=False)
def test_to_code_lnotab(self): # x = 7 # y = 8 # z = 9 concrete = ConcreteBytecode() concrete.consts = [7, 8, 9] concrete.names = ['x', 'y', 'z'] concrete.first_lineno = 3 concrete.extend([ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(4), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_NAME", 2) ]) code = concrete.to_code() if WORDCODE: expected = b'd\x00Z\x00d\x01Z\x01d\x02Z\x02' else: expected = (b'd\x00\x00' b'Z\x00\x00' b'd\x01\x00' b'Z\x01\x00' b'd\x02\x00' b'Z\x02\x00') self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 3) self.assertEqual( code.co_lnotab, b'\x04\x01\x04\x01' if WORDCODE else b'\x06\x01\x06\x01')
def test_to_code_lnotab(self): # x = 7 # y = 8 # z = 9 concrete = ConcreteBytecode() concrete.consts = [7, 8, 9] concrete.names = ["x", "y", "z"] concrete.first_lineno = 3 concrete.extend([ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(4), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_NAME", 2), ]) code = concrete.to_code() expected = (b"d\x00\x00" b"Z\x00\x00" b"d\x01\x00" b"Z\x01\x00" b"d\x02\x00" b"Z\x02\x00") self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 3) self.assertEqual(code.co_lnotab, b"\x06\x01\x06\x01")
def test_cellvar_freevar(self): concrete = ConcreteBytecode() concrete.cellvars = ["cell"] concrete.freevars = ["free"] concrete.append(ConcreteInstr("LOAD_DEREF", 0)) concrete.append(ConcreteInstr("LOAD_DEREF", 1)) code = concrete.to_code() concrete = ConcreteBytecode.from_code(code) self.assertEqual(concrete.cellvars, ["cell"]) self.assertEqual(concrete.freevars, ["free"]) self.assertEqual( list(concrete), [ ConcreteInstr("LOAD_DEREF", 0, lineno=1), ConcreteInstr("LOAD_DEREF", 1, lineno=1), ], ) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.cellvars, ["cell"]) self.assertEqual( list(bytecode), [ Instr("LOAD_DEREF", CellVar("cell"), lineno=1), Instr("LOAD_DEREF", FreeVar("free"), lineno=1), ], )
def test_to_code_lnotab(self): # We use an actual function for the simple case to # ensure we get lnotab right def f(): # # x = 7 # noqa y = 8 # noqa z = 9 # noqa fl = f.__code__.co_firstlineno concrete = ConcreteBytecode() concrete.consts = [None, 7, 8, 9] concrete.varnames = ["x", "y", "z"] concrete.first_lineno = fl concrete.extend([ SetLineno(fl + 3), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_FAST", 0), SetLineno(fl + 4), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_FAST", 1), SetLineno(fl + 5), ConcreteInstr("LOAD_CONST", 3), ConcreteInstr("STORE_FAST", 2), ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("RETURN_VALUE"), ]) code = concrete.to_code() self.assertEqual(code.co_code, f.__code__.co_code) self.assertEqual(code.co_lnotab, f.__code__.co_lnotab) if sys.version_info >= (3, 10): self.assertEqual(code.co_linetable, f.__code__.co_linetable)
def test_negative_lnotab(self): # x = 7 # y = 8 concrete = ConcreteBytecode([ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), # line number goes backward! SetLineno(2), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1) ]) concrete.consts = [7, 8] concrete.names = ['x', 'y'] concrete.first_lineno = 5 if sys.version_info >= (3, 6): code = concrete.to_code() expected = b'd\x00Z\x00d\x01Z\x01' self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 5) self.assertEqual(code.co_lnotab, b'\x04\xfd') else: with self.assertRaises(ValueError) as cm: code = concrete.to_code() self.assertEqual( str(cm.exception), "negative line number delta is not supported " "on Python < 3.6")
def test_extended_lnotab(self): # x = 7 # 200 blank lines # y = 8 concrete = ConcreteBytecode( [ ConcreteInstr("LOAD_CONST", 0), SetLineno(1 + 128), ConcreteInstr("STORE_NAME", 0), # line number goes backward! SetLineno(1 + 129), ConcreteInstr("LOAD_CONST", 1), SetLineno(1), ConcreteInstr("STORE_NAME", 1), ] ) concrete.consts = [7, 8] concrete.names = ["x", "y"] concrete.first_lineno = 1 code = concrete.to_code() expected = b"d\x00Z\x00d\x01Z\x01" self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 1) self.assertEqual(code.co_lnotab, b"\x02\x7f\x00\x01\x02\x01\x02\x80\x00\xff")
def test_async_gen_no_flag_is_async_True(self): # Test inference when we request an async function # Force coroutine code = ConcreteBytecode() code.update_flags(is_async=True) self.assertTrue(bool(code.flags & CompilerFlags.COROUTINE)) # Infer coroutine or async generator for i, expected in ( ("YIELD_VALUE", CompilerFlags.ASYNC_GENERATOR), ("YIELD_FROM", CompilerFlags.COROUTINE), ): code = ConcreteBytecode() code.append(ConcreteInstr(i)) code.update_flags(is_async=True) self.assertTrue(bool(code.flags & expected))
def test_freevar(self): concrete = ConcreteBytecode() concrete.freevars = ["x"] concrete.append(ConcreteInstr("LOAD_DEREF", 0)) code = concrete.to_code() concrete = ConcreteBytecode.from_code(code) self.assertEqual(concrete.cellvars, []) self.assertEqual(concrete.freevars, ["x"]) self.assertEqual(list(concrete), [ConcreteInstr("LOAD_DEREF", 0, lineno=1)]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.cellvars, []) self.assertEqual(list(bytecode), [Instr("LOAD_DEREF", FreeVar("x"), lineno=1)])
def test_cellvar(self): concrete = ConcreteBytecode() concrete.cellvars = ['x'] concrete.append(ConcreteInstr('LOAD_DEREF', 0)) code = concrete.to_code() concrete = ConcreteBytecode.from_code(code) self.assertEqual(concrete.cellvars, ['x']) self.assertEqual(concrete.freevars, []) self.assertEqual(list(concrete), [ConcreteInstr('LOAD_DEREF', 0, lineno=1)]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.cellvars, ['x']) self.assertEqual(list(bytecode), [Instr('LOAD_DEREF', CellVar('x'), lineno=1)])
def test_copy(self): concrete = ConcreteBytecode() concrete.first_lineno = 3 concrete.consts = [7, 8, 9] concrete.names = ["x", "y", "z"] concrete.extend([ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), SetLineno(4), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), SetLineno(5), ConcreteInstr("LOAD_CONST", 2), ConcreteInstr("STORE_NAME", 2), ]) self.assertEqual(concrete, concrete.copy())
def test_negative_lnotab(self): # x = 7 # y = 8 concrete = ConcreteBytecode([ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), # line number goes backward! SetLineno(2), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), ]) concrete.consts = [7, 8] concrete.names = ["x", "y"] concrete.first_lineno = 5 code = concrete.to_code() expected = b"d\x00Z\x00d\x01Z\x01" self.assertEqual(code.co_code, expected) self.assertEqual(code.co_firstlineno, 5) self.assertEqual(code.co_lnotab, b"\x04\xfd")
def test_negative_lnotab(self): # x = 7 # y = 8 concrete = ConcreteBytecode([ ConcreteInstr("LOAD_CONST", 0), ConcreteInstr("STORE_NAME", 0), # line number goes backward! SetLineno(2), ConcreteInstr("LOAD_CONST", 1), ConcreteInstr("STORE_NAME", 1), ]) concrete.consts = [7, 8] concrete.names = ["x", "y"] concrete.first_lineno = 5 with self.assertRaises(ValueError) as cm: concrete.to_code() self.assertEqual( str(cm.exception), "negative line number delta is not supported " "on Python < 3.6", )
def test_load_classderef(self): concrete = ConcreteBytecode() concrete.cellvars = ["__class__"] concrete.freevars = ["__class__"] concrete.extend([ ConcreteInstr("LOAD_CLASSDEREF", 1), ConcreteInstr("STORE_DEREF", 1) ]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.freevars, ["__class__"]) self.assertEqual(bytecode.cellvars, ["__class__"]) self.assertEqual( list(bytecode), [ Instr("LOAD_CLASSDEREF", FreeVar("__class__"), lineno=1), Instr("STORE_DEREF", FreeVar("__class__"), lineno=1), ], ) concrete = bytecode.to_concrete_bytecode() self.assertEqual(concrete.freevars, ["__class__"]) self.assertEqual(concrete.cellvars, ["__class__"]) self.assertEqual( list(concrete), [ ConcreteInstr("LOAD_CLASSDEREF", 1, lineno=1), ConcreteInstr("STORE_DEREF", 1, lineno=1), ], ) code = concrete.to_code() self.assertEqual(code.co_freevars, ("__class__", )) self.assertEqual(code.co_cellvars, ("__class__", )) self.assertEqual( code.co_code, b"\x94\x01\x89\x01" if WORDCODE else b"\x94\x01\x00\x89\x01\x00", )
def test_repr(self): r = repr(ConcreteBytecode()) self.assertIn("ConcreteBytecode", r) self.assertIn("0", r)