def _(tast: DefVar, ctx: Ctx): name = tast.name # declare ctx.add_local(name, Val()) yield from wait(DECLARED) new_ctx = ctx.new() # evaluating internal area bc = new_ctx.bc bc.kwonlyargcount = 0 bc.filename = tast.loc.filename bc.argcount = 0 new_ctx.visit(tast.value) bc.name = name or '<lambda>' bc.append(Instr("RETURN_VALUE")) fix_bytecode(new_ctx.bc) yield from wait(EVALUATED) # resolve delay = [] delay_append = delay.append if any(bc.freevars): for each in bc.freevars: if each not in ctx.local: if each not in ctx.bc.freevars: ctx.bc.freevars.append(each) delay_append( Instr('LOAD_CLOSURE', FreeVar(each), lineno=tast.lineno + 1)) else: if each not in ctx.bc.cellvars: ctx.bc.cellvars.append(each) delay_append( Instr('LOAD_CLOSURE', CellVar(each), lineno=tast.lineno + 1)) delay_append( Instr("BUILD_TUPLE", arg=len(bc.freevars), lineno=tast.lineno + 1)) if ctx.is_nested: bc.flags |= NESTED bc.flags |= OPTIMIZED yield from wait(RESOLVED) code = bc.to_code() # dis.show_code(code) ctx.bc.extend(delay) ctx.bc.append(Instr('LOAD_CONST', arg=code, lineno=tast.lineno + 1)) ctx.bc.append(Instr('LOAD_CONST', arg=bc.name, lineno=tast.lineno + 1)) ctx.bc.append( Instr('MAKE_FUNCTION', arg=8 if any(bc.freevars) else 0, lineno=tast.lineno + 1)) ctx.bc.append(Instr("CALL_FUNCTION", arg=0, lineno=tast.lineno + 1)) if name in ctx.bc.cellvars: ctx.bc.append(Instr("STORE_DEREF", arg=CellVar(name))) else: ctx.bc.append(Instr('STORE_FAST', arg=tast.name))
def test_cellvar_freevar(self): concrete = ConcreteBytecode() concrete.cellvars = ["cell"] concrete.freevars = ["free"] concrete.append(ConcreteInstr("LOAD_DEREF", 0)) concrete.append(ConcreteInstr("LOAD_DEREF", 1)) code = concrete.to_code() concrete = ConcreteBytecode.from_code(code) self.assertEqual(concrete.cellvars, ["cell"]) self.assertEqual(concrete.freevars, ["free"]) self.assertEqual( list(concrete), [ ConcreteInstr("LOAD_DEREF", 0, lineno=1), ConcreteInstr("LOAD_DEREF", 1, lineno=1), ], ) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.cellvars, ["cell"]) self.assertEqual( list(bytecode), [ Instr("LOAD_DEREF", CellVar("cell"), lineno=1), Instr("LOAD_DEREF", FreeVar("free"), lineno=1), ], )
def test_eq(self): code = ConcreteBytecode() self.assertFalse(code == 1) for name, val in ( ("names", ["a"]), ("varnames", ["a"]), ("consts", [1]), ("argcount", 1), ("kwonlyargcount", 2), ("flags", CompilerFlags(CompilerFlags.GENERATOR)), ("first_lineno", 10), ("filename", "xxxx.py"), ("name", "__x"), ("docstring", "x-x-x"), ("cellvars", [CellVar("x")]), ("freevars", [FreeVar("x")]), ): c = ConcreteBytecode() setattr(c, name, val) # For obscure reasons using assertNotEqual here fail self.assertFalse(code == c) if sys.version_info > (3, 8): c = ConcreteBytecode() c.posonlyargcount = 10 self.assertFalse(code == c) c = ConcreteBytecode() c.consts = [1] code.consts = [1] c.append(ConcreteInstr("LOAD_CONST", 0)) self.assertFalse(code == c)
def test_nested_class(self): def func(): # STORE_DEREF, LOAD_CLOSURE, LOAD_CLASSDEREF x = [] class NestedClass: y = x class_attr = NestedClass.y result = class_attr return result freevar_x = FreeVar("x") cellvar_x = CellVar("x") function_block = BasicBlock([ # x = [] Instr("BUILD_LIST", arg=0), Instr("STORE_DEREF", arg=cellvar_x), # class NestedClass: Instr("LOAD_BUILD_CLASS"), Instr("LOAD_CLOSURE", arg=cellvar_x), Instr("BUILD_TUPLE", arg=1), Instr("LOAD_CONST", arg=dummy_code_object), Instr("LOAD_CONST", arg="NestedClass"), Instr("MAKE_FUNCTION", arg=8), Instr("LOAD_CONST", arg="NestedClass"), Instr("CALL_FUNCTION", arg=2), Instr("STORE_FAST", arg="NestedClass"), # class_attr = NestedClass.y Instr("LOAD_FAST", arg="NestedClass"), Instr("LOAD_ATTR", arg="y"), Instr("STORE_FAST", arg="class_attr"), # result = class_attr Instr("LOAD_FAST", arg="class_attr"), Instr("STORE_FAST", arg="result"), # return result Instr("LOAD_FAST", arg="result"), Instr("RETURN_VALUE"), ]) nested_class_block = BasicBlock([ # y = x Instr("LOAD_CLASSDEREF", arg=freevar_x), Instr("STORE_NAME", arg="y"), Instr("LOAD_CONST", arg=None), Instr("RETURN_VALUE"), ]) expected_instructions = [] expected_instructions.extend(function_block) expected_instructions.extend(nested_class_block) dynamic_slice = slice_function_at_return(func.__code__, test_name="test_nested_class") self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions)) self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_eq(self): f1 = FreeVar("a") f2 = FreeVar("b") c1 = CellVar("a") c2 = CellVar("b") for v1, v2, eq in ( (f1, f1, True), (f1, f2, False), (f1, c1, False), (c1, c1, True), (c1, c2, False), ): if eq: self.assertEqual(v1, v2) else: self.assertNotEqual(v1, v2)
def test_closures(self): # Closure function freevar_foo = FreeVar("foo") cellvar_foo = CellVar("foo") module_block = BasicBlock([ # def outer_function(foo): Instr("LOAD_CONST", arg=dummy_code_object), Instr("LOAD_CONST", arg="outer_function"), Instr("MAKE_FUNCTION", arg=0), Instr("STORE_NAME", arg="outer_function"), # inner = outer_function('a') Instr("LOAD_NAME", arg="outer_function"), Instr("LOAD_CONST", arg="a"), Instr("CALL_FUNCTION", arg=1), Instr("STORE_NAME", arg="inner"), # result = inner("abc") Instr("LOAD_NAME", arg="inner"), Instr("LOAD_CONST", arg="abc"), Instr("CALL_FUNCTION", arg=1), Instr("STORE_NAME", arg="result"), Instr("LOAD_CONST", arg=None), Instr("RETURN_VALUE") ]) outer_function_block = BasicBlock([ # def inner_function(bar): Instr("LOAD_CLOSURE", arg=cellvar_foo), Instr("BUILD_TUPLE", arg=1), Instr("LOAD_CONST", arg=dummy_code_object), Instr("LOAD_CONST", arg="outer_function.<locals>.inner_function"), Instr("MAKE_FUNCTION", arg=8), Instr("STORE_FAST", arg="inner_function"), # return inner Instr("LOAD_FAST", arg="inner_function"), Instr("RETURN_VALUE"), ]) inner_function_block = BasicBlock([ # return foo in bar Instr("LOAD_DEREF", arg=freevar_foo), Instr("LOAD_FAST", arg="bar"), Instr("COMPARE_OP", arg=Compare.IN), Instr("RETURN_VALUE"), ]) expected_instructions = [] expected_instructions.extend(module_block) expected_instructions.extend(outer_function_block) expected_instructions.extend(inner_function_block) module_file = "closure.py" module_path = example_modules_path + module_file dynamic_slice = slice_module_at_return(module_path) self.assertEqual(len(expected_instructions), len(dynamic_slice.sliced_instructions)) self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_cellvars(self): code = Bytecode() code.cellvars = ['x'] code.freevars = ['y'] code.extend([Instr('LOAD_DEREF', CellVar('x'), lineno=1), Instr('LOAD_DEREF', FreeVar('y'), lineno=1)]) concrete = code.to_concrete_bytecode() self.assertEqual(concrete.cellvars, ['x']) self.assertEqual(concrete.freevars, ['y']) code.extend([ConcreteInstr("LOAD_DEREF", 0, lineno=1), ConcreteInstr("LOAD_DEREF", 1, lineno=1)])
def test_cellvar(self): concrete = ConcreteBytecode() concrete.cellvars = ["x"] concrete.append(ConcreteInstr("LOAD_DEREF", 0)) code = concrete.to_code() concrete = ConcreteBytecode.from_code(code) self.assertEqual(concrete.cellvars, ["x"]) self.assertEqual(concrete.freevars, []) self.assertEqual(list(concrete), [ConcreteInstr("LOAD_DEREF", 0, lineno=1)]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.cellvars, ["x"]) self.assertEqual(list(bytecode), [Instr("LOAD_DEREF", CellVar("x"), lineno=1)])
def test_invalid_arg(self): label = Label() block = BasicBlock() # EXTENDED_ARG self.assertRaises(ValueError, Instr, "EXTENDED_ARG", 0) # has_jump() self.assertRaises(TypeError, Instr, "JUMP_ABSOLUTE", 1) self.assertRaises(TypeError, Instr, "JUMP_ABSOLUTE", 1.0) Instr("JUMP_ABSOLUTE", label) Instr("JUMP_ABSOLUTE", block) # hasfree self.assertRaises(TypeError, Instr, "LOAD_DEREF", "x") Instr("LOAD_DEREF", CellVar("x")) Instr("LOAD_DEREF", FreeVar("x")) # haslocal self.assertRaises(TypeError, Instr, "LOAD_FAST", 1) Instr("LOAD_FAST", "x") # hasname self.assertRaises(TypeError, Instr, "LOAD_NAME", 1) Instr("LOAD_NAME", "x") # hasconst self.assertRaises(ValueError, Instr, "LOAD_CONST") # UNSET self.assertRaises(ValueError, Instr, "LOAD_CONST", label) self.assertRaises(ValueError, Instr, "LOAD_CONST", block) Instr("LOAD_CONST", 1.0) Instr("LOAD_CONST", object()) # hascompare self.assertRaises(TypeError, Instr, "COMPARE_OP", 1) Instr("COMPARE_OP", Compare.EQ) # HAVE_ARGUMENT self.assertRaises(ValueError, Instr, "CALL_FUNCTION", -1) self.assertRaises(TypeError, Instr, "CALL_FUNCTION", 3.0) Instr("CALL_FUNCTION", 3) # test maximum argument self.assertRaises(ValueError, Instr, "CALL_FUNCTION", 2147483647 + 1) instr = Instr("CALL_FUNCTION", 2147483647) self.assertEqual(instr.arg, 2147483647) # not HAVE_ARGUMENT self.assertRaises(ValueError, Instr, "NOP", 0) Instr("NOP")
def test_cellvars(self): code = Bytecode() code.cellvars = ["x"] code.freevars = ["y"] code.extend([ Instr("LOAD_DEREF", CellVar("x"), lineno=1), Instr("LOAD_DEREF", FreeVar("y"), lineno=1), ]) concrete = code.to_concrete_bytecode() self.assertEqual(concrete.cellvars, ["x"]) self.assertEqual(concrete.freevars, ["y"]) code.extend([ ConcreteInstr("LOAD_DEREF", 0, lineno=1), ConcreteInstr("LOAD_DEREF", 1, lineno=1), ])
def load_name(self, tast): name = tast.name if name in self.local: if name not in self.bc.cellvars: self.bc.append(Instr('LOAD_FAST', name, lineno=tast.lineno + 1)) else: self.bc.append( Instr('LOAD_DEREF', CellVar(name), lineno=tast.lineno + 1)) return if name in self.symtb: self.bc.append( Instr('LOAD_DEREF', FreeVar(name), lineno=tast.lineno + 1)) else: self.bc.append(Instr('LOAD_GLOBAL', name, lineno=tast.lineno + 1))
def test_cellvar_freevar(self): concrete = ConcreteBytecode() concrete.cellvars = ['cell'] concrete.freevars = ['free'] concrete.append(ConcreteInstr('LOAD_DEREF', 0)) concrete.append(ConcreteInstr('LOAD_DEREF', 1)) code = concrete.to_code() concrete = ConcreteBytecode.from_code(code) self.assertEqual(concrete.cellvars, ['cell']) self.assertEqual(concrete.freevars, ['free']) self.assertEqual(list(concrete), [ConcreteInstr('LOAD_DEREF', 0, lineno=1), ConcreteInstr('LOAD_DEREF', 1, lineno=1)]) bytecode = concrete.to_bytecode() self.assertEqual(bytecode.cellvars, ['cell']) self.assertEqual(list(bytecode), [Instr('LOAD_DEREF', CellVar('cell'), lineno=1), Instr('LOAD_DEREF', FreeVar('free'), lineno=1)])
def load_arg(x, cellvars, lineno): if x in cellvars: return PyInstr(InstrNames.LOAD_DEREF, CellVar(x), lineno=lineno) return PyInstr(InstrNames.LOAD_FAST, x, lineno=lineno)
def visit_def_fun(def_fun: DefFun, ctx: Ctx): name = def_fun.name if name: ctx.add_local(name, Val()) yield from wait(DECLARED) args = [arg.name for arg in def_fun.args] new_ctx = ctx.new() for each in def_fun.args: new_ctx.visit(each) bc = new_ctx.bc if isinstance(def_fun, DefFun) and def_fun.doc: bc.docstring = def_fun.doc.text bc.kwonlyargcount = 0 bc.filename = def_fun.loc.filename if any(args): bc.argcount = 1 if len(args) is 1: bc.argnames = args else: arg_name = '.'.join(args) bc.argnames = [arg_name] bc.append( Instr('LOAD_FAST', arg_name, lineno=def_fun.args[0].lineno + 1)) bc.append(Instr('UNPACK_SEQUENCE', arg=len(args))) for arg in args: bc.append(Instr('STORE_FAST', arg=arg)) else: bc.argcount = 0 new_ctx.visit(def_fun.body) bc.name = name or '<lambda>' bc.append(Instr("RETURN_VALUE")) fix_bytecode(bc) # for each in (bc): # print(each) # print('++++++++++') yield from wait(EVALUATED) delay = [] delay_append = delay.append if any(bc.freevars): for each in bc.freevars: if each not in ctx.local: if each not in ctx.bc.freevars: ctx.bc.freevars.append(each) delay_append( Instr('LOAD_CLOSURE', FreeVar(each), lineno=def_fun.lineno + 1)) else: if each not in ctx.bc.cellvars: ctx.bc.cellvars.append(each) delay_append( Instr('LOAD_CLOSURE', CellVar(each), lineno=def_fun.lineno + 1)) delay_append( Instr("BUILD_TUPLE", arg=len(bc.freevars), lineno=def_fun.lineno + 1)) if ctx.is_nested: bc.flags |= NESTED bc.flags |= OPTIMIZED # dump_bytecode(bc) # print(name, ctx.local.keys(), ctx.bc.freevars, ctx.bc.cellvars) yield from wait(RESOLVED) code = bc.to_code() ctx.bc.extend(delay) # dis.show_code(code) ctx.bc.append(Instr('LOAD_CONST', arg=code, lineno=def_fun.lineno + 1)) ctx.bc.append(Instr('LOAD_CONST', arg=bc.name, lineno=def_fun.lineno + 1)) ctx.bc.append( Instr('MAKE_FUNCTION', arg=8 if any(bc.freevars) else 0, lineno=def_fun.lineno + 1)) if name: if name in ctx.bc.cellvars: ctx.bc.append( Instr("STORE_DEREF", CellVar(name), lineno=def_fun.lineno + 1)) else: ctx.bc.append(Instr('STORE_FAST', name, lineno=def_fun.lineno + 1))
def test_nested_class_2(self): # Critical test to ensure that the attributes converted to variables # are taken from the correct scope. def func(): # STORE_DEREF, LOAD_CLOSURE, LOAD_CLASSDEREF x1 = [1] x2 = [2] class Bar: foo = x1 # included! class Foo: foo = x2 # NOT included y = x2 # included y = Foo.y # NOT included class_attr = Bar.foo class_attr2 = Bar.Foo.y result = class_attr + class_attr2 return result freevar_x1 = FreeVar("x1") cellvar_x1 = CellVar("x1") freevar_x2 = FreeVar("x2") cellvar_x2 = CellVar("x2") function_block = BasicBlock([ # x1 = [1] Instr("LOAD_CONST", arg=1), Instr("BUILD_LIST", arg=1), Instr("STORE_DEREF", arg=cellvar_x1), # x2 = [2] Instr("LOAD_CONST", arg=2), Instr("BUILD_LIST", arg=1), Instr("STORE_DEREF", arg=cellvar_x2), # class Bar: Instr("LOAD_BUILD_CLASS"), Instr("LOAD_CLOSURE", arg=cellvar_x1), Instr("LOAD_CLOSURE", arg=freevar_x2), Instr("BUILD_TUPLE", arg=2), Instr("LOAD_CONST", arg=dummy_code_object), Instr("LOAD_CONST", arg="Bar"), Instr("MAKE_FUNCTION", arg=8), Instr("LOAD_CONST", arg="Bar"), Instr("CALL_FUNCTION", arg=2), Instr("STORE_FAST", arg="Bar"), # class_attr = Bar.y Instr("LOAD_FAST", arg="Bar"), Instr("LOAD_ATTR", arg="foo"), Instr("STORE_FAST", arg="class_attr"), # class_attr2 = Bar.Foo.y Instr("LOAD_FAST", arg="Bar"), Instr("LOAD_ATTR", arg="Foo"), Instr("LOAD_ATTR", arg="y"), Instr("STORE_FAST", arg="class_attr2"), # result = class_attr + class_attr2 Instr("LOAD_FAST", arg="class_attr"), Instr("LOAD_FAST", arg="class_attr2"), Instr("BINARY_ADD"), Instr("STORE_FAST", arg="result"), # return result Instr("LOAD_FAST", arg="result"), Instr("RETURN_VALUE"), ]) bar_block = BasicBlock([ # class Foo: Instr("LOAD_BUILD_CLASS"), Instr("LOAD_CLOSURE", arg=cellvar_x2), Instr("BUILD_TUPLE", arg=1), Instr("LOAD_CONST", arg=dummy_code_object), Instr("LOAD_CONST", arg="Foo"), Instr("MAKE_FUNCTION", arg=8), Instr("LOAD_CONST", arg="Foo"), Instr("CALL_FUNCTION", arg=2), Instr("STORE_NAME", arg="Foo"), # foo = x1 Instr("LOAD_CLASSDEREF", arg=freevar_x1), Instr("STORE_NAME", arg="foo"), Instr("LOAD_CONST", arg=None), Instr("RETURN_VALUE"), ]) foo_block = BasicBlock([ # y = x2 Instr("LOAD_CLASSDEREF", arg=freevar_x2), Instr("STORE_NAME", arg="y"), Instr("LOAD_CONST", arg=None), Instr("RETURN_VALUE"), ]) expected_instructions = [] expected_instructions.extend(function_block) expected_instructions.extend(foo_block) expected_instructions.extend(bar_block) dynamic_slice = slice_function_at_return(func.__code__, test_name="test_nested_class_2") self.assertEqual(func(), [1, 2]) self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions)) self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))