def test_builtin(self): ns = {} exec(textwrap.dedent(""" def func(x): return chr(x) """), ns, ns) func = ns['func'] self.assertIn('LOAD_GLOBAL', disassemble(func)) self.assertEqual(func.__code__.co_consts, (None,)) # the specialized bytecode must not use LOAD_GLOBAL, but have # chr in its constants self.assertEqual(len(fat.get_specialized(func)), 1) new_code = fat.get_specialized(func)[0][0] self.assertNotIn('LOAD_GLOBAL', disassemble(new_code)) self.assertEqual(new_code.co_consts, (None, chr)) # call the specialized function self.assertNotIn('chr', globals()) self.assertEqual(func(65), 'A') # chr() is modified in globals(): call the original function # and remove the specialized bytecode ns['chr'] = str self.assertEqual(func(65), '65') self.assertEqual(len(fat.get_specialized(func)), 0)
def test_builtin(self): ns = {} exec( textwrap.dedent(""" def func(x): return chr(x) """), ns, ns) func = ns['func'] self.assertIn('LOAD_GLOBAL', disassemble(func)) self.assertEqual(func.__code__.co_consts, (None, )) # the specialized bytecode must not use LOAD_GLOBAL, but have # chr in its constants self.assertEqual(len(fat.get_specialized(func)), 1) new_code = fat.get_specialized(func)[0][0] self.assertNotIn('LOAD_GLOBAL', disassemble(new_code)) self.assertEqual(new_code.co_consts, (None, chr)) # call the specialized function self.assertNotIn('chr', globals()) self.assertEqual(func(65), 'A') # chr() is modified in globals(): call the original function # and remove the specialized bytecode ns['chr'] = str self.assertEqual(func(65), '65') self.assertEqual(len(fat.get_specialized(func)), 0)
def test_code(self): self.assertIn('LOAD_GLOBAL', disassemble(call_builtin)) self.assertEqual(len(fat.get_specialized(call_builtin)), 1) code = fat.get_specialized(call_builtin)[0][0] self.assertEqual(code.co_name, call_builtin.__name__) self.assertNotIn('LOAD_GLOBAL', disassemble(code))
def test_code(self): def func(): return len("abc") self.assertIn('LOAD_GLOBAL', disassemble(func)) self.assertEqual(len(fat.get_specialized(func)), 1) code = fat.get_specialized(func)[0][0] self.assertEqual(code.co_name, func.__name__) self.assertNotIn('LOAD_GLOBAL', disassemble(code))
def bench_guards(nguard): def func(): pass no_guard = bench(func, number=100) print("no guard: %s" % format_dt(no_guard)) if fat.get_specialized(func): print("ERROR: func already specialized") sys.exit(1) guards = [ fat.GuardDict(globals(), ('global_var', )) for i in range(nguard) ] fat.specialize(func, fast_func, guards) with_guards = bench(func) print("with %s guards on globals: %s" % (nguard, format_dt(with_guards))) dt = with_guards - no_guard print("cost of %s guards: %s (%.1f%%)" % (nguard, format_dt(dt), dt * 100 / no_guard)) dt = dt / nguard print("average cost of 1 guard: %s (%.1f%%)" % (format_dt(dt), dt * 100 / no_guard)) print()
def test_builtin_guard_global_exists(self): code = textwrap.dedent(""" import fat chr = lambda obj: "mock" def func(): return chr(65) def fast(): return "fast: A" guard = fat.GuardBuiltins('chr') fat.specialize(func, fast, [guard]) """) ns = self._exec(code) func = ns['func'] guard = ns['guard'] # chr() is overriden in the global namespace: the specialization must # be ignored self.assertEqual(len(fat.get_specialized(func)), 0) # guard init failed: it must always fail self.assertEqual(guard(), 2)
def test_builtin_guard_builtin_replaced(self): code = textwrap.dedent(""" import fat __builtins__['chr'] = lambda obj: "mock" def func(): return chr(65) def fast(): return "fast: A" guard = fat.GuardBuiltins('chr') fat.specialize(func, fast, [guard]) """) ns = self._exec(code) func = ns['func'] guard = ns['guard'] # chr() was replaced: the specialization must be ignored self.assertEqual(len(fat.get_specialized(func)), 0) # guard init failed: it must always fail self.assertEqual(guard(), 2)
def test_no_guards(self): def func(): pass ns = {} guards = guard_dict(ns, 'key') def func2(): pass fat.specialize(func, func2, guards) def func3(): pass fat.specialize(func, func3, guards) self.check_specialized(func, (func2.__code__, guards), (func3.__code__, guards)) # setting __code__ must remove all specialized functions def mock_func(): return "mock" func.__code__ = mock_func.__code__ self.assertEqual(fat.get_specialized(func), [])
def main(): if fat.get_specialized(func) or fat.get_specialized(func2): print("ERROR: functions already specialized!") sys.exit(1) fat.specialize(func2, fast_func2, [fat.GuardArgType(0, (list,))]) for range_pow10 in (0, 1, 3, 5): print("range(10 ** %s)" % range_pow10) dt = bench_list('func', range_pow10) print("- original bytecode: %s" % format_dt(dt)) dt2 = bench_list('func2', range_pow10) print("- append=obj.append with guards: %s" % compared_dt(dt2, dt)) dt2 = bench_list('fast_func2', range_pow10) print("- append=obj.append: %s" % compared_dt(dt2, dt))
def run_benchmark(bench): bench.timeit(stmt='func("abc")', globals=globals(), name="original bytecode (LOAD_GLOBAL)") bench.timeit(stmt='func_cst("abc")', globals=globals(), name="LOAD_CONST") fat.specialize(func, func_cst, [fat.GuardGlobals(("mylen",))]) assert fat.get_specialized(func) bench.timeit(stmt='func("abc")', globals=globals(), name="LOAD_CONST with guard on globals")
def main(): if fat.get_specialized(func) or fat.get_specialized(func2): print("ERROR: functions already specialized!") sys.exit(1) fat.specialize(func2, fast_func2, [fat.GuardArgType(0, (list, ))]) for range_pow10 in (0, 1, 3, 5): print("range(10 ** %s)" % range_pow10) dt = bench_list('func', range_pow10) print("- original bytecode: %s" % format_dt(dt)) dt2 = bench_list('func2', range_pow10) print("- append=obj.append with guards: %s" % compared_dt(dt2, dt)) dt2 = bench_list('fast_func2', range_pow10) print("- append=obj.append: %s" % compared_dt(dt2, dt))
def check_specialized(self, func, *expected): specialized = fat.get_specialized(func) self.assertEqual(len(specialized), len(expected)) for item1, item2 in zip(specialized, expected): self.assertIsInstance(item1, tuple) self.assertEqual(len(item1), 2) code1, guards1 = item1 code2, guards2 = item2 self.check_guards(guards1, guards2) self.assertEqual(code1.co_name, func.__name__) self.assertEqual(code1.co_code, code2.co_code)
def run_benchmark(bench): bench.timeit(stmt='func("abc")', globals=globals(), name='original bytecode (LOAD_GLOBAL)') bench.timeit(stmt='func_cst("abc")', globals=globals(), name='LOAD_CONST') fat.specialize(func, func_cst, [fat.GuardGlobals(('mylen', ))]) assert fat.get_specialized(func) bench.timeit(stmt='func("abc")', globals=globals(), name='LOAD_CONST with guard on globals')
def run_benchmark(bench): bench.timeit('func()', globals=globals(), name="original bytecode (call len)") bench.timeit('func_cst()', globals=globals(), name="return 3") fat.specialize(func, func_cst, [fat.GuardBuiltins(('len', ))]) assert fat.get_specialized(func) bench.timeit('func()', globals=globals(), name="return 3 with guard on builtins")
def test_qualname(self): def func(x): len(x) def nested(): pass return nested.__qualname__ self.assertEqual(len(fat.get_specialized(func)), 1) # optimizations must not modify function names qualname = func("abc") self.assertEqual(qualname, 'CopyBuiltinToConstant.test_qualname.<locals>.func.<locals>.nested')
def test_import(self): ns = {} code = textwrap.dedent(""" from builtins import str as chr def func(): # chr() is not the expected builtin function, # it must not be optimized return chr(65) """) exec(code, ns, ns) func = ns['func'] self.assertEqual(fat.get_specialized(func), [])
def run_benchmark(bench): bench.timeit(stmt='func("abc")', globals=globals(), name='original bytecode (LOAD_GLOBAL)') bench.timeit(stmt='func_cst("abc")', globals=globals(), name='LOAD_CONST') fat.specialize(func, func_cst, [fat.GuardBuiltins(('len',))]) assert fat.get_specialized(func) bench.timeit(stmt='func("abc")', globals=globals(), name='LOAD_CONST with guard on builtins and globals')
def test_modify_func_code(self): def func(): return "slow" def fast(): return "fast" self.assertNotSpecialized(func) self.assertNotSpecialized(fast) ns = {} guards = guard_dict(ns, 'key') fat.specialize(func, fast, guards) self.assertEqual(func(), 'fast') self.assertEqual(len(fat.get_specialized(func)), 1) def mock_func(): return 'mock' # setting __code__ must disable all optimizations func.__code__ = mock_func.__code__ self.assertEqual(func(), 'mock') self.assertEqual(len(fat.get_specialized(func)), 0)
def run_benchmark(bench): bench.timeit('func()', globals=globals(), name="original bytecode (call len)") bench.timeit('func_cst()', globals=globals(), name="return 3") fat.specialize(func, func_cst, [fat.GuardBuiltins(('len',))]) assert fat.get_specialized(func) bench.timeit('func()', globals=globals(), name="return 3 with guard on builtins")
def test_qualname(self): def func(x): len(x) def nested(): pass return nested.__qualname__ self.assertEqual(len(fat.get_specialized(func)), 1) # optimizations must not modify function names qualname = func("abc") self.assertEqual( qualname, 'CopyBuiltinToConstant.test_qualname.<locals>.func.<locals>.nested' )
def bench_guards(nguard): def func(): pass no_guard = bench(func, number=100) print("no guard: %s" % format_dt(no_guard)) if fat.get_specialized(func): print("ERROR: func already specialized") sys.exit(1) guards = [fat.GuardDict(globals(), ("global_var",)) for i in range(nguard)] fat.specialize(func, fast_func, guards) with_guards = bench(func) print("with %s guards on globals: %s" % (nguard, format_dt(with_guards))) dt = with_guards - no_guard print("cost of %s guards: %s (%.1f%%)" % (nguard, format_dt(dt), dt * 100 / no_guard)) dt = dt / nguard print("average cost of 1 guard: %s (%.1f%%)" % (format_dt(dt), dt * 100 / no_guard)) print()
def test_dict_guard(self): ns = dict(mykey=4) def func(): return 'slow' def func2(): return 'fast' guards = guard_dict(ns, 'mykey') fat.specialize(func, func2, guards) self.check_specialized(func, (func2.__code__, guards)) # Modify func, so the guard will fail def mock_func(): return 'mock' func.__code__ = mock_func.__code__ # Calling the function checks the guards and then removed the # specialized function since the function was modified self.assertEqual(func(), 'mock') self.assertEqual(fat.get_specialized(func), [])
def assertNotSpecialized(self, func): self.assertEqual(fat.get_specialized(func), [])
""" Microbenchmark on the "call builtin functions [with a constant]". The benchmark doesn't use fatoptimize, but specialize explicitly the function. """ import fat import sys from fatoptimizer.benchmark import bench, format_dt, compared_dt def func(): return len("abc") if fat.get_specialized(func): print("ERROR: func() was already specialized") sys.exit(1) def func_cst(): return 3 def run_benchmark(bench): bench.timeit('func()', globals=globals(), name="original bytecode (call len)") bench.timeit('func_cst()', globals=globals(), name="return 3") fat.specialize(func, func_cst, [fat.GuardBuiltins(('len', ))])
def test_qualname(self): self.assertEqual(len(fat.get_specialized(copy_builtin)), 1) # optimizations must not modify the function name qualname = copy_builtin("abc") self.assertEqual(qualname, 'copy_builtin.<locals>.nested')
def test_no_specialized(self): def func(): pass self.assertEqual(fat.get_specialized(func), [])
def isabs(s): """Test whether a path is absolute""" sep = _get_sep(s) return s.startswith(sep) def fast_isabs(s): """Test whether a path is absolute""" sep = _get_sep(s) return s.startswith(sep) # Manually inline _get_sep() in isabs() depending on the type of the s argument def isabs_str(s): return s.startswith('/') for func in (_get_sep, isabs, fast_isabs, isabs_str): if fat.get_specialized(func): print("ERROR: a function is already specialized!") sys.exit(1) fat.specialize(fast_isabs, isabs_str, [fat.GuardArgType(0, (str,)), fat.GuardGlobals(('_get_sep',)), fat.GuardBuiltins(('isinstance',)), fat.GuardFunc(_get_sep)]) dt = bench("isabs('/abc')") print("original isabs() bytecode: %s" % format_dt(dt)) dt2 = bench("fast_isabs('/abc')") print("_get_sep() inlined in isabs(): %s" % compared_dt(dt2, dt))