def test_builtin(self):
ns = {}
exec(textwrap.dedent("""
def func(x):
return chr(x)
"""), ns, ns)
func = ns['func']
self.assertIn('LOAD_GLOBAL', disassemble(func))
self.assertEqual(func.__code__.co_consts, (None,))
# the specialized bytecode must not use LOAD_GLOBAL, but have
# chr in its constants
self.assertEqual(len(fat.get_specialized(func)), 1)
new_code = fat.get_specialized(func)[0][0]
self.assertNotIn('LOAD_GLOBAL', disassemble(new_code))
self.assertEqual(new_code.co_consts, (None, chr))
# call the specialized function
self.assertNotIn('chr', globals())
self.assertEqual(func(65), 'A')
# chr() is modified in globals(): call the original function
# and remove the specialized bytecode
ns['chr'] = str
self.assertEqual(func(65), '65')
self.assertEqual(len(fat.get_specialized(func)), 0)
def test_builtin(self):
ns = {}
exec(
textwrap.dedent("""
def func(x):
return chr(x)
"""), ns, ns)
func = ns['func']
self.assertIn('LOAD_GLOBAL', disassemble(func))
self.assertEqual(func.__code__.co_consts, (None, ))
# the specialized bytecode must not use LOAD_GLOBAL, but have
# chr in its constants
self.assertEqual(len(fat.get_specialized(func)), 1)
new_code = fat.get_specialized(func)[0][0]
self.assertNotIn('LOAD_GLOBAL', disassemble(new_code))
self.assertEqual(new_code.co_consts, (None, chr))
# call the specialized function
self.assertNotIn('chr', globals())
self.assertEqual(func(65), 'A')
# chr() is modified in globals(): call the original function
# and remove the specialized bytecode
ns['chr'] = str
self.assertEqual(func(65), '65')
self.assertEqual(len(fat.get_specialized(func)), 0)
def test_code(self):
self.assertIn('LOAD_GLOBAL', disassemble(call_builtin))
self.assertEqual(len(fat.get_specialized(call_builtin)), 1)
code = fat.get_specialized(call_builtin)[0][0]
self.assertEqual(code.co_name, call_builtin.__name__)
self.assertNotIn('LOAD_GLOBAL', disassemble(code))
def test_code(self):
self.assertIn('LOAD_GLOBAL', disassemble(call_builtin))
self.assertEqual(len(fat.get_specialized(call_builtin)), 1)
code = fat.get_specialized(call_builtin)[0][0]
self.assertEqual(code.co_name, call_builtin.__name__)
self.assertNotIn('LOAD_GLOBAL', disassemble(code))
def test_code(self):
def func():
return len("abc")
self.assertIn('LOAD_GLOBAL', disassemble(func))
self.assertEqual(len(fat.get_specialized(func)), 1)
code = fat.get_specialized(func)[0][0]
self.assertEqual(code.co_name, func.__name__)
self.assertNotIn('LOAD_GLOBAL', disassemble(code))
def test_code(self):
def func():
return len("abc")
self.assertIn('LOAD_GLOBAL', disassemble(func))
self.assertEqual(len(fat.get_specialized(func)), 1)
code = fat.get_specialized(func)[0][0]
self.assertEqual(code.co_name, func.__name__)
self.assertNotIn('LOAD_GLOBAL', disassemble(code))
def bench_guards(nguard):
def func():
pass
no_guard = bench(func, number=100)
print("no guard: %s" % format_dt(no_guard))
if fat.get_specialized(func):
print("ERROR: func already specialized")
sys.exit(1)
guards = [
fat.GuardDict(globals(), ('global_var', )) for i in range(nguard)
]
fat.specialize(func, fast_func, guards)
with_guards = bench(func)
print("with %s guards on globals: %s" % (nguard, format_dt(with_guards)))
dt = with_guards - no_guard
print("cost of %s guards: %s (%.1f%%)" %
(nguard, format_dt(dt), dt * 100 / no_guard))
dt = dt / nguard
print("average cost of 1 guard: %s (%.1f%%)" %
(format_dt(dt), dt * 100 / no_guard))
print()
def test_builtin_guard_global_exists(self):
code = textwrap.dedent("""
import fat
chr = lambda obj: "mock"
def func():
return chr(65)
def fast():
return "fast: A"
guard = fat.GuardBuiltins('chr')
fat.specialize(func, fast, [guard])
""")
ns = self._exec(code)
func = ns['func']
guard = ns['guard']
# chr() is overriden in the global namespace: the specialization must
# be ignored
self.assertEqual(len(fat.get_specialized(func)), 0)
# guard init failed: it must always fail
self.assertEqual(guard(), 2)
def test_builtin_guard_builtin_replaced(self):
code = textwrap.dedent("""
import fat
__builtins__['chr'] = lambda obj: "mock"
def func():
return chr(65)
def fast():
return "fast: A"
guard = fat.GuardBuiltins('chr')
fat.specialize(func, fast, [guard])
""")
ns = self._exec(code)
func = ns['func']
guard = ns['guard']
# chr() was replaced: the specialization must be ignored
self.assertEqual(len(fat.get_specialized(func)), 0)
# guard init failed: it must always fail
self.assertEqual(guard(), 2)
def test_builtin_guard_builtin_replaced(self):
code = textwrap.dedent("""
import fat
__builtins__['chr'] = lambda obj: "mock"
def func():
return chr(65)
def fast():
return "fast: A"
guard = fat.GuardBuiltins('chr')
fat.specialize(func, fast, [guard])
""")
ns = self._exec(code)
func = ns['func']
guard = ns['guard']
# chr() was replaced: the specialization must be ignored
self.assertEqual(len(fat.get_specialized(func)), 0)
# guard init failed: it must always fail
self.assertEqual(guard(), 2)
def test_builtin_guard_global_exists(self):
code = textwrap.dedent("""
import fat
chr = lambda obj: "mock"
def func():
return chr(65)
def fast():
return "fast: A"
guard = fat.GuardBuiltins('chr')
fat.specialize(func, fast, [guard])
""")
ns = self._exec(code)
func = ns['func']
guard = ns['guard']
# chr() is overriden in the global namespace: the specialization must
# be ignored
self.assertEqual(len(fat.get_specialized(func)), 0)
# guard init failed: it must always fail
self.assertEqual(guard(), 2)
def test_no_guards(self):
def func():
pass
ns = {}
guards = guard_dict(ns, 'key')
def func2():
pass
fat.specialize(func, func2, guards)
def func3():
pass
fat.specialize(func, func3, guards)
self.check_specialized(func, (func2.__code__, guards),
(func3.__code__, guards))
# setting __code__ must remove all specialized functions
def mock_func():
return "mock"
func.__code__ = mock_func.__code__
self.assertEqual(fat.get_specialized(func), [])
def test_no_guards(self):
def func():
pass
ns = {}
guards = guard_dict(ns, 'key')
def func2():
pass
fat.specialize(func, func2, guards)
def func3():
pass
fat.specialize(func, func3, guards)
self.check_specialized(func,
(func2.__code__, guards),
(func3.__code__, guards))
# setting __code__ must remove all specialized functions
def mock_func():
return "mock"
func.__code__ = mock_func.__code__
self.assertEqual(fat.get_specialized(func), [])
def main():
if fat.get_specialized(func) or fat.get_specialized(func2):
print("ERROR: functions already specialized!")
sys.exit(1)
fat.specialize(func2, fast_func2, [fat.GuardArgType(0, (list,))])
for range_pow10 in (0, 1, 3, 5):
print("range(10 ** %s)" % range_pow10)
dt = bench_list('func', range_pow10)
print("- original bytecode: %s" % format_dt(dt))
dt2 = bench_list('func2', range_pow10)
print("- append=obj.append with guards: %s" % compared_dt(dt2, dt))
dt2 = bench_list('fast_func2', range_pow10)
print("- append=obj.append: %s" % compared_dt(dt2, dt))
def run_benchmark(bench):
bench.timeit(stmt='func("abc")', globals=globals(), name="original bytecode (LOAD_GLOBAL)")
bench.timeit(stmt='func_cst("abc")', globals=globals(), name="LOAD_CONST")
fat.specialize(func, func_cst, [fat.GuardGlobals(("mylen",))])
assert fat.get_specialized(func)
bench.timeit(stmt='func("abc")', globals=globals(), name="LOAD_CONST with guard on globals")
def main():
if fat.get_specialized(func) or fat.get_specialized(func2):
print("ERROR: functions already specialized!")
sys.exit(1)
fat.specialize(func2, fast_func2, [fat.GuardArgType(0, (list, ))])
for range_pow10 in (0, 1, 3, 5):
print("range(10 ** %s)" % range_pow10)
dt = bench_list('func', range_pow10)
print("- original bytecode: %s" % format_dt(dt))
dt2 = bench_list('func2', range_pow10)
print("- append=obj.append with guards: %s" % compared_dt(dt2, dt))
dt2 = bench_list('fast_func2', range_pow10)
print("- append=obj.append: %s" % compared_dt(dt2, dt))
def check_specialized(self, func, *expected):
specialized = fat.get_specialized(func)
self.assertEqual(len(specialized), len(expected))
for item1, item2 in zip(specialized, expected):
self.assertIsInstance(item1, tuple)
self.assertEqual(len(item1), 2)
code1, guards1 = item1
code2, guards2 = item2
self.check_guards(guards1, guards2)
self.assertEqual(code1.co_name, func.__name__)
self.assertEqual(code1.co_code, code2.co_code)
def check_specialized(self, func, *expected):
specialized = fat.get_specialized(func)
self.assertEqual(len(specialized), len(expected))
for item1, item2 in zip(specialized, expected):
self.assertIsInstance(item1, tuple)
self.assertEqual(len(item1), 2)
code1, guards1 = item1
code2, guards2 = item2
self.check_guards(guards1, guards2)
self.assertEqual(code1.co_name, func.__name__)
self.assertEqual(code1.co_code, code2.co_code)
def run_benchmark(bench):
bench.timeit(stmt='func("abc")',
globals=globals(),
name='original bytecode (LOAD_GLOBAL)')
bench.timeit(stmt='func_cst("abc")', globals=globals(), name='LOAD_CONST')
fat.specialize(func, func_cst, [fat.GuardGlobals(('mylen', ))])
assert fat.get_specialized(func)
bench.timeit(stmt='func("abc")',
globals=globals(),
name='LOAD_CONST with guard on globals')
def run_benchmark(bench):
bench.timeit('func()',
globals=globals(),
name="original bytecode (call len)")
bench.timeit('func_cst()', globals=globals(), name="return 3")
fat.specialize(func, func_cst, [fat.GuardBuiltins(('len', ))])
assert fat.get_specialized(func)
bench.timeit('func()',
globals=globals(),
name="return 3 with guard on builtins")
def test_qualname(self):
def func(x):
len(x)
def nested():
pass
return nested.__qualname__
self.assertEqual(len(fat.get_specialized(func)), 1)
# optimizations must not modify function names
qualname = func("abc")
self.assertEqual(qualname,
'CopyBuiltinToConstant.test_qualname.<locals>.func.<locals>.nested')
def test_import(self):
ns = {}
code = textwrap.dedent("""
from builtins import str as chr
def func():
# chr() is not the expected builtin function,
# it must not be optimized
return chr(65)
""")
exec(code, ns, ns)
func = ns['func']
self.assertEqual(fat.get_specialized(func), [])
def test_import(self):
ns = {}
code = textwrap.dedent("""
from builtins import str as chr
def func():
# chr() is not the expected builtin function,
# it must not be optimized
return chr(65)
""")
exec(code, ns, ns)
func = ns['func']
self.assertEqual(fat.get_specialized(func), [])
def run_benchmark(bench):
bench.timeit(stmt='func("abc")',
globals=globals(),
name='original bytecode (LOAD_GLOBAL)')
bench.timeit(stmt='func_cst("abc")',
globals=globals(),
name='LOAD_CONST')
fat.specialize(func, func_cst, [fat.GuardBuiltins(('len',))])
assert fat.get_specialized(func)
bench.timeit(stmt='func("abc")',
globals=globals(),
name='LOAD_CONST with guard on builtins and globals')
def test_modify_func_code(self):
def func():
return "slow"
def fast():
return "fast"
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
ns = {}
guards = guard_dict(ns, 'key')
fat.specialize(func, fast, guards)
self.assertEqual(func(), 'fast')
self.assertEqual(len(fat.get_specialized(func)), 1)
def mock_func():
return 'mock'
# setting __code__ must disable all optimizations
func.__code__ = mock_func.__code__
self.assertEqual(func(), 'mock')
self.assertEqual(len(fat.get_specialized(func)), 0)
def test_modify_func_code(self):
def func():
return "slow"
def fast():
return "fast"
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
ns = {}
guards = guard_dict(ns, 'key')
fat.specialize(func, fast, guards)
self.assertEqual(func(), 'fast')
self.assertEqual(len(fat.get_specialized(func)), 1)
def mock_func():
return 'mock'
# setting __code__ must disable all optimizations
func.__code__ = mock_func.__code__
self.assertEqual(func(), 'mock')
self.assertEqual(len(fat.get_specialized(func)), 0)
def run_benchmark(bench):
bench.timeit('func()',
globals=globals(),
name="original bytecode (call len)")
bench.timeit('func_cst()',
globals=globals(),
name="return 3")
fat.specialize(func, func_cst, [fat.GuardBuiltins(('len',))])
assert fat.get_specialized(func)
bench.timeit('func()',
globals=globals(),
name="return 3 with guard on builtins")
def test_qualname(self):
def func(x):
len(x)
def nested():
pass
return nested.__qualname__
self.assertEqual(len(fat.get_specialized(func)), 1)
# optimizations must not modify function names
qualname = func("abc")
self.assertEqual(
qualname,
'CopyBuiltinToConstant.test_qualname.<locals>.func.<locals>.nested'
)
def bench_guards(nguard):
def func():
pass
no_guard = bench(func, number=100)
print("no guard: %s" % format_dt(no_guard))
if fat.get_specialized(func):
print("ERROR: func already specialized")
sys.exit(1)
guards = [fat.GuardDict(globals(), ("global_var",)) for i in range(nguard)]
fat.specialize(func, fast_func, guards)
with_guards = bench(func)
print("with %s guards on globals: %s" % (nguard, format_dt(with_guards)))
dt = with_guards - no_guard
print("cost of %s guards: %s (%.1f%%)" % (nguard, format_dt(dt), dt * 100 / no_guard))
dt = dt / nguard
print("average cost of 1 guard: %s (%.1f%%)" % (format_dt(dt), dt * 100 / no_guard))
print()
def test_dict_guard(self):
ns = dict(mykey=4)
def func():
return 'slow'
def func2():
return 'fast'
guards = guard_dict(ns, 'mykey')
fat.specialize(func, func2, guards)
self.check_specialized(func,
(func2.__code__, guards))
# Modify func, so the guard will fail
def mock_func():
return 'mock'
func.__code__ = mock_func.__code__
# Calling the function checks the guards and then removed the
# specialized function since the function was modified
self.assertEqual(func(), 'mock')
self.assertEqual(fat.get_specialized(func), [])
def test_dict_guard(self):
ns = dict(mykey=4)
def func():
return 'slow'
def func2():
return 'fast'
guards = guard_dict(ns, 'mykey')
fat.specialize(func, func2, guards)
self.check_specialized(func, (func2.__code__, guards))
# Modify func, so the guard will fail
def mock_func():
return 'mock'
func.__code__ = mock_func.__code__
# Calling the function checks the guards and then removed the
# specialized function since the function was modified
self.assertEqual(func(), 'mock')
self.assertEqual(fat.get_specialized(func), [])
def assertNotSpecialized(self, func):
self.assertEqual(fat.get_specialized(func), [])
"""
Microbenchmark on the "call builtin functions [with a constant]".
The benchmark doesn't use fatoptimize, but specialize explicitly the function.
"""
import fat
import sys
from fatoptimizer.benchmark import bench, format_dt, compared_dt
def func():
return len("abc")
if fat.get_specialized(func):
print("ERROR: func() was already specialized")
sys.exit(1)
def func_cst():
return 3
def run_benchmark(bench):
bench.timeit('func()',
globals=globals(),
name="original bytecode (call len)")
bench.timeit('func_cst()', globals=globals(), name="return 3")
fat.specialize(func, func_cst, [fat.GuardBuiltins(('len', ))])
def test_qualname(self):
self.assertEqual(len(fat.get_specialized(copy_builtin)), 1)
# optimizations must not modify the function name
qualname = copy_builtin("abc")
self.assertEqual(qualname, 'copy_builtin.<locals>.nested')
def test_no_specialized(self):
def func():
pass
self.assertEqual(fat.get_specialized(func), [])
def assertNotSpecialized(self, func):
self.assertEqual(fat.get_specialized(func), [])
def test_qualname(self):
self.assertEqual(len(fat.get_specialized(copy_builtin)), 1)
# optimizations must not modify the function name
qualname = copy_builtin("abc")
self.assertEqual(qualname, 'copy_builtin.<locals>.nested')
def test_no_specialized(self):
def func():
pass
self.assertEqual(fat.get_specialized(func), [])
def isabs(s):
"""Test whether a path is absolute"""
sep = _get_sep(s)
return s.startswith(sep)
def fast_isabs(s):
"""Test whether a path is absolute"""
sep = _get_sep(s)
return s.startswith(sep)
# Manually inline _get_sep() in isabs() depending on the type of the s argument
def isabs_str(s):
return s.startswith('/')
for func in (_get_sep, isabs, fast_isabs, isabs_str):
if fat.get_specialized(func):
print("ERROR: a function is already specialized!")
sys.exit(1)
fat.specialize(fast_isabs, isabs_str,
[fat.GuardArgType(0, (str,)),
fat.GuardGlobals(('_get_sep',)),
fat.GuardBuiltins(('isinstance',)),
fat.GuardFunc(_get_sep)])
dt = bench("isabs('/abc')")
print("original isabs() bytecode: %s" % format_dt(dt))
dt2 = bench("fast_isabs('/abc')")
print("_get_sep() inlined in isabs(): %s" % compared_dt(dt2, dt))