def test_arg_type(self):
def func(x):
return "slow: %s" % x
def fast(x):
return "fast: %s" % x
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
fat.specialize(func, fast, [fat.GuardArgType(0, (int, ))])
self.assertEqual(func(3), 'fast: 3')
# FIXME: implement keywords
#self.assertEqual(func(x=4), 'fast: 4')
# calling with the wrong number of parameter must not disable the
# optimization
with self.assertRaises(TypeError):
func()
with self.assertRaises(TypeError):
func(
1,
2,
)
self.assertEqual(func(5), 'fast: 5')
# wrong type, skip optimization
self.assertEqual(func(6.0), 'slow: 6.0')
# optimization must not be disabled after call with wrong types
self.assertEqual(func(7), 'fast: 7')
def test_specialize_with_freevars(self):
def func():
return 1
def create_fast():
x = 1
def fast():
return x
return fast
fast = create_fast()
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
ns = {}
guards = guard_dict(ns, 'key')
for bytecode in (False, True):
with self.assertRaises(ValueError) as cm:
if bytecode:
fat.specialize(func, fast.__code__, guards)
else:
fat.specialize(func, fast, guards)
self.assertEqual(
str(cm.exception),
"specialized bytecode uses different free variables")
def test_specialize_with_cellvars(self):
def func(data, cb):
return 3
def fast(data, cb):
return [cb(item) for item in data]
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
self.assertEqual(func.__code__.co_cellvars, ())
self.assertNotEqual(fast.__code__.co_cellvars, ())
ns = {}
guards = guard_dict(ns, 'key')
for bytecode in (False, True):
with self.assertRaises(ValueError) as cm:
if bytecode:
fat.specialize(func, fast.__code__, guards)
else:
fat.specialize(func, fast, guards)
self.assertEqual(
str(cm.exception),
"specialized bytecode uses different cell variables")
def test_arg_type(self):
def func(x):
return "slow: %s" % x
def fast(x):
return "fast: %s" % x
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
fat.specialize(func, fast, [fat.GuardArgType(0, (int,))])
self.assertEqual(func(3), 'fast: 3')
# FIXME: implement keywords
#self.assertEqual(func(x=4), 'fast: 4')
# calling with the wrong number of parameter must not disable the
# optimization
with self.assertRaises(TypeError):
func()
with self.assertRaises(TypeError):
func(1, 2,)
self.assertEqual(func(5), 'fast: 5')
# wrong type, skip optimization
self.assertEqual(func(6.0), 'slow: 6.0')
# optimization must not be disabled after call with wrong types
self.assertEqual(func(7), 'fast: 7')
def bench_guards(nguard):
def func():
pass
no_guard = bench(func, number=100)
print("no guard: %s" % format_dt(no_guard))
if fat.get_specialized(func):
print("ERROR: func already specialized")
sys.exit(1)
guards = [
fat.GuardDict(globals(), ('global_var', )) for i in range(nguard)
]
fat.specialize(func, fast_func, guards)
with_guards = bench(func)
print("with %s guards on globals: %s" % (nguard, format_dt(with_guards)))
dt = with_guards - no_guard
print("cost of %s guards: %s (%.1f%%)" %
(nguard, format_dt(dt), dt * 100 / no_guard))
dt = dt / nguard
print("average cost of 1 guard: %s (%.1f%%)" %
(format_dt(dt), dt * 100 / no_guard))
print()
def test_specialize_with_freevars(self):
def func():
return 1
def create_fast():
x = 1
def fast():
return x
return fast
fast = create_fast()
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
ns = {}
guards = guard_dict(ns, 'key')
for bytecode in (False, True):
with self.assertRaises(ValueError) as cm:
if bytecode:
fat.specialize(func, fast.__code__, guards)
else:
fat.specialize(func, fast, guards)
self.assertEqual(str(cm.exception),
"specialized bytecode uses different free variables")
def test_no_guards(self):
def func():
pass
ns = {}
guards = guard_dict(ns, 'key')
def func2():
pass
fat.specialize(func, func2, guards)
def func3():
pass
fat.specialize(func, func3, guards)
self.check_specialized(func, (func2.__code__, guards),
(func3.__code__, guards))
# setting __code__ must remove all specialized functions
def mock_func():
return "mock"
func.__code__ = mock_func.__code__
self.assertEqual(fat.get_specialized(func), [])
def test_over_specialized(self):
def func():
pass
def func2():
pass
def func3():
pass
self.assertNotSpecialized(func)
self.assertNotSpecialized(func2)
self.assertNotSpecialized(func3)
ns = {}
guards = guard_dict(ns, 'key')
fat.specialize(func2, func3, guards)
with self.assertRaises(ValueError) as cm:
fat.specialize(func, func2, guards)
self.assertEqual(
str(cm.exception),
'cannot specialize a function with another function '
'which is already specialized')
def test_no_guards(self):
def func():
pass
ns = {}
guards = guard_dict(ns, 'key')
def func2():
pass
fat.specialize(func, func2, guards)
def func3():
pass
fat.specialize(func, func3, guards)
self.check_specialized(func,
(func2.__code__, guards),
(func3.__code__, guards))
# setting __code__ must remove all specialized functions
def mock_func():
return "mock"
func.__code__ = mock_func.__code__
self.assertEqual(fat.get_specialized(func), [])
def test_specialize_error(self):
def func():
pass
with self.assertRaises(ValueError) as cm:
fat.specialize(func, func, [{'guard_type': 'func', 'func': func}])
self.assertEqual(str(cm.exception),
"a function cannot specialize itself")
def test_specialize_error(self):
def func():
pass
with self.assertRaises(ValueError) as cm:
fat.specialize(func, func, [{'guard_type': 'func', 'func': func}])
self.assertEqual(str(cm.exception),
"a function cannot specialize itself")
def run_benchmark(bench):
bench.timeit(stmt='func("abc")', globals=globals(), name="original bytecode (LOAD_GLOBAL)")
bench.timeit(stmt='func_cst("abc")', globals=globals(), name="LOAD_CONST")
fat.specialize(func, func_cst, [fat.GuardGlobals(("mylen",))])
assert fat.get_specialized(func)
bench.timeit(stmt='func("abc")', globals=globals(), name="LOAD_CONST with guard on globals")
def test_invalid_guard_type(self):
def func():
pass
def func2():
pass
self.assertNotSpecialized(func)
self.assertNotSpecialized(func2)
with self.assertRaises(TypeError):
fat.specialize(func, func2, ['xxx'])
def test_invalid_guard_type(self):
def func():
pass
def func2():
pass
self.assertNotSpecialized(func)
self.assertNotSpecialized(func2)
with self.assertRaises(TypeError):
fat.specialize(func, func2, ['xxx'])
def run_benchmark(bench):
bench.timeit(stmt='func("abc")',
globals=globals(),
name='original bytecode (LOAD_GLOBAL)')
bench.timeit(stmt='func_cst("abc")', globals=globals(), name='LOAD_CONST')
fat.specialize(func, func_cst, [fat.GuardGlobals(('mylen', ))])
assert fat.get_specialized(func)
bench.timeit(stmt='func("abc")',
globals=globals(),
name='LOAD_CONST with guard on globals')
def run_benchmark(bench):
bench.timeit('func()',
globals=globals(),
name="original bytecode (call len)")
bench.timeit('func_cst()', globals=globals(), name="return 3")
fat.specialize(func, func_cst, [fat.GuardBuiltins(('len', ))])
assert fat.get_specialized(func)
bench.timeit('func()',
globals=globals(),
name="return 3 with guard on builtins")
def test_arg_type_int(self):
def func(obj):
return 'slow'
def fast(obj):
return 'fast'
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
fat.specialize(func, fast, [fat.GuardArgType(0, (int,))])
self.assertEqual(func(5), 'fast')
self.assertEqual(func("abc"), 'slow')
def test_func_guard(self):
def inlined():
pass
def func():
pass
def func2():
pass
fat.specialize(func, func2, [fat.GuardFunc(inlined)])
guard = fat.GuardFunc(inlined)
self.check_specialized(func, (func2.__code__, [guard]))
def test_arg_type_int(self):
def func(obj):
return 'slow'
def fast(obj):
return 'fast'
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
fat.specialize(func, fast, [fat.GuardArgType(0, (int, ))])
self.assertEqual(func(5), 'fast')
self.assertEqual(func("abc"), 'slow')
def test_func_guard(self):
def inlined():
pass
def func():
pass
def func2():
pass
fat.specialize(func, func2, [fat.GuardFunc(inlined)])
guard = fat.GuardFunc(inlined)
self.check_specialized(func,
(func2.__code__, [guard]))
def test_arg_type_type_list_guard(self):
def func(a, b, c):
return 'slow'
def func2(a, b, c):
return 'fast'
fat.specialize(func, func2, [fat.GuardArgType(2, [list, set])])
guard = fat.GuardArgType(2, (list, set))
self.check_specialized(func, (func2.__code__, [guard]))
# If the guard fails, the specialized function must no be removed
self.assertEqual(func(0, 1, "abc"), 'slow')
self.check_specialized(func, (func2.__code__, [guard]))
def run_benchmark(bench):
bench.timeit('func()',
globals=globals(),
name="original bytecode (call len)")
bench.timeit('func_cst()',
globals=globals(),
name="return 3")
fat.specialize(func, func_cst, [fat.GuardBuiltins(('len',))])
assert fat.get_specialized(func)
bench.timeit('func()',
globals=globals(),
name="return 3 with guard on builtins")
def run_benchmark(bench):
bench.timeit(stmt='func("abc")',
globals=globals(),
name='original bytecode (LOAD_GLOBAL)')
bench.timeit(stmt='func_cst("abc")',
globals=globals(),
name='LOAD_CONST')
fat.specialize(func, func_cst, [fat.GuardBuiltins(('len',))])
assert fat.get_specialized(func)
bench.timeit(stmt='func("abc")',
globals=globals(),
name='LOAD_CONST with guard on builtins and globals')
def test_arg_type_type_list_guard(self):
def func(a, b, c):
return 'slow'
def func2(a, b, c):
return 'fast'
fat.specialize(func, func2, [fat.GuardArgType(2, [list, set])])
guard = fat.GuardArgType(2, (list, set))
self.check_specialized(func,
(func2.__code__, [guard]))
# If the guard fails, the specialized function must no be removed
self.assertEqual(func(0, 1, "abc"), 'slow')
self.check_specialized(func,
(func2.__code__, [guard]))
def test_kwdefaults(self):
def func(*, x=1):
return 1
def fast(*, x=2):
return 2
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
ns = {}
guards = guard_dict(ns, 'key')
with self.assertRaises(ValueError) as cm:
fat.specialize(func, fast, guards)
self.assertEqual(
str(cm.exception), "specialized function doesn't have "
"the same keyword parameter defaults")
def test_arg_type_class(self):
class MyClass:
pass
def func(obj):
return 'slow'
def fast(obj):
return 'fast'
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
fat.specialize(func, fast, [fat.GuardArgType(0, (MyClass, ))])
obj = MyClass()
self.assertEqual(func(obj), 'fast')
self.assertEqual(func("test"), 'slow')
class MyClass:
def meth(self):
return 'slow'
def _fast(self):
return 'fast'
fat.specialize(meth, _fast, guards)
del _fast
def test_arg_type_class(self):
class MyClass:
pass
def func(obj):
return 'slow'
def fast(obj):
return 'fast'
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
fat.specialize(func, fast, [fat.GuardArgType(0, (MyClass,))])
obj = MyClass()
self.assertEqual(func(obj), 'fast')
self.assertEqual(func("test"), 'slow')
def main():
if fat.get_specialized(func) or fat.get_specialized(func2):
print("ERROR: functions already specialized!")
sys.exit(1)
fat.specialize(func2, fast_func2, [fat.GuardArgType(0, (list,))])
for range_pow10 in (0, 1, 3, 5):
print("range(10 ** %s)" % range_pow10)
dt = bench_list('func', range_pow10)
print("- original bytecode: %s" % format_dt(dt))
dt2 = bench_list('func2', range_pow10)
print("- append=obj.append with guards: %s" % compared_dt(dt2, dt))
dt2 = bench_list('fast_func2', range_pow10)
print("- append=obj.append: %s" % compared_dt(dt2, dt))
def test_duplicated(self):
def func():
pass
# register the same function twice
def func2():
pass
self.assertNotSpecialized(func)
self.assertNotSpecialized(func2)
ns = {}
guards = guard_dict(ns, 'key')
fat.specialize(func, func2, guards)
fat.specialize(func, func2, guards)
self.check_specialized(func, (func2.__code__, guards),
(func2.__code__, guards))
def main():
if fat.get_specialized(func) or fat.get_specialized(func2):
print("ERROR: functions already specialized!")
sys.exit(1)
fat.specialize(func2, fast_func2, [fat.GuardArgType(0, (list, ))])
for range_pow10 in (0, 1, 3, 5):
print("range(10 ** %s)" % range_pow10)
dt = bench_list('func', range_pow10)
print("- original bytecode: %s" % format_dt(dt))
dt2 = bench_list('func2', range_pow10)
print("- append=obj.append with guards: %s" % compared_dt(dt2, dt))
dt2 = bench_list('fast_func2', range_pow10)
print("- append=obj.append: %s" % compared_dt(dt2, dt))
def test_kwdefaults(self):
def func(*, x=1):
return 1
def fast(*, x=2):
return 2
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
ns = {}
guards = guard_dict(ns, 'key')
with self.assertRaises(ValueError) as cm:
fat.specialize(func, fast, guards)
self.assertEqual(str(cm.exception),
"specialized function doesn't have "
"the same keyword parameter defaults")
def test_duplicated(self):
def func():
pass
# register the same function twice
def func2():
pass
self.assertNotSpecialized(func)
self.assertNotSpecialized(func2)
ns = {}
guards = guard_dict(ns, 'key')
fat.specialize(func, func2, guards)
fat.specialize(func, func2, guards)
self.check_specialized(func,
(func2.__code__, guards),
(func2.__code__, guards))
def test_add_func_guard_error(self):
with self.assertRaises(TypeError) as cm:
fat.GuardFunc()
with self.assertRaises(TypeError) as cm:
# invalid function type
fat.GuardFunc('abc')
self.assertEqual(str(cm.exception), "func must be a function, not str")
def func():
pass
def func2():
pass
with self.assertRaises(ValueError) as cm:
# must not watch itself
fat.specialize(func, func2, [fat.GuardFunc(func)])
self.assertEqual(str(cm.exception),
"useless GuardFunc, a function already watch itself")
def test_add_func_guard_error(self):
with self.assertRaises(TypeError) as cm:
fat.GuardFunc()
with self.assertRaises(TypeError) as cm:
# invalid function type
fat.GuardFunc('abc')
self.assertEqual(str(cm.exception),
"func must be a function, not str")
def func():
pass
def func2():
pass
with self.assertRaises(ValueError) as cm:
# must not watch itself
fat.specialize(func, func2, [fat.GuardFunc(func)])
self.assertEqual(str(cm.exception),
"useless GuardFunc, a function already watch itself")
def test_kwonlyargcount(self):
def func(*, x=1):
return 1
def fast(*, x=1, y=1):
return 2
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
ns = {}
guards = guard_dict(ns, 'key')
for bytecode in (False, True):
with self.assertRaises(ValueError) as cm:
if bytecode:
fat.specialize(func, fast.__code__, guards)
else:
fat.specialize(func, fast, guards)
self.assertEqual(str(cm.exception),
"specialized bytecode doesn't have the same "
"number of parameters")
def test_kwonlyargcount(self):
def func(*, x=1):
return 1
def fast(*, x=1, y=1):
return 2
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
ns = {}
guards = guard_dict(ns, 'key')
for bytecode in (False, True):
with self.assertRaises(ValueError) as cm:
if bytecode:
fat.specialize(func, fast.__code__, guards)
else:
fat.specialize(func, fast, guards)
self.assertEqual(
str(cm.exception),
"specialized bytecode doesn't have the same "
"number of parameters")
def bench_guards(nguard):
def func():
pass
no_guard = bench(func, number=100)
print("no guard: %s" % format_dt(no_guard))
if fat.get_specialized(func):
print("ERROR: func already specialized")
sys.exit(1)
guards = [fat.GuardDict(globals(), ("global_var",)) for i in range(nguard)]
fat.specialize(func, fast_func, guards)
with_guards = bench(func)
print("with %s guards on globals: %s" % (nguard, format_dt(with_guards)))
dt = with_guards - no_guard
print("cost of %s guards: %s (%.1f%%)" % (nguard, format_dt(dt), dt * 100 / no_guard))
dt = dt / nguard
print("average cost of 1 guard: %s (%.1f%%)" % (format_dt(dt), dt * 100 / no_guard))
print()
def test_dict_guard(self):
ns = dict(mykey=4)
def func():
return 'slow'
def func2():
return 'fast'
guards = guard_dict(ns, 'mykey')
fat.specialize(func, func2, guards)
self.check_specialized(func,
(func2.__code__, guards))
# Modify func, so the guard will fail
def mock_func():
return 'mock'
func.__code__ = mock_func.__code__
# Calling the function checks the guards and then removed the
# specialized function since the function was modified
self.assertEqual(func(), 'mock')
self.assertEqual(fat.get_specialized(func), [])
def test_modify_func_code(self):
def func():
return "slow"
def fast():
return "fast"
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
ns = {}
guards = guard_dict(ns, 'key')
fat.specialize(func, fast, guards)
self.assertEqual(func(), 'fast')
self.assertEqual(len(fat.get_specialized(func)), 1)
def mock_func():
return 'mock'
# setting __code__ must disable all optimizations
func.__code__ = mock_func.__code__
self.assertEqual(func(), 'mock')
self.assertEqual(len(fat.get_specialized(func)), 0)
def test_specialize_with_cellvars(self):
def func(data, cb):
return 3
def fast(data, cb):
return [cb(item) for item in data]
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
self.assertEqual(func.__code__.co_cellvars, ())
self.assertNotEqual(fast.__code__.co_cellvars, ())
ns = {}
guards = guard_dict(ns, 'key')
for bytecode in (False, True):
with self.assertRaises(ValueError) as cm:
if bytecode:
fat.specialize(func, fast.__code__, guards)
else:
fat.specialize(func, fast, guards)
self.assertEqual(str(cm.exception),
"specialized bytecode uses different cell variables")
def test_dict_guard(self):
ns = dict(mykey=4)
def func():
return 'slow'
def func2():
return 'fast'
guards = guard_dict(ns, 'mykey')
fat.specialize(func, func2, guards)
self.check_specialized(func, (func2.__code__, guards))
# Modify func, so the guard will fail
def mock_func():
return 'mock'
func.__code__ = mock_func.__code__
# Calling the function checks the guards and then removed the
# specialized function since the function was modified
self.assertEqual(func(), 'mock')
self.assertEqual(fat.get_specialized(func), [])
def test_over_specialized(self):
def func():
pass
def func2():
pass
def func3():
pass
self.assertNotSpecialized(func)
self.assertNotSpecialized(func2)
self.assertNotSpecialized(func3)
ns = {}
guards = guard_dict(ns, 'key')
fat.specialize(func2, func3, guards)
with self.assertRaises(ValueError) as cm:
fat.specialize(func, func2, guards)
self.assertEqual(str(cm.exception),
'cannot specialize a function with another function '
'which is already specialized')
def test_modify_func_code(self):
def func():
return "slow"
def fast():
return "fast"
self.assertNotSpecialized(func)
self.assertNotSpecialized(fast)
ns = {}
guards = guard_dict(ns, 'key')
fat.specialize(func, fast, guards)
self.assertEqual(func(), 'fast')
self.assertEqual(len(fat.get_specialized(func)), 1)
def mock_func():
return 'mock'
# setting __code__ must disable all optimizations
func.__code__ = mock_func.__code__
self.assertEqual(func(), 'mock')
self.assertEqual(len(fat.get_specialized(func)), 0)
"""Test whether a path is absolute"""
sep = _get_sep(s)
return s.startswith(sep)
def fast_isabs(s):
"""Test whether a path is absolute"""
sep = _get_sep(s)
return s.startswith(sep)
# Manually inline _get_sep() in isabs() depending on the type of the s argument
def isabs_str(s):
return s.startswith('/')
for func in (_get_sep, isabs, fast_isabs, isabs_str):
if fat.get_specialized(func):
print("ERROR: a function is already specialized!")
sys.exit(1)
fat.specialize(fast_isabs, isabs_str,
[fat.GuardArgType(0, (str,)),
fat.GuardGlobals(('_get_sep',)),
fat.GuardBuiltins(('isinstance',)),
fat.GuardFunc(_get_sep)])
dt = bench("isabs('/abc')")
print("original isabs() bytecode: %s" % format_dt(dt))
dt2 = bench("fast_isabs('/abc')")
print("_get_sep() inlined in isabs(): %s" % compared_dt(dt2, dt))