def __init__(self, pyfunc, signature):
self.py_func = pyfunc
self.signature = signature
self.name = pyfunc.__name__
# recreate for each UDF, as linking is destructive to the
# precompiled module
impala_typing = impala_typing_context()
impala_targets = ImpalaTargetContext(impala_typing)
args, return_type = sigutils.normalize_signature(signature)
flags = Flags()
flags.set('no_compile')
self._cres = compile_extra(typingctx=impala_typing,
targetctx=impala_targets,
func=pyfunc,
args=args,
return_type=return_type,
flags=flags,
locals={})
llvm_func = impala_targets.finalize(self._cres.llvm_func, return_type,
args)
self.llvm_func = llvm_func
# numba_module = llvm_func.module
self.llvm_module = llvm_func.module
# link in the precompiled module
# bc it's destructive, load a fresh version
precompiled = lc.Module.from_bitcode(
pkgutil.get_data("impala.udf", "precompiled/impyla.bc"))
self.llvm_module.link_in(precompiled)
def test_array_expr(self):
flags = Flags()
flags.set("enable_pyobject")
global cnd_array_jitted
scalty = types.float64
arrty = types.Array(scalty, 1, 'C')
cr1 = compile_isolated(cnd_array, args=(arrty, ), flags=flags)
cnd_array_jitted = cr1.entry_point
cr2 = compile_isolated(blackscholes_arrayexpr_jitted,
args=(arrty, arrty, arrty, scalty, scalty),
flags=flags)
jitted_bs = cr2.entry_point
OPT_N = 400
iterations = 10
stockPrice = randfloat(self.random.random_sample(OPT_N), 5.0, 30.0)
optionStrike = randfloat(self.random.random_sample(OPT_N), 1.0, 100.0)
optionYears = randfloat(self.random.random_sample(OPT_N), 0.25, 10.0)
args = stockPrice, optionStrike, optionYears, RISKFREE, VOLATILITY
callResultGold, putResultGold = blackscholes_arrayexpr(*args)
callResultNumba, putResultNumba = jitted_bs(*args)
delta = np.abs(callResultGold - callResultNumba)
L1norm = delta.sum() / np.abs(callResultGold).sum()
print("L1 norm: %E" % L1norm)
print("Max absolute error: %E" % delta.max())
self.assertEqual(delta.max(), 0)
def test_array_expr(self):
flags = Flags()
flags.set("enable_pyobject")
global cnd_array_jitted
scalty = types.float64
arrty = types.Array(scalty, 1, 'C')
cr1 = compile_isolated(cnd_array, args=(arrty,), flags=flags)
cnd_array_jitted = cr1.entry_point
cr2 = compile_isolated(blackscholes_arrayexpr_jitted,
args=(arrty, arrty, arrty, scalty, scalty),
flags=flags)
jitted_bs = cr2.entry_point
OPT_N = 400
iterations = 10
stockPrice = randfloat(self.random.random_sample(OPT_N), 5.0, 30.0)
optionStrike = randfloat(self.random.random_sample(OPT_N), 1.0, 100.0)
optionYears = randfloat(self.random.random_sample(OPT_N), 0.25, 10.0)
args = stockPrice, optionStrike, optionYears, RISKFREE, VOLATILITY
callResultGold, putResultGold = blackscholes_arrayexpr(*args)
callResultNumba, putResultNumba = jitted_bs(*args)
delta = np.abs(callResultGold - callResultNumba)
L1norm = delta.sum() / np.abs(callResultGold).sum()
print("L1 norm: %E" % L1norm)
print("Max absolute error: %E" % delta.max())
self.assertEqual(delta.max(), 0)
def test_exercise_code_path_with_lifted_loop(self):
"""
Ensures that lifted loops are handled correctly in obj mode
"""
# the functions to jit
def bar(x):
return x
def foo(x):
h = 0.
for k in range(x):
h = h + k
if x:
h = h - bar(x)
return h
# compile into an isolated context
flags = Flags()
flags.set('enable_pyobject')
flags.set('enable_looplift')
cres = compile_isolated(foo, [types.intp], flags=flags)
ta = cres.type_annotation
buf = StringIO()
ta.html_annotate(buf)
output = buf.getvalue()
buf.close()
self.assertIn("bar", output)
self.assertIn("foo", output)
self.assertIn("LiftedLoop", output)
def __init__(self, pyfunc, signature):
self.py_func = pyfunc
self.signature = signature
self.name = pyfunc.__name__
# recreate for each UDF, as linking is destructive to the
# precompiled module
impala_typing = impala_typing_context()
impala_targets = ImpalaTargetContext(impala_typing)
args, return_type = sigutils.normalize_signature(signature)
flags = Flags()
flags.set('no_compile')
self._cres = compile_extra(typingctx=impala_typing,
targetctx=impala_targets, func=pyfunc,
args=args, return_type=return_type,
flags=flags, locals={})
llvm_func = impala_targets.finalize(self._cres.llvm_func, return_type,
args)
self.llvm_func = llvm_func
# numba_module = llvm_func.module
self.llvm_module = llvm_func.module
# link in the precompiled module
# bc it's destructive, load a fresh version
precompiled = lc.Module.from_bitcode(
pkgutil.get_data("impala.udf", "precompiled/impyla.bc"))
self.llvm_module.link_in(precompiled)
def test_exercise_code_path_with_lifted_loop(self):
"""
Ensures that lifted loops are handled correctly in obj mode
"""
# the functions to jit
def bar(x):
return x
def foo(x):
h = 0.
for k in range(x):
h = h + k
if x:
h = h - bar(x)
return h
# compile into an isolated context
flags = Flags()
flags.set('enable_pyobject')
flags.set('enable_looplift')
cres = compile_isolated(foo, [types.intp], flags=flags)
ta = cres.type_annotation
buf = StringIO()
ta.html_annotate(buf)
output = buf.getvalue()
buf.close()
self.assertIn("bar", output)
self.assertIn("foo", output)
self.assertIn("LiftedLoop", output)
def _cull_exports(self):
"""Read all the exported functions/modules in the translator
environment, and join them into a single LLVM module.
Resets the export environment afterwards.
"""
self.exported_signatures = export_registry
# Create new module containing everything
llvm_module = lc.Module.new(self.module_name)
# Compile all exported functions
typing_ctx = CPUTarget.typing_context
# TODO Use non JIT-ing target
target_ctx = CPUTarget.target_context
modules = []
flags = Flags()
if not self.export_python_wrap:
flags.set("no_compile")
for entry in self.exported_signatures:
cres = compile_extra(typing_ctx,
target_ctx,
entry.function,
entry.signature.args,
entry.signature.return_type,
flags,
locals={})
if self.export_python_wrap:
module = cres.llvm_func.module
cres.llvm_func.linkage = lc.LINKAGE_INTERNAL
wrappername = "wrapper." + cres.llvm_func.name
wrapper = module.get_function_named(wrappername)
wrapper.name = entry.symbol
else:
cres.llvm_func.name = entry.symbol
modules.append(cres.llvm_module)
# Link all exported functions
for mod in modules:
llvm_module.link_in(mod, preserve=self.export_python_wrap)
# Optimize
tm = le.TargetMachine.new(opt=3)
pms = lp.build_pass_managers(tm=tm,
opt=3,
loop_vectorize=True,
fpm=False)
pms.pm.run(llvm_module)
if self.export_python_wrap:
self._emit_python_wrapper(llvm_module)
del self.exported_signatures[:]
print(llvm_module)
return llvm_module
def test_is_this_a_none_objmode(self):
pyfunc = is_this_a_none
flags = Flags()
flags.set('force_pyobject')
cres = compile_isolated(pyfunc, [types.intp], flags=flags)
cfunc = cres.entry_point
self.assertTrue(cres.objectmode)
for v in [-1, 0, 1, 2]:
self.assertPreciseEqual(pyfunc(v), cfunc(v))
def __init__(self, *args):
self.flags = Flags()
self.flags.set('nrt')
# flags for njit(fastmath=True)
self.fastflags = Flags()
self.fastflags.set('nrt')
self.fastflags.set('fastmath')
super(TestSVML, self).__init__(*args)
def __init__(self, *args):
# flags for njit()
self.cflags = Flags()
self.cflags.set('nrt')
# flags for njit(parallel=True)
self.pflags = Flags()
self.pflags.set('auto_parallel', cpu.ParallelOptions(True))
self.pflags.set('nrt')
super(TestParforsBase, self).__init__(*args)
def autogenerate(cls):
test_flags = ['fastmath', ] # TODO: add 'auto_parallel' ?
# generate all the combinations of the flags
test_flags = sum([list(combinations(test_flags, x)) for x in range( \
len(test_flags)+1)], [])
flag_list = [] # create Flag class instances
for ft in test_flags:
flags = Flags()
flags.set('nrt')
flags.set('error_model', 'numpy')
flags.__name__ = '_'.join(ft+('usecase',))
for f in ft:
flags.set(f)
flag_list.append(flags)
# main loop covering all the modes and use-cases
for dtype in ('complex64', 'float64', 'float32', 'int32', ):
for vlen in vlen2cpu:
for flags in flag_list:
for mode in "scalar", "range", "prange", "numpy":
cls._inject_test(dtype, mode, vlen, flags)
# mark important
if sys.version_info[0] > 2:
for n in ( "test_int32_range4_usecase", # issue #3016
):
setattr(cls, n, tag("important")(getattr(cls, n)))
def mk_pipeline(cls, args, return_type=None, flags=None, locals={},
library=None, typing_context=None, target_context=None):
if not flags:
flags = Flags()
flags.nrt = True
if typing_context is None:
typing_context = registry.cpu_target.typing_context
if target_context is None:
target_context = registry.cpu_target.target_context
return cls(typing_context, target_context, library, args, return_type,
flags, locals)
def mk_pipeline(cls, args, return_type=None, flags=None, locals={},
library=None, typing_context=None, target_context=None):
if not flags:
flags = Flags()
flags.nrt = True
if typing_context is None:
typing_context = typing.Context()
if target_context is None:
target_context = cpu.CPUContext(typing_context)
return cls(typing_context, target_context, library, args, return_type,
flags, locals)
def mk_pipeline(cls, args, return_type=None, flags=None, locals={},
library=None, typing_context=None, target_context=None):
if not flags:
flags = Flags()
flags.nrt = True
if typing_context is None:
typing_context = registry.cpu_target.typing_context
if target_context is None:
target_context = registry.cpu_target.target_context
return cls(typing_context, target_context, library, args, return_type,
flags, locals)
def test_raise_object(self):
args = [types.int32]
flags = Flags()
flags.set('force_pyobject')
cres = compile_isolated(pyfunc, args,
flags=flags)
cfunc = cres.entry_point
cfunc(0)
with self.assertRaises(MyError):
cfunc(1)
def mk_pipeline(cls, args, return_type=None, flags=None, locals={},
library=None, typing_context=None, target_context=None):
if not flags:
flags = Flags()
flags.nrt = True
if typing_context is None:
typing_context = typing.Context()
if target_context is None:
target_context = cpu.CPUContext(typing_context)
return cls(typing_context, target_context, library, args, return_type,
flags, locals)
def test_usecase(self):
n = 10
obs_got = np.zeros(n)
obs_expected = obs_got.copy()
flags = Flags()
flags.set("enable_pyobject")
cres = compile_isolated(usecase, (), flags=flags)
cres.entry_point(obs_got, n, 10.0, 1.0, 2.0, 3.0, 4.0, 5.0)
usecase(obs_expected, n, 10.0, 1.0, 2.0, 3.0, 4.0, 5.0)
self.assertTrue(np.allclose(obs_got, obs_expected))
def _cull_exports(self):
"""Read all the exported functions/modules in the translator
environment, and join them into a single LLVM module.
Resets the export environment afterwards.
"""
self.exported_signatures = export_registry
# Create new module containing everything
llvm_module = lc.Module.new(self.module_name)
# Compile all exported functions
typing_ctx = CPUTarget.typing_context
# TODO Use non JIT-ing target
target_ctx = CPUTarget.target_context
modules = []
flags = Flags()
if not self.export_python_wrap:
flags.set("no_compile")
for entry in self.exported_signatures:
cres = compile_extra(typing_ctx, target_ctx, entry.function,
entry.signature.args,
entry.signature.return_type, flags,
locals={})
if self.export_python_wrap:
module = cres.llvm_func.module
cres.llvm_func.linkage = lc.LINKAGE_INTERNAL
wrappername = "wrapper." + cres.llvm_func.name
wrapper = module.get_function_named(wrappername)
wrapper.name = entry.symbol
else:
cres.llvm_func.name = entry.symbol
modules.append(cres.llvm_module)
# Link all exported functions
for mod in modules:
llvm_module.link_in(mod, preserve=self.export_python_wrap)
# Optimize
tm = le.TargetMachine.new(opt=3)
pms = lp.build_pass_managers(tm=tm, opt=3, loop_vectorize=True,
fpm=False)
pms.pm.run(llvm_module)
if self.export_python_wrap:
self._emit_python_wrapper(llvm_module)
#del self.exported_signatures[:]
print(llvm_module)
return llvm_module
def mk_no_rw_pipeline(cls,
args,
return_type=None,
flags=None,
locals={},
library=None,
**kws):
if not flags:
flags = Flags()
flags.no_rewrites = True
return cls.mk_pipeline(args, return_type, flags, locals, library,
**kws)
def test_usecase(self):
n = 10
obs_got = np.zeros(n)
obs_expected = obs_got.copy()
flags = Flags()
flags.set("enable_pyobject")
cres = compile_isolated(usecase, (), flags=flags)
cres.entry_point(obs_got, n, 10.0, 1.0, 2.0, 3.0, 4.0, 5.0)
usecase(obs_expected, n, 10.0, 1.0, 2.0, 3.0, 4.0, 5.0)
self.assertTrue(np.allclose(obs_got, obs_expected))
def _cull_exports(self):
"""Read all the exported functions/modules in the translator
environment, and join them into a single LLVM module.
Resets the export environment afterwards.
"""
self.exported_signatures = export_registry
self.exported_function_types = {}
typing_ctx = CPUTarget.typing_context
target_ctx = CPUTarget.target_context.subtarget(aot_mode=True)
codegen = target_ctx.aot_codegen(self.module_name)
library = codegen.create_library(self.module_name)
# Generate IR for all exported functions
flags = Flags()
flags.set("no_compile")
for entry in self.exported_signatures:
cres = compile_extra(typing_ctx,
target_ctx,
entry.function,
entry.signature.args,
entry.signature.return_type,
flags,
locals={},
library=library)
func_name = cres.fndesc.llvm_func_name
llvm_func = cres.library.get_function(func_name)
if self.export_python_wrap:
# XXX: unsupported (necessary?)
llvm_func.linkage = lc.LINKAGE_INTERNAL
wrappername = cres.fndesc.llvm_cpython_wrapper_name
wrapper = cres.library.get_function(wrappername)
wrapper.name = entry.symbol
wrapper.linkage = lc.LINKAGE_EXTERNAL
fnty = cres.target_context.call_conv.get_function_type(
cres.fndesc.restype, cres.fndesc.argtypes)
self.exported_function_types[entry] = fnty
else:
llvm_func.linkage = lc.LINKAGE_EXTERNAL
llvm_func.name = entry.symbol
if self.export_python_wrap:
wrapper_module = library.create_ir_module("wrapper")
self._emit_python_wrapper(wrapper_module)
library.add_ir_module(wrapper_module)
return library
def test_usecase(self):
n = 10
obs_got = np.zeros(n)
obs_expected = obs_got.copy()
flags = Flags()
flags.set("enable_pyobject")
cres = compile_isolated(usecase, (types.float64[:], types.intp),
flags=flags)
cres.entry_point(obs_got, n)
usecase(obs_expected, n)
self.assertTrue(np.allclose(obs_got, obs_expected))
def test_usecase(self):
n = 10
obs_got = np.zeros(n)
obs_expected = obs_got.copy()
flags = Flags()
flags.set("nrt")
cres = compile_isolated(usecase, (types.float64[:], types.intp),
flags=flags)
cres.entry_point(obs_got, n)
usecase(obs_expected, n)
self.assertPreciseEqual(obs_got, obs_expected)
def test_usecase(self):
n = 10
obs_got = np.zeros(n)
obs_expected = obs_got.copy()
flags = Flags()
flags.set("enable_pyobject")
cres = compile_isolated(usecase, (types.float64[:], types.intp),
flags=flags)
cres.entry_point(obs_got, n)
usecase(obs_expected, n)
self.assertTrue(np.allclose(obs_got, obs_expected))
def _cull_exports(self):
"""Read all the exported functions/modules in the translator
environment, and join them into a single LLVM module.
"""
self.exported_function_types = {}
self.function_environments = {}
codegen = self.context.codegen()
library = codegen.create_library(self.module_name)
# Generate IR for all exported functions
flags = Flags()
flags.set("no_compile")
if not self.export_python_wrap:
flags.set("no_cpython_wrapper")
if self.use_nrt:
flags.set("nrt")
# Compile NRT helpers
nrt_module, _ = atomicops.create_nrt_module(self.context)
library.add_ir_module(nrt_module)
for entry in self.export_entries:
cres = compile_extra(self.typing_context,
self.context,
entry.function,
entry.signature.args,
entry.signature.return_type,
flags,
locals={},
library=library)
func_name = cres.fndesc.llvm_func_name
llvm_func = cres.library.get_function(func_name)
if self.export_python_wrap:
llvm_func.linkage = lc.LINKAGE_INTERNAL
wrappername = cres.fndesc.llvm_cpython_wrapper_name
wrapper = cres.library.get_function(wrappername)
wrapper.name = self._mangle_method_symbol(entry.symbol)
wrapper.linkage = lc.LINKAGE_EXTERNAL
fnty = cres.target_context.call_conv.get_function_type(
cres.fndesc.restype, cres.fndesc.argtypes)
self.exported_function_types[entry] = fnty
self.function_environments[entry] = cres.environment
else:
llvm_func.name = entry.symbol
self.dll_exports.append(entry.symbol)
if self.export_python_wrap:
wrapper_module = library.create_ir_module("wrapper")
self._emit_python_wrapper(wrapper_module)
library.add_ir_module(wrapper_module)
# Hide all functions in the DLL except those explicitly exported
library.finalize()
for fn in library.get_defined_functions():
if fn.name not in self.dll_exports:
fn.visibility = "hidden"
return library
def _cull_exports(self):
"""Read all the exported functions/modules in the translator
environment, and join them into a single LLVM module.
Resets the export environment afterwards.
"""
self.exported_signatures = export_registry
self.exported_function_types = {}
typing_ctx = CPUTarget.typing_context
target_ctx = CPUTarget.target_context
codegen = target_ctx.aot_codegen(self.module_name)
library = codegen.create_library(self.module_name)
# Generate IR for all exported functions
flags = Flags()
flags.set("no_compile")
for entry in self.exported_signatures:
cres = compile_extra(typing_ctx, target_ctx, entry.function,
entry.signature.args,
entry.signature.return_type, flags,
locals={}, library=library)
func_name = cres.fndesc.llvm_func_name
llvm_func = cres.library.get_function(func_name)
if self.export_python_wrap:
# XXX: unsupported (necessary?)
llvm_func.linkage = lc.LINKAGE_INTERNAL
wrappername = cres.fndesc.llvm_cpython_wrapper_name
wrapper = cres.library.get_function(wrappername)
wrapper.name = entry.symbol
wrapper.linkage = lc.LINKAGE_EXTERNAL
fnty = cres.target_context.call_conv.get_function_type(
cres.fndesc.restype, cres.fndesc.argtypes)
self.exported_function_types[entry] = fnty
else:
llvm_func.linkage = lc.LINKAGE_EXTERNAL
llvm_func.name = entry.symbol
if self.export_python_wrap:
wrapper_module = library.create_ir_module("wrapper")
self._emit_python_wrapper(wrapper_module)
library.add_ir_module(wrapper_module)
return library
def __init__(self, pyfunc, signature):
self.py_func = pyfunc
self.signature = signature
self.name = pyfunc.__name__
args, return_type = sigutils.normalize_signature(signature)
flags = Flags()
flags.set('no_compile')
self._cres = compile_extra(typingctx=impala_typing,
targetctx=impala_targets, func=pyfunc,
args=args, return_type=return_type,
flags=flags, locals={})
llvm_func = impala_targets.finalize(self._cres.llvm_func, return_type,
args)
self.llvm_func = llvm_func
self.llvm_module = llvm_func.module
def test_scalar(self):
flags = Flags()
global cnd_jitted
cr1 = compile_isolated(cnd, (types.float64, ))
cnd_jitted = cr1.entry_point
tyctx = cr1.typing_context
ctx = cr1.target_context
ctx.dynamic_map_function(cnd_jitted)
tyctx.insert_user_function(cnd_jitted,
ctx.get_user_function(cnd_jitted))
array = types.Array(types.float64, 1, 'C')
argtys = (array, ) * 5 + (types.float64, types.float64)
cr2 = compile_extra(tyctx,
ctx,
blackscholes_scalar_jitted,
args=argtys,
return_type=None,
flags=flags,
locals={})
jitted_bs = cr2.entry_point
OPT_N = 400
iterations = 10
callResultGold = np.zeros(OPT_N)
putResultGold = np.zeros(OPT_N)
callResultNumba = np.zeros(OPT_N)
putResultNumba = np.zeros(OPT_N)
stockPrice = randfloat(np.random.random(OPT_N), 5.0, 30.0)
optionStrike = randfloat(np.random.random(OPT_N), 1.0, 100.0)
optionYears = randfloat(np.random.random(OPT_N), 0.25, 10.0)
args = stockPrice, optionStrike, optionYears, RISKFREE, VOLATILITY
ts = timer()
for i in range(iterations):
blackscholes_scalar(callResultGold, putResultGold, *args)
te = timer()
pytime = te - ts
ts = timer()
for i in range(iterations):
jitted_bs(callResultNumba, putResultNumba, *args)
te = timer()
jittime = te - ts
print("Python", pytime)
print("Numba", jittime)
print("Speedup: %s" % (pytime / jittime))
delta = np.abs(callResultGold - callResultNumba)
L1norm = delta.sum() / np.abs(callResultGold).sum()
print("L1 norm: %E" % L1norm)
print("Max absolute error: %E" % delta.max())
self.assertAlmostEqual(delta.max(), 0)
def _cull_exports(self):
"""Read all the exported functions/modules in the translator
environment, and join them into a single LLVM module.
"""
self.exported_function_types = {}
self.function_environments = {}
self.environment_gvs = {}
codegen = self.context.codegen()
library = codegen.create_library(self.module_name)
# Generate IR for all exported functions
flags = Flags()
flags.set("no_compile")
if not self.export_python_wrap:
flags.set("no_cpython_wrapper")
if self.use_nrt:
flags.set("nrt")
# Compile NRT helpers
nrt_module, _ = nrtdynmod.create_nrt_module(self.context)
library.add_ir_module(nrt_module)
for entry in self.export_entries:
cres = compile_extra(self.typing_context, self.context,
entry.function,
entry.signature.args,
entry.signature.return_type, flags,
locals={}, library=library)
func_name = cres.fndesc.llvm_func_name
llvm_func = cres.library.get_function(func_name)
if self.export_python_wrap:
llvm_func.linkage = lc.LINKAGE_INTERNAL
wrappername = cres.fndesc.llvm_cpython_wrapper_name
wrapper = cres.library.get_function(wrappername)
wrapper.name = self._mangle_method_symbol(entry.symbol)
wrapper.linkage = lc.LINKAGE_EXTERNAL
fnty = cres.target_context.call_conv.get_function_type(
cres.fndesc.restype, cres.fndesc.argtypes)
self.exported_function_types[entry] = fnty
self.function_environments[entry] = cres.environment
self.environment_gvs[entry] = cres.fndesc.env_name
else:
llvm_func.name = entry.symbol
self.dll_exports.append(entry.symbol)
if self.export_python_wrap:
wrapper_module = library.create_ir_module("wrapper")
self._emit_python_wrapper(wrapper_module)
library.add_ir_module(wrapper_module)
# Hide all functions in the DLL except those explicitly exported
library.finalize()
for fn in library.get_defined_functions():
if fn.name not in self.dll_exports:
fn.visibility = "hidden"
return library
def __init__(self, pyfunc, signature):
self.py_func = pyfunc
self.signature = signature
self.name = pyfunc.__name__
args, return_type = sigutils.normalize_signature(signature)
flags = Flags()
flags.set('no_compile')
self._cres = compile_extra(typingctx=impala_typing,
targetctx=impala_targets,
func=pyfunc,
args=args,
return_type=return_type,
flags=flags,
locals={})
llvm_func = impala_targets.finalize(self._cres.llvm_func, return_type,
args)
self.llvm_func = llvm_func
self.llvm_module = llvm_func.module
def test_array_expr(self):
flags = Flags()
flags.set("enable_pyobject")
global cnd_array_jitted
cr1 = compile_isolated(cnd_array, args=(), flags=flags)
cnd_array_jitted = cr1.entry_point
cr2 = compile_isolated(blackscholes_arrayexpr_jitted, args=(),
flags=flags)
jitted_bs = cr2.entry_point
OPT_N = 400
iterations = 10
stockPrice = randfloat(self.random.random_sample(OPT_N), 5.0, 30.0)
optionStrike = randfloat(self.random.random_sample(OPT_N), 1.0, 100.0)
optionYears = randfloat(self.random.random_sample(OPT_N), 0.25, 10.0)
args = stockPrice, optionStrike, optionYears, RISKFREE, VOLATILITY
ts = timer()
for i in range(iterations):
callResultGold, putResultGold = blackscholes_arrayexpr(*args)
te = timer()
pytime = te - ts
ts = timer()
for i in range(iterations):
callResultNumba, putResultNumba = jitted_bs(*args)
te = timer()
jittime = te - ts
print("Python", pytime)
print("Numba", jittime)
print("Speedup: %s" % (pytime / jittime))
delta = np.abs(callResultGold - callResultNumba)
L1norm = delta.sum() / np.abs(callResultGold).sum()
print("L1 norm: %E" % L1norm)
print("Max absolute error: %E" % delta.max())
self.assertEqual(delta.max(), 0)
def test_array_expr(self):
flags = Flags()
flags.set("enable_pyobject")
global cnd_array_jitted
cr1 = compile_isolated(cnd_array, args=(), flags=flags)
cnd_array_jitted = cr1.entry_point
cr2 = compile_isolated(blackscholes_arrayexpr_jitted,
args=(),
flags=flags)
jitted_bs = cr2.entry_point
OPT_N = 400
iterations = 10
stockPrice = randfloat(np.random.random(OPT_N), 5.0, 30.0)
optionStrike = randfloat(np.random.random(OPT_N), 1.0, 100.0)
optionYears = randfloat(np.random.random(OPT_N), 0.25, 10.0)
args = stockPrice, optionStrike, optionYears, RISKFREE, VOLATILITY
ts = timer()
for i in range(iterations):
callResultGold, putResultGold = blackscholes_arrayexpr(*args)
te = timer()
pytime = te - ts
ts = timer()
for i in range(iterations):
callResultNumba, putResultNumba = jitted_bs(*args)
te = timer()
jittime = te - ts
print("Python", pytime)
print("Numba", jittime)
print("Speedup: %s" % (pytime / jittime))
delta = np.abs(callResultGold - callResultNumba)
L1norm = delta.sum() / np.abs(callResultGold).sum()
print("L1 norm: %E" % L1norm)
print("Max absolute error: %E" % delta.max())
self.assertEqual(delta.max(), 0)
def test_scalar(self):
flags = Flags()
# Compile the inner function
global cnd_jitted
cr1 = compile_isolated(cnd, (types.float64, ))
cnd_jitted = cr1.entry_point
# Manually type the compiled function for calling into
tyctx = cr1.typing_context
ctx = cr1.target_context
signature = typing.make_concrete_template("cnd_jitted", cnd_jitted,
[cr1.signature])
tyctx.insert_user_function(cnd_jitted, signature)
# Compile the outer function
array = types.Array(types.float64, 1, 'C')
argtys = (array, ) * 5 + (types.float64, types.float64)
cr2 = compile_extra(tyctx,
ctx,
blackscholes_scalar_jitted,
args=argtys,
return_type=None,
flags=flags,
locals={})
jitted_bs = cr2.entry_point
OPT_N = 400
iterations = 10
callResultGold = np.zeros(OPT_N)
putResultGold = np.zeros(OPT_N)
callResultNumba = np.zeros(OPT_N)
putResultNumba = np.zeros(OPT_N)
stockPrice = randfloat(self.random.random_sample(OPT_N), 5.0, 30.0)
optionStrike = randfloat(self.random.random_sample(OPT_N), 1.0, 100.0)
optionYears = randfloat(self.random.random_sample(OPT_N), 0.25, 10.0)
args = stockPrice, optionStrike, optionYears, RISKFREE, VOLATILITY
blackscholes_scalar(callResultGold, putResultGold, *args)
jitted_bs(callResultNumba, putResultNumba, *args)
delta = np.abs(callResultGold - callResultNumba)
L1norm = delta.sum() / np.abs(callResultGold).sum()
print("L1 norm: %E" % L1norm)
print("Max absolute error: %E" % delta.max())
self.assertAlmostEqual(delta.max(), 0)
def autogenerate(cls):
test_flags = ['fastmath', ] # TODO: add 'auto_parallel' ?
# generate all the combinations of the flags
test_flags = sum([list(combinations(test_flags, x)) for x in range( \
len(test_flags)+1)], [])
flag_list = [] # create Flag class instances
for ft in test_flags:
flags = Flags()
flags.set('nrt')
flags.set('error_model', 'numpy')
flags.__name__ = '_'.join(ft+('usecase',))
for f in ft:
flags.set(f)
flag_list.append(flags)
# main loop covering all the modes and use-cases
for dtype in ('float64', 'float32'):
for vlen in vlen2cpu:
for flags in flag_list:
for mode in "scalar", "range", "prange", "numpy":
cls._inject_test(dtype, mode, vlen, flags)
# Tests numpy methods of <class 'function'>
from __future__ import print_function, absolute_import, division
import itertools
import math
import sys
import numpy as np
from numba import unittest_support as unittest
from numba.compiler import compile_isolated, Flags, utils
from numba import types
from .support import TestCase, CompilationCache
no_pyobj_flags = Flags()
no_pyobj_flags.set("nrt")
no_pyobj_flags = [no_pyobj_flags]
def sinc(x):
return np.sinc(x)
def angle(x, deg):
return np.angle(x, deg)
class TestNPFunctions(TestCase):
"""
Contains tests and test helpers for numpy methods the are of type
"class< 'function' >.
"""
def setUp(self):
from __future__ import print_function
import numpy as np
import numba.unittest_support as unittest
from numba.compiler import compile_isolated, Flags
from numba import errors, types, typeof
from .support import TestCase, MemoryLeakMixin, tag
enable_pyobj_flags = Flags()
enable_pyobj_flags.set("enable_pyobject")
force_pyobj_flags = Flags()
force_pyobj_flags.set("force_pyobject")
no_pyobj_flags = Flags()
no_pyobj_flags.set("nrt")
def unpack_list(l):
a, b, c = l
return (a, b, c)
def unpack_shape(a):
x, y, z = a.shape
return x + y + z
def unpack_range():
a, b, c = range(3)
from __future__ import print_function
import numpy as np
import numba.unittest_support as unittest
from numba.compiler import compile_isolated, Flags
from numba import types, utils, njit, errors
from numba.tests import usecases
from .support import TestCase
import decimal
enable_pyobj_flags = Flags()
enable_pyobj_flags.set("enable_pyobject")
Noflags = Flags()
def slicing_1d_usecase(a, start, stop, step):
return a[start:stop:step]
def slicing_1d_usecase2(a, start, stop, step):
b = a[start:stop:step]
total = 0
for i in range(b.shape[0]):
total += b[i] * (i + 1)
return total
def slicing_1d_usecase3(a, start, stop):
b = a[start:stop]
import itertools
import operator
import warnings
import numpy as np
from numba.compiler import compile_isolated, Flags
from numba import types, typeinfer, utils
from numba.config import PYVERSION
from .support import TestCase
from numba.tests.true_div_usecase import truediv_usecase, itruediv_usecase
import numba.unittest_support as unittest
Noflags = Flags()
force_pyobj_flags = Flags()
force_pyobj_flags.set("enable_pyobject")
force_pyobj_flags = Flags()
force_pyobj_flags.set("force_pyobject")
class LiteralOperatorImpl(object):
@staticmethod
def add_usecase(x, y):
return x + y
@staticmethod
def iadd_usecase(x, y):
x += y
import numba.unittest_support as unittest
from numba.compiler import compile_isolated, Flags
from numba import types, utils
enable_pyobj_flags = Flags()
enable_pyobj_flags.set("enable_pyobject")
force_pyobj_flags = Flags()
force_pyobj_flags.set("force_pyobject")
def is_in_mandelbrot(c):
i = 0
z = 0.0j
for i in range(100):
z = z**2 + c
if (z.real * z.real + z.imag * z.imag) >= 4:
return False
return True
class TestMandelbrot(unittest.TestCase):
def test_mandelbrot(self):
pyfunc = is_in_mandelbrot
cr = compile_isolated(pyfunc, (types.complex64, ))
cfunc = cr.entry_point
points = [0 + 0j, 1 + 0j, 0 + 1j, 1 + 1j, 0.1 + 0.1j]
for p in points:
self.assertEqual(cfunc(p), pyfunc(p))
import numpy as np
import sys
from numba.compiler import compile_isolated, Flags
from numba import jit, types
from numba import unittest_support as unittest
from .support import TestCase
force_pyobj_flags = Flags()
force_pyobj_flags.set("force_pyobject")
no_pyobj_flags = Flags()
class MyError(Exception):
pass
class OtherError(Exception):
pass
def raise_class(exc):
def raiser(i):
if i == 1:
raise exc
elif i == 2:
raise ValueError
return i
return raiser
from __future__ import absolute_import, division, print_function
import math
import os
import sys
import numpy as np
from numba import unittest_support as unittest
from numba import njit
from numba.compiler import compile_isolated, Flags, types
from numba.runtime import rtsys
from .support import MemoryLeakMixin, TestCase
enable_nrt_flags = Flags()
enable_nrt_flags.set("nrt")
class Dummy(object):
alive = 0
def __init__(self):
type(self).alive += 1
def __del__(self):
type(self).alive -= 1
class TestNrtMemInfo(unittest.TestCase):
"""
Unitest for core MemInfo functionality
from __future__ import print_function
import numba.unittest_support as unittest
import numpy as np
from numba.compiler import compile_isolated, Flags
from numba import types, utils
from numba.tests import usecases
from .support import TestCase
import decimal
enable_pyobj_flags = Flags()
enable_pyobj_flags.set("enable_pyobject")
Noflags = Flags()
def slicing_1d_usecase(a, start, stop, step):
return a[start:stop:step]
def slicing_1d_usecase2(a, start, stop, step):
b = a[start:stop:step]
total = 0
for i in range(b.shape[0]):
total += b[i] * (i + 1)
return total
def slicing_1d_usecase3(a, start, stop):
b = a[start:stop]
total = 0
from __future__ import print_function
import decimal
import itertools
import numpy as np
import numba.unittest_support as unittest
from numba.compiler import compile_isolated, Flags
from numba import types, utils, njit, errors, typeof
from .support import TestCase
enable_pyobj_flags = Flags()
enable_pyobj_flags.set("enable_pyobject")
Noflags = Flags()
Noflags.set("nrt")
def slicing_1d_usecase(a, start, stop, step):
return a[start:stop:step]
def slicing_1d_usecase2(a, start, stop, step):
b = a[start:stop:step]
total = 0
for i in range(b.shape[0]):
total += b[i] * (i + 1)
return total
import numba.unittest_support as unittest
from numba.compiler import compile_isolated, Flags
from numba import utils, jit
from .support import TestCase
def complex_constant(n):
tmp = n + 4
return tmp + 3j
def long_constant(n):
return n + 100000000000000000000000000000000000000000000000
forceobj = Flags()
forceobj.set("force_pyobject")
def loop_nest_3(x, y):
n = 0
for i in range(x):
for j in range(y):
for k in range(x+y):
n += i * j
return n
def array_of_object(x):
return x
simple_class_spec = [('h', types.int32)]
def simple_class_user(obj):
return obj.h
def unsupported_parfor(a, b):
return np.dot(a, b) # dot as gemm unsupported
def supported_parfor(n):
a = np.ones(n)
for i in prange(n):
a[i] = a[i] + np.sin(i)
return a
force_parallel_flags = Flags()
force_parallel_flags.set("auto_parallel", ParallelOptions(True))
force_parallel_flags.set('nrt')
class DebugTestBase(TestCase):
all_dumps = set(['bytecode', 'cfg', 'ir', 'typeinfer', 'llvm',
'func_opt_llvm', 'optimized_llvm', 'assembly'])
def assert_fails(self, *args, **kwargs):
self.assertRaises(AssertionError, *args, **kwargs)
def check_debug_output(self, out, dump_names):
enabled_dumps = dict.fromkeys(self.all_dumps, False)
for name in dump_names:
assert name in enabled_dumps
class TestSVML(TestCase):
""" Tests SVML behaves as expected """
# env mutating, must not run in parallel
_numba_parallel_test_ = False
def __init__(self, *args):
self.flags = Flags()
self.flags.set('nrt')
# flags for njit(fastmath=True)
self.fastflags = Flags()
self.fastflags.set('nrt')
self.fastflags.set('fastmath')
super(TestSVML, self).__init__(*args)
def compile(self, func, *args, **kwargs):
assert not kwargs
sig = tuple([numba.typeof(x) for x in args])
std = compile_isolated(func, sig, flags=self.flags)
fast = compile_isolated(func, sig, flags=self.fastflags)
return std, fast
def copy_args(self, *args):
if not args:
return tuple()
new_args = []
for x in args:
if isinstance(x, np.ndarray):
new_args.append(x.copy('k'))
elif isinstance(x, np.number):
new_args.append(x.copy())
elif isinstance(x, numbers.Number):
new_args.append(x)
else:
raise ValueError('Unsupported argument type encountered')
return tuple(new_args)
def check(self, pyfunc, *args, **kwargs):
jitstd, jitfast = self.compile(pyfunc, *args)
std_pattern = kwargs.pop('std_pattern', None)
fast_pattern = kwargs.pop('fast_pattern', None)
cpu_name = kwargs.pop('cpu_name', 'skylake-avx512')
# python result
py_expected = pyfunc(*self.copy_args(*args))
# jit result
jitstd_result = jitstd.entry_point(*self.copy_args(*args))
# fastmath result
jitfast_result = jitfast.entry_point(*self.copy_args(*args))
# assert numerical equality
np.testing.assert_almost_equal(jitstd_result, py_expected, **kwargs)
np.testing.assert_almost_equal(jitfast_result, py_expected, **kwargs)
# look for specific patters in the asm for a given target
with override_env_config('NUMBA_CPU_NAME', cpu_name), \
override_env_config('NUMBA_CPU_FEATURES', ''):
# recompile for overridden CPU
jitstd, jitfast = self.compile(pyfunc, *args)
if std_pattern:
self.check_svml_presence(jitstd, std_pattern)
if fast_pattern:
self.check_svml_presence(jitfast, fast_pattern)
def check_svml_presence(self, func, pattern):
asm = func.library.get_asm_str()
self.assertIn(pattern, asm)
def test_scalar_context(self):
# SVML will not be used.
pat = '$_sin' if numba.config.IS_OSX else '$sin'
self.check(math_sin_scalar, 7., std_pattern=pat)
self.check(math_sin_scalar, 7., fast_pattern=pat)
@tag('important')
def test_svml(self):
# loops both with and without fastmath should use SVML.
# The high accuracy routines are dropped if `fastmath` is set
std = "__svml_sin8_ha,"
fast = "__svml_sin8," # No `_ha`!
self.check(math_sin_loop, 10, std_pattern=std, fast_pattern=fast)
def test_svml_disabled(self):
code = """if 1:
import os
import numpy as np
import math
def math_sin_loop(n):
ret = np.empty(n, dtype=np.float64)
for x in range(n):
ret[x] = math.sin(np.float64(x))
return ret
def check_no_svml():
try:
# ban the use of SVML
os.environ['NUMBA_DISABLE_INTEL_SVML'] = '1'
# delay numba imports to account for env change as
# numba.__init__ picks up SVML and it is too late by
# then to override using `numba.config`
import numba
from numba import config
from numba.tests.support import override_env_config
from numba.compiler import compile_isolated, Flags
# compile for overridden CPU, with and without fastmath
with override_env_config('NUMBA_CPU_NAME', 'skylake-avx512'), \
override_env_config('NUMBA_CPU_FEATURES', ''):
sig = (numba.int32,)
f = Flags()
f.set('nrt')
std = compile_isolated(math_sin_loop, sig, flags=f)
f.set('fastmath')
fast = compile_isolated(math_sin_loop, sig, flags=f)
fns = std, fast
# assert no SVML call is present in the asm
for fn in fns:
asm = fn.library.get_asm_str()
assert '__svml_sin' not in asm
finally:
# not really needed as process is separate
os.environ['NUMBA_DISABLE_INTEL_SVML'] = '0'
config.reload_config()
check_no_svml()
"""
popen = subprocess.Popen(
[sys.executable, "-c", code],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = popen.communicate()
if popen.returncode != 0:
raise AssertionError(
"process failed with code %s: stderr follows\n%s\n" %
(popen.returncode, err.decode()))
from __future__ import print_function, division, absolute_import
import numpy as np
from numba import types
from numba import unittest_support as unittest
from numba.compiler import compile_isolated, Flags
from .support import TestCase, tag
looplift_flags = Flags()
looplift_flags.set("enable_pyobject")
looplift_flags.set("enable_looplift")
pyobject_looplift_flags = looplift_flags.copy()
pyobject_looplift_flags.set("enable_pyobject_looplift")
def lift1(x):
# Outer needs object mode because of np.empty()
a = np.empty(3)
for i in range(a.size):
# Inner is nopython-compliant
a[i] = x
return a
def lift2(x):
# Outer needs object mode because of np.empty()
a = np.empty((3, 4))
for i in range(a.shape[0]):
for j in range(a.shape[1]):
import time
import io
import ctypes
import multiprocessing as mp
from contextlib import contextmanager
import numpy as np
from numba import config, errors, typing, utils, numpy_support, testing
from numba.compiler import compile_extra, compile_isolated, Flags, DEFAULT_FLAGS
from numba.targets import cpu
import numba.unittest_support as unittest
from numba.runtime import rtsys
from numba.six import PY2
enable_pyobj_flags = Flags()
enable_pyobj_flags.set("enable_pyobject")
force_pyobj_flags = Flags()
force_pyobj_flags.set("force_pyobject")
no_pyobj_flags = Flags()
nrt_flags = Flags()
nrt_flags.set("nrt")
tag = testing.make_tag_decorator(['important', 'long_running'])
_windows_py27 = (sys.platform.startswith('win32')
and sys.version_info[:2] == (2, 7))
_32bit = sys.maxsize <= 2**32
from __future__ import print_function
from functools import partial
from itertools import permutations
import numba.unittest_support as unittest
import numpy as np
from numba.compiler import compile_isolated, Flags
from numba import jit, types, from_dtype, errors, typeof
from numba.errors import TypingError
from .support import TestCase, MemoryLeakMixin, CompilationCache, tag
enable_pyobj_flags = Flags()
enable_pyobj_flags.set("enable_pyobject")
no_pyobj_flags = Flags()
no_pyobj_flags.set('nrt')
def from_generic(pyfuncs_to_use):
"""Decorator for generic check functions.
Iterates over 'pyfuncs_to_use', calling 'func' with the iterated
item as first argument. Example:
@from_generic(numpy_array_reshape, array_reshape)
def check_only_shape(pyfunc, arr, shape, expected_shape):
# Only check Numba result to avoid Numpy bugs
self.memory_leak_setup()
got = generic_run(pyfunc, arr, shape)
self.assertEqual(got.shape, expected_shape)
def mk_no_rw_pipeline(cls, args, return_type=None, flags=None, locals={},
library=None, **kws):
if not flags:
flags = Flags()
flags.no_rewrites = True
return cls.mk_pipeline(args, return_type, flags, locals, library, **kws)
simple_class_spec = [('h', types.int32)]
def simple_class_user(obj):
return obj.h
def unsupported_parfor(a, b):
return np.dot(a, b) # dot as gemm unsupported
def supported_parfor(n):
a = np.ones(n)
for i in prange(n):
a[i] = a[i] + np.sin(i)
return a
force_parallel_flags = Flags()
force_parallel_flags.set("auto_parallel", ParallelOptions(True))
force_parallel_flags.set('nrt')
class DebugTestBase(TestCase):
all_dumps = set(['bytecode', 'cfg', 'ir', 'typeinfer', 'llvm',
'func_opt_llvm', 'optimized_llvm', 'assembly'])
def assert_fails(self, *args, **kwargs):
self.assertRaises(AssertionError, *args, **kwargs)
def check_debug_output(self, out, dump_names):
enabled_dumps = dict.fromkeys(self.all_dumps, False)
for name in dump_names:
assert name in enabled_dumps
from __future__ import print_function, division, absolute_import
import numpy as np
from numba import types, utils
from numba import unittest_support as unittest
from numba.compiler import compile_isolated, Flags
from .support import TestCase, tag, MemoryLeakMixin
looplift_flags = Flags()
looplift_flags.set("enable_pyobject")
looplift_flags.set("enable_looplift")
pyobject_looplift_flags = looplift_flags.copy()
pyobject_looplift_flags.set("enable_pyobject_looplift")
def lift1(x):
# Outer needs object mode because of np.empty()
a = np.empty(3)
for i in range(a.size):
# Inner is nopython-compliant
a[i] = x
return a
def lift2(x):
# Outer needs object mode because of np.empty()
a = np.empty((3, 4))
for i in range(a.shape[0]):
for j in range(a.shape[1]):
def compile_parallel(self, func, arg_types):
fast_pflags = Flags()
fast_pflags.set('auto_parallel', cpu.ParallelOptions(True))
fast_pflags.set('nrt')
fast_pflags.set('fastmath', cpu.FastMathOptions(True))
return compile_isolated(func, arg_types, flags=fast_pflags).entry_point