def compile(self, sig): with self._compile_lock: # XXX this is mostly duplicated from Dispatcher. flags = self.flags args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exists # (e.g. if another thread compiled it before we got the lock) existing = self.overloads.get(tuple(args)) if existing is not None: return existing.entry_point assert not flags.enable_looplift, "Enable looplift flags is on" cres = compiler.compile_ir(typingctx=self.typingctx, targetctx=self.targetctx, func_ir=self.func_ir, args=args, return_type=return_type, flags=flags, locals=self.locals, lifted=(), lifted_from=self.lifted_from) # Check typing error if object mode is used if cres.typing_error is not None and not flags.enable_pyobject: raise cres.typing_error self.add_overload(cres) return cres.entry_point
def compile(self, sig, locals={}, **targetoptions): locs = self.locals.copy() locs.update(locals) topt = self.targetoptions.copy() topt.update(targetoptions) flags = compiler.Flags() self.targetdescr.options.parse_as_flags(flags, topt) flags.set("no_compile") flags.set("no_cpython_wrapper") flags.set("error_model", "numpy") # Disable loop lifting # The feature requires a real python function flags.unset("enable_looplift") typingctx = self.targetdescr.typing_context targetctx = self.targetdescr.target_context args, return_type = sigutils.normalize_signature(sig) cres = compiler.compile_extra(typingctx, targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=locals) self.overloads[cres.signature] = cres return cres
def add(self, sig=None, argtypes=None, restype=None): # Handle argtypes if argtypes is not None: warnings.warn("Keyword argument argtypes is deprecated", DeprecationWarning) assert sig is None if restype is None: sig = tuple(argtypes) else: sig = restype(*argtypes) del argtypes del restype # compile core as device function args, return_type = sigutils.normalize_signature(sig) devfnsig = signature(return_type, *args) funcname = self.pyfunc.__name__ kernelsource = self._get_kernel_source(self._kernel_template, devfnsig, funcname) corefn, return_type = self._compile_core(devfnsig) glbl = self._get_globals(corefn) sig = signature(types.void, *([a[:] for a in args] + [return_type[:]])) _exec(kernelsource, glbl) stager = glbl['__vectorized_%s' % funcname] kernel = self._compile_kernel(stager, sig) argdtypes = tuple(to_dtype(t) for t in devfnsig.args) resdtype = to_dtype(return_type) self.kernelmap[tuple(argdtypes)] = resdtype, kernel
def compile(self, sig): with self._compile_lock: args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exists # (e.g. if another thread compiled it before we got the lock) existing = self.overloads.get(tuple(args)) if existing is not None: return existing flags = compiler.Flags() self.targetdescr.options.parse_as_flags(flags, self.targetoptions) cres = compiler.compile_extra(self.typingctx, self.targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=self.locals) # Check typing error if object mode is used if cres.typing_error is not None and not flags.enable_pyobject: raise cres.typing_error self.add_overload(cres) return cres.entry_point
def compile(self, sig): with self._compile_lock: args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exists existing = self.overloads.get(tuple(args)) if existing is not None: return existing # Try to load from disk cache cres = self._cache.load_overload(sig, self.targetctx) if cres is not None: # XXX fold this in add_overload()? (also see compiler.py) if not cres.objectmode and not cres.interpmode: self.targetctx.insert_user_function(cres.entry_point, cres.fndesc, [cres.library]) self.add_overload(cres) return cres.entry_point flags = compiler.Flags() self.targetdescr.options.parse_as_flags(flags, self.targetoptions) cres = compiler.compile_extra(self.typingctx, self.targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=self.locals) # Check typing error if object mode is used if cres.typing_error is not None and not flags.enable_pyobject: raise cres.typing_error self.add_overload(cres) self._cache.save_overload(sig, cres) return cres.entry_point
def convert_types(restype, argtypes): # eval type string if sigutils.is_signature(restype): assert argtypes is None argtypes, restype = sigutils.normalize_signature(restype) return restype, argtypes
def add(self, sig=None, argtypes=None, restype=None): # Handle argtypes if argtypes is not None: warnings.warn("Keyword argument argtypes is deprecated", DeprecationWarning) assert sig is None if restype is None: sig = tuple(argtypes) else: sig = restype(*argtypes) del argtypes del restype indims = [len(x) for x in self.inputsig] outdims = [len(x) for x in self.outputsig] funcname = self.py_func.__name__ src = expand_gufunc_template(self._kernel_template, indims, outdims, funcname) glbls = self._get_globals(sig) _exec(src, glbls) fnobj = glbls['__gufunc_{name}'.format(name=funcname)] args, return_type = sigutils.normalize_signature(sig) outertys = list(_determine_gufunc_outer_types(args, indims + outdims)) kernel = self._compile_kernel(fnobj, sig=tuple(outertys)) dtypes = tuple(np.dtype(str(t.dtype)) for t in outertys) self.kernelmap[tuple(dtypes[:-1])] = dtypes[-1], kernel
def compile(self, sig): with self._compile_lock(): # FIXME this is mostly duplicated from Overloaded flags = self.flags args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exist. existing = self.overloads.get(tuple(args)) if existing is not None: return existing.entry_point assert not flags.enable_looplift, "Enable looplift flags is on" cres = compiler.compile_bytecode(typingctx=self.typingctx, targetctx=self.targetctx, bc=self.bytecode, args=args, return_type=return_type, flags=flags, locals=self.locals) # Check typing error if object mode is used if cres.typing_error is not None and not flags.enable_pyobject: raise cres.typing_error self.add_overload(cres) return cres.entry_point
def compile(self, sig): if not self._can_compile: raise RuntimeError("compilation disabled") # Use counter to track recursion compilation depth with self._compiling_counter: args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exists existing = self.overloads.get(tuple(args)) if existing is not None: return existing.entry_point # Try to load from disk cache cres = self._cache.load_overload(sig, self.targetctx) if cres is not None: self._cache_hits[sig] += 1 # XXX fold this in add_overload()? (also see compiler.py) if not cres.objectmode and not cres.interpmode: self.targetctx.insert_user_function( cres.entry_point, cres.fndesc, [cres.library]) self.add_overload(cres) return cres.entry_point self._cache_misses[sig] += 1 cres = self._compiler.compile(args, return_type) self.add_overload(cres) self._cache.save_overload(sig, cres) return cres.entry_point
def add(self, sig=None, argtypes=None, restype=None): # Handle argtypes if argtypes is not None: warnings.warn("Keyword argument argtypes is deprecated", DeprecationWarning) assert sig is None if restype is None: sig = tuple(argtypes) else: sig = restype(*argtypes) # Do compilation # Return CompileResult to test cres = self.nb_func.compile(sig) args, return_type = sigutils.normalize_signature(sig) if return_type is None: if cres.objectmode: # Object mode is used and return type is not specified raise TypeError("return type must be specified for object mode") else: return_type = cres.signature.return_type assert return_type != types.pyobject sig = return_type(*args) # Store the final signature self._sigs.append(sig) self._cres[sig] = cres return cres
def compile(self, sig, locals={}, **targetoptions): with self._compile_lock(): locs = self.locals.copy() locs.update(locals) topt = self.targetoptions.copy() topt.update(targetoptions) flags = compiler.Flags() self.targetdescr.options.parse_as_flags(flags, topt) glctx = self.targetdescr typingctx = glctx.typing_context targetctx = glctx.target_context args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exist. existing = self.overloads.get(tuple(args)) if existing is not None: return existing.entry_point cres = compiler.compile_extra(typingctx, targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=locs) # Check typing error if object mode is used if cres.typing_error is not None and not flags.enable_pyobject: raise cres.typing_error self.add_overload(cres) return cres.entry_point
def add(self, sig=None, argtypes=None, restype=None): # Handle argtypes if argtypes is not None: warnings.warn("Keyword argument argtypes is deprecated", DeprecationWarning) assert sig is None if restype is None: sig = tuple(argtypes) else: sig = restype(*argtypes) # Do compilation # Return CompileResult to test cres = self.nb_func.compile(sig, **self.targetoptions) args, return_type = sigutils.normalize_signature(sig) if not cres.objectmode and cres.signature.return_type != types.void: raise TypeError("gufunc kernel must have void return type") if return_type is None: return_type = types.void # Store the final signature sig = return_type(*args) self._sigs.append(sig) self._cres[sig] = cres return cres
def compile(self, sig): if not self._can_compile: raise RuntimeError("compilation disabled") with self._compile_lock: args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exists existing = self.overloads.get(tuple(args)) if existing is not None: return existing.entry_point # Try to load from disk cache cres = self._cache.load_overload(sig, self.targetctx) if cres is not None: self._cache_hits[sig] += 1 # XXX fold this in add_overload()? (also see compiler.py) if not cres.objectmode and not cres.interpmode: self.targetctx.insert_user_function(cres.entry_point, cres.fndesc, [cres.library]) self.add_overload(cres) return cres.entry_point self._cache_misses[sig] += 1 cres = self._compiler.compile(args, return_type) self.add_overload(cres) self._cache.save_overload(sig, cres) return cres.entry_point
def compile(self, sig, locals={}, **targetoptions): locs = self.locals.copy() locs.update(locals) topt = self.targetoptions.copy() topt.update(targetoptions) if topt.get('nopython', True) == False: raise TypeError("nopython option must be False") topt['nopython'] = True flags = compiler.Flags() flags.set("no_compile") self.targetdescr.options.parse_as_flags(flags, topt) typingctx = self.targetdescr.typing_context targetctx = self.targetdescr.target_context args, return_type = sigutils.normalize_signature(sig) cres = compiler.compile_extra(typingctx, targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=locals) self.overloads[cres.signature] = cres return cres
def get_pipeline(self, sig, library=None): if library is None: library = self.library args, return_type = sigutils.normalize_signature(sig) return compiler.Pipeline(self.typingctx, self.targetctx, self.library, args, return_type, self.flags, self.locals)
def __init__(self, pyfunc, signature): self.py_func = pyfunc self.signature = signature self.name = pyfunc.__name__ # recreate for each UDF, as linking is destructive to the # precompiled module impala_typing = impala_typing_context() impala_targets = ImpalaTargetContext(impala_typing) args, return_type = sigutils.normalize_signature(signature) flags = Flags() flags.set('no_compile') self._cres = compile_extra(typingctx=impala_typing, targetctx=impala_targets, func=pyfunc, args=args, return_type=return_type, flags=flags, locals={}) llvm_func = impala_targets.finalize(self._cres.llvm_func, return_type, args) self.llvm_func = llvm_func # numba_module = llvm_func.module self.llvm_module = llvm_func.module # link in the precompiled module # bc it's destructive, load a fresh version precompiled = lc.Module.from_bitcode( pkgutil.get_data("impala.udf", "precompiled/impyla.bc")) self.llvm_module.link_in(precompiled)
def _device_jit(signature): argtypes, restype = sigutils.normalize_signature(signature) def _wrapped(pyfunc): return compile_device(pyfunc, restype, argtypes) return _wrapped
def compile(self, sig): with self._compile_lock: args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exists # (e.g. if another thread compiled it before we got the lock) existing = self.overloads.get(tuple(args)) if existing is not None: return existing flags = compiler.Flags() self.targetdescr.options.parse_as_flags(flags, self.targetoptions) cres = compiler.compile_extra( self.typingctx, self.targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=self.locals, ) # Check typing error if object mode is used if cres.typing_error is not None and not flags.enable_pyobject: raise cres.typing_error self.add_overload(cres) return cres.entry_point
def compile(self, sig): with self._compile_lock: # XXX this is mostly duplicated from Dispatcher. flags = self.flags args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exists # (e.g. if another thread compiled it before we got the lock) existing = self.overloads.get(tuple(args)) if existing is not None: return existing.entry_point assert not flags.enable_looplift, "Enable looplift flags is on" cres = compiler.compile_ir(typingctx=self.typingctx, targetctx=self.targetctx, interp=self.interp, args=args, return_type=return_type, flags=flags, locals=self.locals, lifted=(), lifted_from=self.lifted_from) # Check typing error if object mode is used if cres.typing_error is not None and not flags.enable_pyobject: raise cres.typing_error self.add_overload(cres) return cres.entry_point
def compile(self, sig, locals={}, **targetoptions): locs = self.locals.copy() locs.update(locals) topt = self.targetoptions.copy() topt.update(targetoptions) flags = compiler.Flags() self.targetdescr.options.parse_as_flags(flags, topt) flags.set("no_compile") flags.set("no_cpython_wrapper") # Disable loop lifting # The feature requires a real python function flags.unset("enable_looplift") typingctx = self.targetdescr.typing_context targetctx = self.targetdescr.target_context args, return_type = sigutils.normalize_signature(sig) cres = compiler.compile_extra(typingctx, targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=locals) self.overloads[cres.signature] = cres return cres
def add(self, sig=None, argtypes=None, restype=None): # Handle argtypes if argtypes is not None: warnings.warn("Keyword argument argtypes is deprecated", DeprecationWarning) assert sig is None if restype is None: sig = tuple(argtypes) else: sig = restype(*argtypes) # Do compilation # Return CompileResult to test cres = self.nb_func.compile(sig) args, return_type = sigutils.normalize_signature(sig) if return_type is None: if cres.objectmode: # Object mode is used and return type is not specified raise TypeError( "return type must be specified for object mode") else: return_type = cres.signature.return_type assert return_type != types.pyobject sig = return_type(*args) # Store the final signature self._sigs.append(sig) self._cres[sig] = cres return cres
def compile(self, sig, locals={}, **targetoptions): with self._compile_lock(): locs = self.locals.copy() locs.update(locals) topt = self.targetoptions.copy() topt.update(targetoptions) flags = compiler.Flags() self.targetdescr.options.parse_as_flags(flags, topt) args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exist. existing = self.overloads.get(tuple(args)) if existing is not None: return existing cres = compiler.compile_extra(self.typingctx, self.targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=locs) # Check typing error if object mode is used if cres.typing_error is not None and not flags.enable_pyobject: raise cres.typing_error self.add_overload(cres) return cres.entry_point
def compile(self, sig): # Use cache and compiler in a critical section with compiler.lock_compiler: # Use counter to track recursion compilation depth with self._compiling_counter: # XXX this is mostly duplicated from Dispatcher. flags = self.flags args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exists # (e.g. if another thread compiled it before we got the lock) existing = self.overloads.get(tuple(args)) if existing is not None: return existing.entry_point assert not flags.enable_looplift, "Enable looplift flags is on" # Clone IR to avoid mutation in rewrite pass cloned_func_ir = self.func_ir.copy() cres = compiler.compile_ir(typingctx=self.typingctx, targetctx=self.targetctx, func_ir=cloned_func_ir, args=args, return_type=return_type, flags=flags, locals=self.locals, lifted=(), lifted_from=self.lifted_from) # Check typing error if object mode is used if cres.typing_error is not None and not flags.enable_pyobject: raise cres.typing_error self.add_overload(cres) return cres.entry_point
def compile(self, py_func, sig, library=None): if library is None: library = self.library args, return_type = sigutils.normalize_signature(sig) return compiler.compile_extra(self.typingctx, self.targetctx, py_func, args=args, return_type=return_type, flags=self.flags, locals=self.locals, library=library)
def _kernel_jit(signature): argtypes, restype = sigutils.normalize_signature(signature) if restype is not None and restype != types.void: msg = "HSA kernel must have void return type but got {restype}" raise TypeError(msg.format(restype=restype)) def _wrapped(pyfunc): return compile_kernel(pyfunc, argtypes) return _wrapped
def export(self, exported_name, sig): """ Mark a function for exporting in the extension module. """ fn_args, fn_retty = sigutils.normalize_signature(sig) sig = typing.signature(fn_retty, *fn_args) if exported_name in self._exported_functions: raise KeyError("duplicated export symbol %s" % (exported_name)) def decorator(func): entry = ExportEntry(exported_name, sig, func) self._exported_functions[exported_name] = entry return func return decorator
def __init__(self, pyfunc, signature): self.py_func = pyfunc self.signature = signature self.name = pyfunc.__name__ args, return_type = sigutils.normalize_signature(signature) flags = Flags() flags.set('no_compile') self._cres = compile_extra(typingctx=impala_typing, targetctx=impala_targets, func=pyfunc, args=args, return_type=return_type, flags=flags, locals={}) llvm_func = impala_targets.finalize(self._cres.llvm_func, return_type, args) self.llvm_func = llvm_func self.llvm_module = llvm_func.module
def compile(self, sig): ''' Compile and bind to the current context a version of this kernel specialized for the given signature. ''' argtypes, return_type = sigutils.normalize_signature(sig) assert return_type is None kernel = self.definitions.get(argtypes) if kernel is None: if 'link' not in self.targetoptions: self.targetoptions['link'] = () kernel = compile_kernel(self.py_func, argtypes, **self.targetoptions) self.definitions[argtypes] = kernel if self.bind: kernel.bind() return kernel
def _compile_core(self, sig, flags, locals): """ Trigger the compiler on the core function or load a previously compiled version from the cache. Returns the CompileResult. """ typingctx = self.targetdescr.typing_context targetctx = self.targetdescr.target_context cres = self.cache.load_overload(sig, targetctx) if cres is not None: # Use cached version return cres # Compile args, return_type = sigutils.normalize_signature(sig) cres = compiler.compile_extra(typingctx, targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=locals) self.cache.save_overload(sig, cres) return cres
def _compile_element_wise_function(nb_func, targetoptions, sig=None, argtypes=None, restype=None): # Handle argtypes if argtypes is not None: warnings.warn("Keyword argument argtypes is deprecated", DeprecationWarning) assert sig is None if restype is None: sig = tuple(argtypes) else: sig = restype(*argtypes) # Do compilation # Return CompileResult to test cres = nb_func.compile(sig, **targetoptions) args, return_type = sigutils.normalize_signature(sig) return cres, args, return_type
def _compile_core(self, sig, flags, locals): """ Trigger the compiler on the core function or load a previously compiled version from the cache. Returns the CompileResult. """ typingctx = self.targetdescr.typing_context targetctx = self.targetdescr.target_context @contextmanager def store_overloads_on_success(): # use to ensure overloads are stored on success try: yield except: raise else: exists = self.overloads.get(cres.signature) if exists is None: self.overloads[cres.signature] = cres # Use cache and compiler in a critical section with compiler.lock_compiler: with store_overloads_on_success(): # attempt look up of existing cres = self.cache.load_overload(sig, targetctx) if cres is not None: return cres # Compile args, return_type = sigutils.normalize_signature(sig) cres = compiler.compile_extra(typingctx, targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=locals) # cache lookup failed before so safe to save self.cache.save_overload(sig, cres) return cres
def _compile_core(self, sig, flags, locals): """ Trigger the compiler on the core function or load a previously compiled version from the cache. Returns the CompileResult. """ typingctx = self.targetdescr.typing_context targetctx = self.targetdescr.target_context # Use cache and compiler in a critical section with compiler.lock_compiler: cres = self.cache.load_overload(sig, targetctx) if cres is not None: # Use cached version return cres # Compile args, return_type = sigutils.normalize_signature(sig) cres = compiler.compile_extra(typingctx, targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=locals) self.cache.save_overload(sig, cres) return cres
def compile(self, sig): with self._compile_lock: args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exists existing = self.overloads.get(tuple(args)) if existing is not None: return existing.entry_point # Try to load from disk cache cres = self._cache.load_overload(sig, self.targetctx) if cres is not None: # XXX fold this in add_overload()? (also see compiler.py) if not cres.objectmode and not cres.interpmode: self.targetctx.insert_user_function( cres.entry_point, cres.fndesc, [cres.library]) self.add_overload(cres) return cres.entry_point cres = self._compiler.compile(args, return_type) self.add_overload(cres) self._cache.save_overload(sig, cres) return cres.entry_point
def _compile_core(self, sig, flags, locals): """ Trigger the compiler on the core function or load a previously compiled version from the cache. Returns the CompileResult. """ typingctx = self.targetdescr.typing_context targetctx = self.targetdescr.target_context @contextmanager def store_overloads_on_success(): # use to ensure overloads are stored on success try: yield except: raise else: exists = self.overloads.get(cres.signature) if exists is None: self.overloads[cres.signature] = cres # Use cache and compiler in a critical section with global_compiler_lock: with store_overloads_on_success(): # attempt look up of existing cres = self.cache.load_overload(sig, targetctx) if cres is not None: return cres # Compile args, return_type = sigutils.normalize_signature(sig) cres = compiler.compile_extra(typingctx, targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=locals) # cache lookup failed before so safe to save self.cache.save_overload(sig, cres) return cres
def compile(self, sig): with self._compile_lock: args, return_type = sigutils.normalize_signature(sig) # Don't recompile if signature already exists existing = self.overloads.get(tuple(args)) if existing is not None: return existing # Try to load from disk cache cres = self._cache.load_overload(sig, self.targetctx) if cres is not None: # XXX fold this in add_overload()? (also see compiler.py) if not cres.objectmode and not cres.interpmode: self.targetctx.insert_user_function( cres.entry_point, cres.fndesc, [cres.library]) self.add_overload(cres) return cres.entry_point flags = compiler.Flags() self.targetdescr.options.parse_as_flags(flags, self.targetoptions) cres = compiler.compile_extra(self.typingctx, self.targetctx, self.py_func, args=args, return_type=return_type, flags=flags, locals=self.locals) # Check typing error if object mode is used if cres.typing_error is not None and not flags.enable_pyobject: raise cres.typing_error self.add_overload(cres) self._cache.save_overload(sig, cres) return cres.entry_point
def get_overload(self, sig): """ Return the compiled function for the given signature. """ args, return_type = sigutils.normalize_signature(sig) return self.overloads[tuple(args)].entry_point
def _compile_element_wise_function(nb_func, targetoptions, sig): # Do compilation # Return CompileResult to test cres = nb_func.compile(sig, **targetoptions) args, return_type = sigutils.normalize_signature(sig) return cres, args, return_type
def get_overload(self, sig): args, return_type = sigutils.normalize_signature(sig) return self.overloads[tuple(args)]
def get_overload(self, sig): args, return_type = sigutils.normalize_signature(sig) return self.overloads[tuple(args)].entry_point
def wrappped(func): fn_argtys, fn_retty = sigutils.normalize_signature(sig) signature = typing.signature(fn_retty, *fn_argtys) entry = ExportEntry(symbol=sym, signature=signature, function=func) export_registry.append(entry)