def test_test1(self): typingctx = typing.Context() targetctx = cpu.CPUContext(typingctx) test_ir = compiler.run_frontend(test1) with cpu_target.nested_context(typingctx, targetctx): one_arg = numba.types.npytypes.Array( numba.types.scalars.Float(name="float64"), 1, 'C') args = (one_arg, one_arg, one_arg, one_arg, one_arg) tp = TestPipeline(typingctx, targetctx, args, test_ir) numba.rewrites.rewrite_registry.apply('before-inference', tp, tp.func_ir) tp.typemap, tp.return_type, tp.calltypes = compiler.type_inference_stage( tp.typingctx, tp.func_ir, tp.args, None) type_annotation = type_annotations.TypeAnnotation( func_ir=tp.func_ir, typemap=tp.typemap, calltypes=tp.calltypes, lifted=(), lifted_from=None, args=tp.args, return_type=tp.return_type, html_output=config.HTML) numba.rewrites.rewrite_registry.apply('after-inference', tp, tp.func_ir) parfor_pass = numba.parfor.ParforPass(tp.func_ir, tp.typemap, tp.calltypes, tp.return_type, tp.typingctx) parfor_pass.run() self.assertTrue(countParfors(test_ir) == 1)
def test1(self): typingctx = typing.Context() targetctx = cpu.CPUContext(typingctx) test_ir = compiler.run_frontend(test_will_propagate) #print("Num blocks = ", len(test_ir.blocks)) #print(test_ir.dump()) with cpu_target.nested_context(typingctx, targetctx): typingctx.refresh() targetctx.refresh() args = (types.int64, types.int64, types.int64) typemap, return_type, calltypes = compiler.type_inference_stage(typingctx, test_ir, args, None) #print("typemap = ", typemap) #print("return_type = ", return_type) type_annotation = type_annotations.TypeAnnotation( func_ir=test_ir, typemap=typemap, calltypes=calltypes, lifted=(), lifted_from=None, args=args, return_type=return_type, html_output=config.HTML) remove_dels(test_ir.blocks) in_cps, out_cps = copy_propagate(test_ir.blocks, typemap) apply_copy_propagate(test_ir.blocks, in_cps, get_name_var_table(test_ir.blocks), typemap, calltypes) remove_dead(test_ir.blocks, test_ir.arg_names, test_ir) self.assertFalse(findLhsAssign(test_ir, "x"))
def test1(self): typingctx = typing.Context() targetctx = cpu.CPUContext(typingctx) test_ir = compiler.run_frontend(test_will_propagate) #print("Num blocks = ", len(test_ir.blocks)) #print(test_ir.dump()) with cpu_target.nested_context(typingctx, targetctx): typingctx.refresh() targetctx.refresh() args = (types.int64, types.int64, types.int64) typemap, return_type, calltypes = compiler.type_inference_stage(typingctx, test_ir, args, None) #print("typemap = ", typemap) #print("return_type = ", return_type) type_annotation = type_annotations.TypeAnnotation( func_ir=test_ir, typemap=typemap, calltypes=calltypes, lifted=(), lifted_from=None, args=args, return_type=return_type, html_output=config.HTML) remove_dels(test_ir.blocks) in_cps, out_cps = copy_propagate(test_ir.blocks, typemap) apply_copy_propagate(test_ir.blocks, in_cps, get_name_var_table(test_ir.blocks), typemap, calltypes) remove_dead(test_ir.blocks, test_ir.arg_names, test_ir) self.assertFalse(findLhsAssign(test_ir, "x"))
def test_mk_func_literal(self): """make sure make_function is passed to typer class as a literal """ test_ir = compiler.run_frontend(mk_func_test_impl) typingctx = cpu_target.typing_context typingctx.refresh() typemap, _, _ = compiler.type_inference_stage( typingctx, test_ir, (), None) self.assertTrue(any(isinstance(a, types.MakeFunctionLiteral) for a in typemap.values()))
def countParfors(test_func, args, **kws): typingctx = typing.Context() targetctx = cpu.CPUContext(typingctx) test_ir = compiler.run_frontend(test_func) if kws: options = cpu.ParallelOptions(kws) else: options = cpu.ParallelOptions(True) with cpu_target.nested_context(typingctx, targetctx): tp = TestPipeline(typingctx, targetctx, args, test_ir) inline_pass = inline_closurecall.InlineClosureCallPass( tp.func_ir, options) inline_pass.run() numba.rewrites.rewrite_registry.apply('before-inference', tp, tp.func_ir) tp.typemap, tp.return_type, tp.calltypes = compiler.type_inference_stage( tp.typingctx, tp.func_ir, tp.args, None) type_annotations.TypeAnnotation(func_ir=tp.func_ir, typemap=tp.typemap, calltypes=tp.calltypes, lifted=(), lifted_from=None, args=tp.args, return_type=tp.return_type, html_output=config.HTML) preparfor_pass = numba.parfor.PreParforPass(tp.func_ir, tp.typemap, tp.calltypes, tp.typingctx, options) preparfor_pass.run() numba.rewrites.rewrite_registry.apply('after-inference', tp, tp.func_ir) parfor_pass = numba.parfor.ParforPass(tp.func_ir, tp.typemap, tp.calltypes, tp.return_type, tp.typingctx, options) parfor_pass.run() ret_count = 0 for label, block in test_ir.blocks.items(): for i, inst in enumerate(block.body): if isinstance(inst, numba.parfor.Parfor): ret_count += 1 return ret_count
def generic(self, args, kws): """ Type the overloaded function by compiling the appropriate implementation for the given args. """ disp, new_args = self._get_impl(args, kws) if disp is None: return # Compile and type it for the given types disp_type = types.Dispatcher(disp) # Store the compiled overload for use in the lowering phase if there's # no inlining required (else functions are being compiled which will # never be used as they are inlined) if not self._inline.is_never_inline: # need to run the compiler front end up to type inference to compute # a signature from numba import compiler ir = compiler.run_frontend(disp_type.dispatcher.py_func) resolve = disp_type.dispatcher.get_call_template template, pysig, folded_args, kws = resolve(new_args, kws) typemap, return_type, calltypes = compiler.type_inference_stage( self.context, ir, folded_args, None) sig = Signature(return_type, folded_args, None) # this stores a load of info for the cost model function if supplied # it by default is None self._inline_overloads[sig.args] = {'folded_args': folded_args} # this stores the compiled overloads, if there's no compiled # overload available i.e. function is always inlined, the key still # needs to exist for type resolution self._compiled_overloads[sig.args] = None if not self._inline.is_always_inline: # this branch is here because a user has supplied a function to # determine whether to inline or not. As a result both compiled # function and inliner info needed, delaying the computation of # this leads to an internal state mess at present. TODO: Fix! sig = disp_type.get_call_type(self.context, new_args, kws) self._compiled_overloads[sig.args] = disp_type.get_overload( sig) # store the inliner information, it's used later in the cost # model function call iinfo = _inline_info(ir, typemap, calltypes, sig) self._inline_overloads[sig.args] = { 'folded_args': folded_args, 'iinfo': iinfo } else: sig = disp_type.get_call_type(self.context, new_args, kws) self._compiled_overloads[sig.args] = disp_type.get_overload(sig) return sig
def test_obj_func_match(self): """Test matching of an object method (other than Array see #3449) """ def test_func(): d = Dummy([1]) d.val.append(2) test_ir = compiler.run_frontend(test_func) typingctx = cpu_target.typing_context typemap, _, _ = compiler.type_inference_stage(typingctx, test_ir, (), None) matched_call = numba.ir_utils.find_callname( test_ir, test_ir.blocks[0].body[14].value, typemap) self.assertTrue( isinstance(matched_call, tuple) and len(matched_call) == 2 and matched_call[0] == 'append')
def test_obj_func_match(self): """Test matching of an object method (other than Array see #3449) """ def test_func(): d = Dummy([1]) d.val.append(2) test_ir = compiler.run_frontend(test_func) typingctx = cpu_target.typing_context typemap, _, _ = compiler.type_inference_stage( typingctx, test_ir, (), None) matched_call = numba.ir_utils.find_callname( test_ir, test_ir.blocks[0].body[14].value, typemap) self.assertTrue(isinstance(matched_call, tuple) and len(matched_call) == 2 and matched_call[0] == 'append')
def get_return_type(self, argtys): if config.DEBUG_ARRAY_OPT == 1: print("get_return_type", argtys) ir_utils.dump_blocks(self.kernel_ir.blocks) if not isinstance(argtys[0], types.npytypes.Array): raise ValueError("The first argument to a stencil kernel must " "be the primary input array.") typemap, return_type, calltypes = compiler.type_inference_stage( self._typingctx, self.kernel_ir, argtys, None, {}) if isinstance(return_type, types.npytypes.Array): raise ValueError( "Stencil kernel must return a scalar and not a numpy array.") real_ret = types.npytypes.Array(return_type, argtys[0].ndim, argtys[0].layout) return (real_ret, typemap, calltypes)
def run(self): dprint_func_ir(self.func_ir, "starting hiframes") topo_order = find_topo_order(self.func_ir.blocks) for label in topo_order: new_body = [] for inst in self.func_ir.blocks[label].body: # df['col'] = arr if isinstance( inst, ir.StaticSetItem) and inst.target.name in self.df_vars: df_name = inst.target.name self.df_vars[df_name][inst.index] = inst.value self._update_df_cols() elif isinstance(inst, ir.Assign): out_nodes = self._run_assign(inst) if isinstance(out_nodes, list): new_body.extend(out_nodes) if isinstance(out_nodes, dict): label = include_new_blocks(self.func_ir.blocks, out_nodes, label, new_body) new_body = [] else: new_body.append(inst) self.func_ir.blocks[label].body = new_body self.func_ir._definitions = _get_definitions(self.func_ir.blocks) #remove_dead(self.func_ir.blocks, self.func_ir.arg_names) if config._has_h5py: io_pass = pio.PIO(self.func_ir, self.locals) io_pass.run() remove_dead(self.func_ir.blocks, self.func_ir.arg_names) DummyFlags = namedtuple('DummyFlags', 'auto_parallel') inline_pass = InlineClosureCallPass(self.func_ir, DummyFlags(True)) inline_pass.run() self.typemap, self.return_type, self.calltypes = numba_compiler.type_inference_stage( self.typingctx, self.func_ir, self.args, None) self.fix_series_filter(self.func_ir.blocks) self.func_ir._definitions = _get_definitions(self.func_ir.blocks) dprint_func_ir(self.func_ir, "after hiframes") if numba.config.DEBUG_ARRAY_OPT == 1: print("df_vars: ", self.df_vars) return
def get_return_type(self, argtys): if config.DEBUG_ARRAY_OPT == 1: print("get_return_type", argtys) ir_utils.dump_blocks(self.kernel_ir.blocks) if not isinstance(argtys[0], types.npytypes.Array): raise ValueError("The first argument to a stencil kernel must " "be the primary input array.") typemap, return_type, calltypes = compiler.type_inference_stage( self._typingctx, self.kernel_ir, argtys, None, {}) if isinstance(return_type, types.npytypes.Array): raise ValueError( "Stencil kernel must return a scalar and not a numpy array.") real_ret = types.npytypes.Array(return_type, argtys[0].ndim, argtys[0].layout) return (real_ret, typemap, calltypes)
def get_stencil_ir(sf, typingctx, args, scope, loc, input_dict, typemap, calltypes): """get typed IR from stencil bytecode """ from numba.targets.cpu import CPUContext from numba.targets.registry import cpu_target from numba.annotations import type_annotations from numba.compiler import type_inference_stage # get untyped IR stencil_func_ir = sf.kernel_ir.copy() # copy the IR nodes to avoid changing IR in the StencilFunc object stencil_blocks = copy.deepcopy(stencil_func_ir.blocks) stencil_func_ir.blocks = stencil_blocks name_var_table = ir_utils.get_name_var_table(stencil_func_ir.blocks) if "out" in name_var_table: raise ValueError("Cannot use the reserved word 'out' in stencil kernels.") # get typed IR with a dummy pipeline (similar to test_parfors.py) targetctx = CPUContext(typingctx) with cpu_target.nested_context(typingctx, targetctx): tp = DummyPipeline(typingctx, targetctx, args, stencil_func_ir) numba.rewrites.rewrite_registry.apply( 'before-inference', tp, tp.func_ir) tp.typemap, tp.return_type, tp.calltypes = type_inference_stage( tp.typingctx, tp.func_ir, tp.args, None) type_annotations.TypeAnnotation( func_ir=tp.func_ir, typemap=tp.typemap, calltypes=tp.calltypes, lifted=(), lifted_from=None, args=tp.args, return_type=tp.return_type, html_output=numba.config.HTML) # make block labels unique stencil_blocks = ir_utils.add_offset_to_labels(stencil_blocks, ir_utils.next_label()) min_label = min(stencil_blocks.keys()) max_label = max(stencil_blocks.keys()) ir_utils._max_label = max_label if config.DEBUG_ARRAY_OPT == 1: print("Initial stencil_blocks") ir_utils.dump_blocks(stencil_blocks) # rename variables, var_dict = {} for v, typ in tp.typemap.items(): new_var = ir.Var(scope, mk_unique_var(v), loc) var_dict[v] = new_var typemap[new_var.name] = typ # add new var type for overall function ir_utils.replace_vars(stencil_blocks, var_dict) if config.DEBUG_ARRAY_OPT == 1: print("After replace_vars") ir_utils.dump_blocks(stencil_blocks) # add call types to overall function for call, call_typ in tp.calltypes.items(): calltypes[call] = call_typ arg_to_arr_dict = {} # replace arg with arr for block in stencil_blocks.values(): for stmt in block.body: if isinstance(stmt, ir.Assign) and isinstance(stmt.value, ir.Arg): if config.DEBUG_ARRAY_OPT == 1: print("input_dict", input_dict, stmt.value.index, stmt.value.name, stmt.value.index in input_dict) arg_to_arr_dict[stmt.value.name] = input_dict[stmt.value.index].name stmt.value = input_dict[stmt.value.index] if config.DEBUG_ARRAY_OPT == 1: print("arg_to_arr_dict", arg_to_arr_dict) print("After replace arg with arr") ir_utils.dump_blocks(stencil_blocks) ir_utils.remove_dels(stencil_blocks) stencil_func_ir.blocks = stencil_blocks return stencil_func_ir, sf.get_return_type(args)[0], arg_to_arr_dict
def inline_closure_call(func_ir, glbls, block, i, callee, typingctx=None, arg_typs=None, typemap=None, calltypes=None, work_list=None): """Inline the body of `callee` at its callsite (`i`-th instruction of `block`) `func_ir` is the func_ir object of the caller function and `glbls` is its global variable environment (func_ir.func_id.func.__globals__). `block` is the IR block of the callsite and `i` is the index of the callsite's node. `callee` is either the called function or a make_function node. `typingctx`, `typemap` and `calltypes` are typing data structures of the caller, available if we are in a typed pass. `arg_typs` includes the types of the arguments at the callsite. """ scope = block.scope instr = block.body[i] call_expr = instr.value debug_print = _make_debug_print("inline_closure_call") debug_print("Found closure call: ", instr, " with callee = ", callee) # support both function object and make_function Expr callee_code = callee.code if hasattr(callee, 'code') else callee.__code__ callee_defaults = callee.defaults if hasattr(callee, 'defaults') else callee.__defaults__ callee_closure = callee.closure if hasattr(callee, 'closure') else callee.__closure__ # first, get the IR of the callee callee_ir = get_ir_of_code(glbls, callee_code) callee_blocks = callee_ir.blocks # 1. relabel callee_ir by adding an offset max_label = max(func_ir.blocks.keys()) callee_blocks = add_offset_to_labels(callee_blocks, max_label + 1) callee_blocks = simplify_CFG(callee_blocks) callee_ir.blocks = callee_blocks min_label = min(callee_blocks.keys()) max_label = max(callee_blocks.keys()) # reset globals in ir_utils before we use it ir_utils._max_label = max_label debug_print("After relabel") _debug_dump(callee_ir) # 2. rename all local variables in callee_ir with new locals created in func_ir callee_scopes = _get_all_scopes(callee_blocks) debug_print("callee_scopes = ", callee_scopes) # one function should only have one local scope assert(len(callee_scopes) == 1) callee_scope = callee_scopes[0] var_dict = {} for var in callee_scope.localvars._con.values(): if not (var.name in callee_code.co_freevars): new_var = scope.define(mk_unique_var(var.name), loc=var.loc) var_dict[var.name] = new_var debug_print("var_dict = ", var_dict) replace_vars(callee_blocks, var_dict) debug_print("After local var rename") _debug_dump(callee_ir) # 3. replace formal parameters with actual arguments args = list(call_expr.args) if callee_defaults: debug_print("defaults = ", callee_defaults) if isinstance(callee_defaults, tuple): # Python 3.5 args = args + list(callee_defaults) elif isinstance(callee_defaults, ir.Var) or isinstance(callee_defaults, str): defaults = func_ir.get_definition(callee_defaults) assert(isinstance(defaults, ir.Const)) loc = defaults.loc args = args + [ir.Const(value=v, loc=loc) for v in defaults.value] else: raise NotImplementedError( "Unsupported defaults to make_function: {}".format(defaults)) debug_print("After arguments rename: ") _debug_dump(callee_ir) # 4. replace freevar with actual closure var if callee_closure: closure = func_ir.get_definition(callee_closure) debug_print("callee's closure = ", closure) if isinstance(closure, tuple): cellget = ctypes.pythonapi.PyCell_Get cellget.restype = ctypes.py_object cellget.argtypes = (ctypes.py_object,) items = tuple(cellget(x) for x in closure) else: assert(isinstance(closure, ir.Expr) and closure.op == 'build_tuple') items = closure.items assert(len(callee_code.co_freevars) == len(items)) _replace_freevars(callee_blocks, items) debug_print("After closure rename") _debug_dump(callee_ir) if typingctx: from numba import compiler f_typemap, f_return_type, f_calltypes = compiler.type_inference_stage( typingctx, callee_ir, arg_typs, None) canonicalize_array_math(callee_ir, f_typemap, f_calltypes, typingctx) # remove argument entries like arg.a from typemap arg_names = [vname for vname in f_typemap if vname.startswith("arg.")] for a in arg_names: f_typemap.pop(a) typemap.update(f_typemap) calltypes.update(f_calltypes) _replace_args_with(callee_blocks, args) # 5. split caller blocks into two new_blocks = [] new_block = ir.Block(scope, block.loc) new_block.body = block.body[i + 1:] new_label = next_label() func_ir.blocks[new_label] = new_block new_blocks.append((new_label, new_block)) block.body = block.body[:i] block.body.append(ir.Jump(min_label, instr.loc)) # 6. replace Return with assignment to LHS topo_order = find_topo_order(callee_blocks) _replace_returns(callee_blocks, instr.target, new_label) # remove the old definition of instr.target too if (instr.target.name in func_ir._definitions): func_ir._definitions[instr.target.name] = [] # 7. insert all new blocks, and add back definitions for label in topo_order: # block scope must point to parent's block = callee_blocks[label] block.scope = scope _add_definitions(func_ir, block) func_ir.blocks[label] = block new_blocks.append((label, block)) debug_print("After merge in") _debug_dump(func_ir) if work_list != None: for block in new_blocks: work_list.append(block) return callee_blocks