def get_ctxmgr_obj(var_ref): """Return the context-manager object and extra info. The extra contains the arguments if the context-manager is used as a call. """ # If the contextmanager used as a Call dfn = func_ir.get_definition(var_ref) if isinstance(dfn, ir.Expr) and dfn.op == 'call': args = [get_var_dfn(x) for x in dfn.args] kws = {k: get_var_dfn(v) for k, v in dfn.kws} extra = {'args': args, 'kwargs': kws} var_ref = dfn.func else: extra = None ctxobj = ir_utils.guard(ir_utils.find_global_value, func_ir, var_ref) # check the contextmanager object if ctxobj is ir.UNDEFINED: raise errors.CompilerError( "Undefined variable used as context manager", loc=blocks[blk_start].loc, ) if ctxobj is None: raise errors.CompilerError(_illegal_cm_msg, loc=dfn.loc) return ctxobj, extra
def _legalize_with_head(blk): """Given *blk*, the head block of the with-context, check that it doesn't do anything else. """ counters = defaultdict(int) for stmt in blk.body: counters[type(stmt)] += 1 if counters.pop(ir.EnterWith) != 1: raise errors.CompilerError( "with's head-block must have exactly 1 ENTER_WITH", loc=blk.loc, ) if counters.pop(ir.Jump) != 1: raise errors.CompilerError( "with's head-block must have exactly 1 JUMP", loc=blk.loc, ) # Can have any number of del counters.pop(ir.Del, None) # There MUST NOT be any other statements if counters: raise errors.CompilerError( "illegal statements in with's head-block", loc=blk.loc, )
def _get_with_contextmanager(func_ir, blocks, blk_start): """Get the global object used for the context manager """ _illegal_cm_msg = "Illegal use of context-manager." def get_var_dfn(var): """Get the definition given a variable""" return func_ir.get_definition(var) def get_ctxmgr_obj(var_ref): """Return the context-manager object and extra info. The extra contains the arguments if the context-manager is used as a call. """ # If the contextmanager used as a Call dfn = func_ir.get_definition(var_ref) if isinstance(dfn, ir.Expr) and dfn.op == 'call': args = [get_var_dfn(x) for x in dfn.args] kws = {k: get_var_dfn(v) for k, v in dfn.kws} extra = {'args': args, 'kwargs': kws} var_ref = dfn.func else: extra = None ctxobj = ir_utils.guard(ir_utils.find_global_value, func_ir, var_ref) # check the contextmanager object if ctxobj is ir.UNDEFINED: raise errors.CompilerError( "Undefined variable used as context manager", loc=blocks[blk_start].loc, ) if ctxobj is None: raise errors.CompilerError(_illegal_cm_msg, loc=dfn.loc) return ctxobj, extra # Scan the start of the with-region for the contextmanager for stmt in blocks[blk_start].body: if isinstance(stmt, ir.EnterWith): var_ref = stmt.contextmanager ctxobj, extra = get_ctxmgr_obj(var_ref) if not hasattr(ctxobj, 'mutate_with_body'): raise errors.CompilerError( "Unsupported context manager in use", loc=blocks[blk_start].loc, ) return ctxobj, extra # No contextmanager found? raise errors.CompilerError( "malformed with-context usage", loc=blocks[blk_start].loc, )
def find_ranges(blocks): cfg = compute_cfg_from_blocks(blocks) sus_setups, sus_pops = set(), set() # traverse the cfg and collect all suspected SETUP_WITH and POP_BLOCK # statements so that we can iterate over them for label, block in blocks.items(): for stmt in block.body: if ir_utils.is_setup_with(stmt): sus_setups.add(label) if ir_utils.is_pop_block(stmt): sus_pops.add(label) # now that we do have the statements, iterate through them in reverse # topo order and from each start looking for pop_blocks setup_with_to_pop_blocks_map = defaultdict(set) for setup_block in cfg.topo_sort(sus_setups, reverse=True): # begin pop_block, search to_visit, seen = [], [] to_visit.append(setup_block) while to_visit: # get whatever is next and record that we have seen it block = to_visit.pop() seen.append(block) # go through the body of the block, looking for statements for stmt in blocks[block].body: # raise detected before pop_block if ir_utils.is_raise(stmt): raise errors.CompilerError( 'unsupported control flow due to raise ' 'statements inside with block') # special case 3.7, return before POP_BLOCK if PYVERSION < (3, 8) and ir_utils.is_return(stmt): raise errors.CompilerError( 'unsupported control flow: due to return ' 'statements inside with block') # if a pop_block, process it if ir_utils.is_pop_block(stmt) and block in sus_pops: # record the jump target of this block belonging to this setup setup_with_to_pop_blocks_map[setup_block].add(block) # remove the block from blocks to be matched sus_pops.remove(block) # stop looking, we have reached the frontier break # if we are still here, by the block terminator, # add all its targets to the to_visit stack, unless we # have seen them already if ir_utils.is_terminator(stmt): for t in stmt.get_targets(): if t not in seen: to_visit.append(t) return setup_with_to_pop_blocks_map
def _run_ssa(blocks): """Run SSA reconstruction on IR blocks of a function. """ if not blocks: # Empty blocks? return {} # Run CFG on the blocks cfg = compute_cfg_from_blocks(blocks) df_plus = _iterated_domfronts(cfg) # Find SSA violators violators = _find_defs_violators(blocks) # Process one SSA-violating variable at a time for varname in violators: _logger.debug( "Fix SSA violator on var %s", varname, ) # Fix up the LHS # Put fresh variables for all assignments to the variable blocks, defmap = _fresh_vars(blocks, varname) _logger.debug("Replaced assignments: %s", pformat(defmap)) # Fix up the RHS # Re-associate the variable uses with the reaching definition blocks = _fix_ssa_vars(blocks, varname, defmap, cfg, df_plus) # Post-condition checks. # CFG invariant cfg_post = compute_cfg_from_blocks(blocks) if cfg_post != cfg: raise errors.CompilerError("CFG mutated in SSA pass") return blocks
def find_setupwiths(blocks): """Find all top-level with. Returns a list of ranges for the with-regions. """ def find_ranges(blocks): for blk in blocks.values(): for ew in blk.find_insts(ir.EnterWith): yield ew.begin, ew.end def previously_occurred(start, known_ranges): for a, b in known_ranges: if s >= a and s < b: return True return False known_ranges = [] for s, e in sorted(find_ranges(blocks)): if not previously_occurred(s, known_ranges): if e not in blocks: # this's possible if there's an exit path in the with-block raise errors.CompilerError( 'unsupported controlflow due to return/raise ' 'statements inside with block') assert s in blocks, 'starting offset is not a label' known_ranges.append((s, e)) return known_ranges
def _legalize_args(self, args, kwargs, loc, func_globals, func_closures): """ Legalize arguments to the context-manager Parameters ---------- args: tuple Positional arguments to the with-context call as IR nodes. kwargs: dict Keyword arguments to the with-context call as IR nodes. loc: numba.core.ir.Loc Source location of the with-context call. func_globals: dict The globals dictionary of the calling function. func_closures: dict The resolved closure variables of the calling function. """ if args: raise errors.CompilerError( "objectmode context doesn't take any positional arguments", ) typeanns = {} for k, v in kwargs.items(): if isinstance(v, ir.Const) and isinstance(v.value, str): typeanns[k] = sigutils._parse_signature_string(v.value) elif isinstance(v, ir.FreeVar): try: v = func_closures[v.name] except KeyError: raise errors.CompilerError( f"Freevar {v.name!r} is not defined") typeanns[k] = v elif isinstance(v, ir.Global): try: v = func_globals[v.name] except KeyError: raise errors.CompilerError( f"Global {v.name!r} is not defined") typeanns[k] = v else: raise errors.CompilerError( "objectmode context requires constants string for " "type annotation", ) return typeanns
def _legalize_withs_cfg(withs, cfg, blocks): """Verify the CFG of the with-context(s).""" doms = cfg.dominators() postdoms = cfg.post_dominators() # Verify that the with-context has no side-exits for s, e in withs: loc = blocks[s].loc if s not in doms[e]: # Not sure what condition can trigger this error. msg = "Entry of with-context not dominating the exit." raise errors.CompilerError(msg, loc=loc) if e not in postdoms[s]: msg = ( "Does not support with-context that contain branches " "(i.e. break/return/raise) that can leave the with-context. " "Details: exit of with-context not post-dominating the entry. " ) raise errors.CompilerError(msg, loc=loc)
def _legalize_args(self, extra, loc): """ Legalize arguments to the context-manager """ if extra is None: return {} if len(extra['args']) != 0: raise errors.CompilerError( "objectmode context doesn't take any positional arguments", ) callkwargs = extra['kwargs'] typeanns = {} for k, v in callkwargs.items(): if not isinstance(v, ir.Const) or not isinstance(v.value, str): raise errors.CompilerError( "objectmode context requires constants string for " "type annotation", ) typeanns[k] = sigutils._parse_signature_string(v.value) return typeanns
def check_compatible(self, orig_disp): """ This implementation checks that `self.output_target == orig_disp._required_target_backend` """ required_target = orig_disp._required_target_backend output_target = self.output_target if required_target is not None: if output_target != required_target: m = ("The output target does match the required target: " f"{output_target} != {required_target}.") raise errors.CompilerError(m)
def find_setupwiths(blocks): """Find all top-level with. Returns a list of ranges for the with-regions. """ def find_ranges(blocks): for blk in blocks.values(): for ew in blk.find_insts(ir.EnterWith): if PYVERSION < (3, 9): end = ew.end for offset in blocks: if ew.end <= offset: end = offset break else: # Since py3.9, the `with finally` handling is injected into # caller function. However, the numba byteflow doesn't # account for that block, which is where `ew.end` is # pointing to. We need to point to the block before # `ew.end`. end = ew.end last_offset = None for offset in blocks: if ew.end < offset: end = last_offset break last_offset = offset yield ew.begin, end def previously_occurred(start, known_ranges): for a, b in known_ranges: if s >= a and s < b: return True return False known_ranges = [] for s, e in sorted(find_ranges(blocks)): if not previously_occurred(s, known_ranges): if e not in blocks: # this's possible if there's an exit path in the with-block raise errors.CompilerError( 'unsupported controlflow due to return/raise ' 'statements inside with block' ) assert s in blocks, 'starting offset is not a label' known_ranges.append((s, e)) return known_ranges
def _legalize_arg_type(self, name, typ, loc): """Legalize the argument type Parameters ---------- name: str argument name. typ: numba.core.types.Type argument type. loc: numba.core.ir.Loc source location for error reporting. """ if getattr(typ, "reflected", False): msgbuf = [ "Objmode context failed.", f"Argument {name!r} is declared as " f"an unsupported type: {typ}.", f"Reflected types are not supported.", ] raise errors.CompilerError(" ".join(msgbuf), loc=loc)
def report_error(varname, msg, loc): raise errors.CompilerError( f"Error handling objmode argument {varname!r}. {msg}", loc=loc, )
def _legalize_args(self, func_ir, args, kwargs, loc, func_globals, func_closures): """ Legalize arguments to the context-manager Parameters ---------- func_ir: FunctionIR args: tuple Positional arguments to the with-context call as IR nodes. kwargs: dict Keyword arguments to the with-context call as IR nodes. loc: numba.core.ir.Loc Source location of the with-context call. func_globals: dict The globals dictionary of the calling function. func_closures: dict The resolved closure variables of the calling function. """ if args: raise errors.CompilerError( "objectmode context doesn't take any positional arguments", ) typeanns = {} def report_error(varname, msg, loc): raise errors.CompilerError( f"Error handling objmode argument {varname!r}. {msg}", loc=loc, ) for k, v in kwargs.items(): if isinstance(v, ir.Const) and isinstance(v.value, str): typeanns[k] = sigutils._parse_signature_string(v.value) elif isinstance(v, ir.FreeVar): try: v = func_closures[v.name] except KeyError: report_error( varname=k, msg=f"Freevar {v.name!r} is not defined.", loc=loc, ) typeanns[k] = v elif isinstance(v, ir.Global): try: v = func_globals[v.name] except KeyError: report_error( varname=k, msg=f"Global {v.name!r} is not defined.", loc=loc, ) typeanns[k] = v elif isinstance(v, ir.Expr) and v.op == "getattr": try: base_obj = func_ir.infer_constant(v.value) typ = getattr(base_obj, v.attr) except (errors.ConstantInferenceError, AttributeError): report_error( varname=k, msg="Getattr cannot be resolved at compile-time.", loc=loc, ) else: typeanns[k] = typ else: report_error( varname=k, msg=("The value must be a compile-time constant either as " "a non-local variable or a getattr expression that " "refers to a Numba type."), loc=loc ) # Legalize the types for objmode for name, typ in typeanns.items(): self._legalize_arg_type(name, typ, loc) return typeanns
def _compile_for_args(self, *args, **kws): """ For internal use. Compile a specialized version of the function for the given *args* and *kws*, and return the resulting callable. """ assert not kws # call any initialisation required for the compilation chain (e.g. # extension point registration). self._compilation_chain_init_hook() def error_rewrite(e, issue_type): """ Rewrite and raise Exception `e` with help supplied based on the specified issue_type. """ if config.SHOW_HELP: help_msg = errors.error_extras[issue_type] e.patch_message('\n'.join((str(e).rstrip(), help_msg))) if config.FULL_TRACEBACKS: raise e else: reraise(type(e), e, None) argtypes = [] for a in args: if isinstance(a, OmittedArg): argtypes.append(types.Omitted(a.value)) else: argtypes.append(self.typeof_pyval(a)) try: return self.compile(tuple(argtypes)) except errors.ForceLiteralArg as e: # Received request for compiler re-entry with the list of arguments # indicated by e.requested_args. # First, check if any of these args are already Literal-ized already_lit_pos = [i for i in e.requested_args if isinstance(args[i], types.Literal)] if already_lit_pos: # Abort compilation if any argument is already a Literal. # Letting this continue will cause infinite compilation loop. m = ("Repeated literal typing request.\n" "{}.\n" "This is likely caused by an error in typing. " "Please see nested and suppressed exceptions.") info = ', '.join('Arg #{} is {}'.format(i, args[i]) for i in sorted(already_lit_pos)) raise errors.CompilerError(m.format(info)) # Convert requested arguments into a Literal. args = [(types.literal if i in e.requested_args else lambda x: x)(args[i]) for i, v in enumerate(args)] # Re-enter compilation with the Literal-ized arguments return self._compile_for_args(*args) except errors.TypingError as e: # Intercept typing error that may be due to an argument # that failed inferencing as a Numba type failed_args = [] for i, arg in enumerate(args): val = arg.value if isinstance(arg, OmittedArg) else arg try: tp = typeof(val, Purpose.argument) except ValueError as typeof_exc: failed_args.append((i, str(typeof_exc))) else: if tp is None: failed_args.append( (i, "cannot determine Numba type of value %r" % (val,))) if failed_args: # Patch error message to ease debugging msg = str(e).rstrip() + ( "\n\nThis error may have been caused by the following argument(s):\n%s\n" % "\n".join("- argument %d: %s" % (i, err) for i, err in failed_args)) e.patch_message(msg) error_rewrite(e, 'typing') except errors.UnsupportedError as e: # Something unsupported is present in the user code, add help info error_rewrite(e, 'unsupported_error') except (errors.NotDefinedError, errors.RedefinedError, errors.VerificationError) as e: # These errors are probably from an issue with either the code supplied # being syntactically or otherwise invalid error_rewrite(e, 'interpreter') except errors.ConstantInferenceError as e: # this is from trying to infer something as constant when it isn't # or isn't supported as a constant error_rewrite(e, 'constant_inference') except Exception as e: if config.SHOW_HELP: if hasattr(e, 'patch_message'): help_msg = errors.error_extras['reportable'] e.patch_message('\n'.join((str(e).rstrip(), help_msg))) # ignore the FULL_TRACEBACKS config, this needs reporting! raise e
def find_setupwiths(func_ir): """Find all top-level with. Returns a list of ranges for the with-regions. """ def find_ranges(blocks): cfg = compute_cfg_from_blocks(blocks) sus_setups, sus_pops = set(), set() # traverse the cfg and collect all suspected SETUP_WITH and POP_BLOCK # statements so that we can iterate over them for label, block in blocks.items(): for stmt in block.body: if ir_utils.is_setup_with(stmt): sus_setups.add(label) if ir_utils.is_pop_block(stmt): sus_pops.add(label) # now that we do have the statements, iterate through them in reverse # topo order and from each start looking for pop_blocks setup_with_to_pop_blocks_map = defaultdict(set) for setup_block in cfg.topo_sort(sus_setups, reverse=True): # begin pop_block, search to_visit, seen = [], [] to_visit.append(setup_block) while to_visit: # get whatever is next and record that we have seen it block = to_visit.pop() seen.append(block) # go through the body of the block, looking for statements for stmt in blocks[block].body: # raise detected before pop_block if ir_utils.is_raise(stmt): raise errors.CompilerError( 'unsupported control flow due to raise ' 'statements inside with block') # special case 3.7, return before POP_BLOCK if PYVERSION < (3, 8) and ir_utils.is_return(stmt): raise errors.CompilerError( 'unsupported control flow: due to return ' 'statements inside with block') # if a pop_block, process it if ir_utils.is_pop_block(stmt) and block in sus_pops: # record the jump target of this block belonging to this setup setup_with_to_pop_blocks_map[setup_block].add(block) # remove the block from blocks to be matched sus_pops.remove(block) # stop looking, we have reached the frontier break # if we are still here, by the block terminator, # add all its targets to the to_visit stack, unless we # have seen them already if ir_utils.is_terminator(stmt): for t in stmt.get_targets(): if t not in seen: to_visit.append(t) return setup_with_to_pop_blocks_map blocks = func_ir.blocks # initial find, will return a dictionary, mapping indices of blocks # containing SETUP_WITH statements to a set of indices of blocks containing # POP_BLOCK statements with_ranges_dict = find_ranges(blocks) # rewrite the CFG in case there are multiple POP_BLOCK statements for one # with func_ir = consolidate_multi_exit_withs(with_ranges_dict, blocks, func_ir) # here we need to turn the withs back into a list of tuples so that the # rest of the code can cope with_ranges_tuple = [(s, list(p)[0]) for (s, p) in with_ranges_dict.items()] # check for POP_BLOCKS with multiple outgoing edges and reject for (_, p) in with_ranges_tuple: targets = blocks[p].terminator.get_targets() if len(targets) != 1: raise errors.CompilerError( "unsupported control flow: with-context contains branches " "(i.e. break/return/raise) that can leave the block ") # now we check for returns inside with and reject them for (_, p) in with_ranges_tuple: target_block = blocks[p] if ir_utils.is_return(func_ir.blocks[ target_block.terminator.get_targets()[0]].terminator): if PYVERSION == (3, 8): # 3.8 needs to bail here, if this is the case, because the # later code can't handle it. raise errors.CompilerError( "unsupported control flow: due to return statements " "inside with block") _rewrite_return(func_ir, p) # now we need to rewrite the tuple such that we have SETUP_WITH matching the # successor of the block that contains the POP_BLOCK. with_ranges_tuple = [(s, func_ir.blocks[p].terminator.get_targets()[0]) for (s, p) in with_ranges_tuple] # finally we check for nested with statements and reject them with_ranges_tuple = _eliminate_nested_withs(with_ranges_tuple) return with_ranges_tuple, func_ir
def _runPass(self, index, pss, internal_state): mutated = False def check(func, compiler_state): mangled = func(compiler_state) if mangled not in (True, False): msg = ( "CompilerPass implementations should return True/False. " "CompilerPass with name '%s' did not.") raise ValueError(msg % pss.name()) return mangled def debug_print(pass_name, print_condition, printable_condition): if pass_name in print_condition: fid = internal_state.func_id args = (fid.modname, fid.func_qualname, self.pipeline_name, printable_condition, pass_name) print(("%s.%s: %s: %s %s" % args).center(120, '-')) if internal_state.func_ir is not None: internal_state.func_ir.dump() else: print("func_ir is None") # debug print before this pass? debug_print(pss.name(), self._print_before + self._print_wrap, "BEFORE") # wire in the analysis info so it's accessible pss.analysis = self._analysis with SimpleTimer() as init_time: mutated |= check(pss.run_initialization, internal_state) with SimpleTimer() as pass_time: mutated |= check(pss.run_pass, internal_state) with SimpleTimer() as finalize_time: mutated |= check(pss.run_finalizer, internal_state) # Check that if the pass is an instance of a FunctionPass that it hasn't # emitted ir.Dels. if isinstance(pss, FunctionPass): enforce_no_dels(internal_state.func_ir) if self._ENFORCING: # TODO: Add in self consistency enforcement for # `func_ir._definitions` etc if _pass_registry.get(pss.__class__).mutates_CFG: if mutated: # block level changes, rebuild all PostProcessor(internal_state.func_ir).run() else: # CFG level changes rebuild CFG internal_state.func_ir.blocks = transforms.canonicalize_cfg( internal_state.func_ir.blocks) # Check the func_ir has exactly one Scope instance if not legalize_single_scope(internal_state.func_ir.blocks): raise errors.CompilerError( f"multiple scope in func_ir detected in {pss}", ) # inject runtimes pt = pass_timings(init_time.elapsed, pass_time.elapsed, finalize_time.elapsed) self.exec_times["%s_%s" % (index, pss.name())] = pt # debug print after this pass? debug_print(pss.name(), self._print_after + self._print_wrap, "AFTER")