def get_ctxmgr_obj(var_ref):
"""Return the context-manager object and extra info.
The extra contains the arguments if the context-manager is used
as a call.
"""
# If the contextmanager used as a Call
dfn = func_ir.get_definition(var_ref)
if isinstance(dfn, ir.Expr) and dfn.op == 'call':
args = [get_var_dfn(x) for x in dfn.args]
kws = {k: get_var_dfn(v) for k, v in dfn.kws}
extra = {'args': args, 'kwargs': kws}
var_ref = dfn.func
else:
extra = None
ctxobj = ir_utils.guard(ir_utils.find_global_value, func_ir, var_ref)
# check the contextmanager object
if ctxobj is ir.UNDEFINED:
raise errors.CompilerError(
"Undefined variable used as context manager",
loc=blocks[blk_start].loc,
)
if ctxobj is None:
raise errors.CompilerError(_illegal_cm_msg, loc=dfn.loc)
return ctxobj, extra
def _legalize_with_head(blk):
"""Given *blk*, the head block of the with-context, check that it doesn't
do anything else.
"""
counters = defaultdict(int)
for stmt in blk.body:
counters[type(stmt)] += 1
if counters.pop(ir.EnterWith) != 1:
raise errors.CompilerError(
"with's head-block must have exactly 1 ENTER_WITH",
loc=blk.loc,
)
if counters.pop(ir.Jump) != 1:
raise errors.CompilerError(
"with's head-block must have exactly 1 JUMP",
loc=blk.loc,
)
# Can have any number of del
counters.pop(ir.Del, None)
# There MUST NOT be any other statements
if counters:
raise errors.CompilerError(
"illegal statements in with's head-block",
loc=blk.loc,
)
def _get_with_contextmanager(func_ir, blocks, blk_start):
"""Get the global object used for the context manager
"""
_illegal_cm_msg = "Illegal use of context-manager."
def get_var_dfn(var):
"""Get the definition given a variable"""
return func_ir.get_definition(var)
def get_ctxmgr_obj(var_ref):
"""Return the context-manager object and extra info.
The extra contains the arguments if the context-manager is used
as a call.
"""
# If the contextmanager used as a Call
dfn = func_ir.get_definition(var_ref)
if isinstance(dfn, ir.Expr) and dfn.op == 'call':
args = [get_var_dfn(x) for x in dfn.args]
kws = {k: get_var_dfn(v) for k, v in dfn.kws}
extra = {'args': args, 'kwargs': kws}
var_ref = dfn.func
else:
extra = None
ctxobj = ir_utils.guard(ir_utils.find_global_value, func_ir, var_ref)
# check the contextmanager object
if ctxobj is ir.UNDEFINED:
raise errors.CompilerError(
"Undefined variable used as context manager",
loc=blocks[blk_start].loc,
)
if ctxobj is None:
raise errors.CompilerError(_illegal_cm_msg, loc=dfn.loc)
return ctxobj, extra
# Scan the start of the with-region for the contextmanager
for stmt in blocks[blk_start].body:
if isinstance(stmt, ir.EnterWith):
var_ref = stmt.contextmanager
ctxobj, extra = get_ctxmgr_obj(var_ref)
if not hasattr(ctxobj, 'mutate_with_body'):
raise errors.CompilerError(
"Unsupported context manager in use",
loc=blocks[blk_start].loc,
)
return ctxobj, extra
# No contextmanager found?
raise errors.CompilerError(
"malformed with-context usage",
loc=blocks[blk_start].loc,
)
def find_ranges(blocks):
cfg = compute_cfg_from_blocks(blocks)
sus_setups, sus_pops = set(), set()
# traverse the cfg and collect all suspected SETUP_WITH and POP_BLOCK
# statements so that we can iterate over them
for label, block in blocks.items():
for stmt in block.body:
if ir_utils.is_setup_with(stmt):
sus_setups.add(label)
if ir_utils.is_pop_block(stmt):
sus_pops.add(label)
# now that we do have the statements, iterate through them in reverse
# topo order and from each start looking for pop_blocks
setup_with_to_pop_blocks_map = defaultdict(set)
for setup_block in cfg.topo_sort(sus_setups, reverse=True):
# begin pop_block, search
to_visit, seen = [], []
to_visit.append(setup_block)
while to_visit:
# get whatever is next and record that we have seen it
block = to_visit.pop()
seen.append(block)
# go through the body of the block, looking for statements
for stmt in blocks[block].body:
# raise detected before pop_block
if ir_utils.is_raise(stmt):
raise errors.CompilerError(
'unsupported control flow due to raise '
'statements inside with block')
# special case 3.7, return before POP_BLOCK
if PYVERSION < (3, 8) and ir_utils.is_return(stmt):
raise errors.CompilerError(
'unsupported control flow: due to return '
'statements inside with block')
# if a pop_block, process it
if ir_utils.is_pop_block(stmt) and block in sus_pops:
# record the jump target of this block belonging to this setup
setup_with_to_pop_blocks_map[setup_block].add(block)
# remove the block from blocks to be matched
sus_pops.remove(block)
# stop looking, we have reached the frontier
break
# if we are still here, by the block terminator,
# add all its targets to the to_visit stack, unless we
# have seen them already
if ir_utils.is_terminator(stmt):
for t in stmt.get_targets():
if t not in seen:
to_visit.append(t)
return setup_with_to_pop_blocks_map
def _run_ssa(blocks):
"""Run SSA reconstruction on IR blocks of a function.
"""
if not blocks:
# Empty blocks?
return {}
# Run CFG on the blocks
cfg = compute_cfg_from_blocks(blocks)
df_plus = _iterated_domfronts(cfg)
# Find SSA violators
violators = _find_defs_violators(blocks)
# Process one SSA-violating variable at a time
for varname in violators:
_logger.debug(
"Fix SSA violator on var %s", varname,
)
# Fix up the LHS
# Put fresh variables for all assignments to the variable
blocks, defmap = _fresh_vars(blocks, varname)
_logger.debug("Replaced assignments: %s", pformat(defmap))
# Fix up the RHS
# Re-associate the variable uses with the reaching definition
blocks = _fix_ssa_vars(blocks, varname, defmap, cfg, df_plus)
# Post-condition checks.
# CFG invariant
cfg_post = compute_cfg_from_blocks(blocks)
if cfg_post != cfg:
raise errors.CompilerError("CFG mutated in SSA pass")
return blocks
def find_setupwiths(blocks):
"""Find all top-level with.
Returns a list of ranges for the with-regions.
"""
def find_ranges(blocks):
for blk in blocks.values():
for ew in blk.find_insts(ir.EnterWith):
yield ew.begin, ew.end
def previously_occurred(start, known_ranges):
for a, b in known_ranges:
if s >= a and s < b:
return True
return False
known_ranges = []
for s, e in sorted(find_ranges(blocks)):
if not previously_occurred(s, known_ranges):
if e not in blocks:
# this's possible if there's an exit path in the with-block
raise errors.CompilerError(
'unsupported controlflow due to return/raise '
'statements inside with block')
assert s in blocks, 'starting offset is not a label'
known_ranges.append((s, e))
return known_ranges
def _legalize_args(self, args, kwargs, loc, func_globals, func_closures):
"""
Legalize arguments to the context-manager
Parameters
----------
args: tuple
Positional arguments to the with-context call as IR nodes.
kwargs: dict
Keyword arguments to the with-context call as IR nodes.
loc: numba.core.ir.Loc
Source location of the with-context call.
func_globals: dict
The globals dictionary of the calling function.
func_closures: dict
The resolved closure variables of the calling function.
"""
if args:
raise errors.CompilerError(
"objectmode context doesn't take any positional arguments", )
typeanns = {}
for k, v in kwargs.items():
if isinstance(v, ir.Const) and isinstance(v.value, str):
typeanns[k] = sigutils._parse_signature_string(v.value)
elif isinstance(v, ir.FreeVar):
try:
v = func_closures[v.name]
except KeyError:
raise errors.CompilerError(
f"Freevar {v.name!r} is not defined")
typeanns[k] = v
elif isinstance(v, ir.Global):
try:
v = func_globals[v.name]
except KeyError:
raise errors.CompilerError(
f"Global {v.name!r} is not defined")
typeanns[k] = v
else:
raise errors.CompilerError(
"objectmode context requires constants string for "
"type annotation", )
return typeanns
def _legalize_withs_cfg(withs, cfg, blocks):
"""Verify the CFG of the with-context(s)."""
doms = cfg.dominators()
postdoms = cfg.post_dominators()
# Verify that the with-context has no side-exits
for s, e in withs:
loc = blocks[s].loc
if s not in doms[e]:
# Not sure what condition can trigger this error.
msg = "Entry of with-context not dominating the exit."
raise errors.CompilerError(msg, loc=loc)
if e not in postdoms[s]:
msg = (
"Does not support with-context that contain branches "
"(i.e. break/return/raise) that can leave the with-context. "
"Details: exit of with-context not post-dominating the entry. "
)
raise errors.CompilerError(msg, loc=loc)
def _legalize_args(self, extra, loc):
"""
Legalize arguments to the context-manager
"""
if extra is None:
return {}
if len(extra['args']) != 0:
raise errors.CompilerError(
"objectmode context doesn't take any positional arguments", )
callkwargs = extra['kwargs']
typeanns = {}
for k, v in callkwargs.items():
if not isinstance(v, ir.Const) or not isinstance(v.value, str):
raise errors.CompilerError(
"objectmode context requires constants string for "
"type annotation", )
typeanns[k] = sigutils._parse_signature_string(v.value)
return typeanns
def check_compatible(self, orig_disp):
"""
This implementation checks that
`self.output_target == orig_disp._required_target_backend`
"""
required_target = orig_disp._required_target_backend
output_target = self.output_target
if required_target is not None:
if output_target != required_target:
m = ("The output target does match the required target: "
f"{output_target} != {required_target}.")
raise errors.CompilerError(m)
def find_setupwiths(blocks):
"""Find all top-level with.
Returns a list of ranges for the with-regions.
"""
def find_ranges(blocks):
for blk in blocks.values():
for ew in blk.find_insts(ir.EnterWith):
if PYVERSION < (3, 9):
end = ew.end
for offset in blocks:
if ew.end <= offset:
end = offset
break
else:
# Since py3.9, the `with finally` handling is injected into
# caller function. However, the numba byteflow doesn't
# account for that block, which is where `ew.end` is
# pointing to. We need to point to the block before
# `ew.end`.
end = ew.end
last_offset = None
for offset in blocks:
if ew.end < offset:
end = last_offset
break
last_offset = offset
yield ew.begin, end
def previously_occurred(start, known_ranges):
for a, b in known_ranges:
if s >= a and s < b:
return True
return False
known_ranges = []
for s, e in sorted(find_ranges(blocks)):
if not previously_occurred(s, known_ranges):
if e not in blocks:
# this's possible if there's an exit path in the with-block
raise errors.CompilerError(
'unsupported controlflow due to return/raise '
'statements inside with block'
)
assert s in blocks, 'starting offset is not a label'
known_ranges.append((s, e))
return known_ranges
def _legalize_arg_type(self, name, typ, loc):
"""Legalize the argument type
Parameters
----------
name: str
argument name.
typ: numba.core.types.Type
argument type.
loc: numba.core.ir.Loc
source location for error reporting.
"""
if getattr(typ, "reflected", False):
msgbuf = [
"Objmode context failed.",
f"Argument {name!r} is declared as "
f"an unsupported type: {typ}.",
f"Reflected types are not supported.",
]
raise errors.CompilerError(" ".join(msgbuf), loc=loc)
def report_error(varname, msg, loc):
raise errors.CompilerError(
f"Error handling objmode argument {varname!r}. {msg}",
loc=loc,
)
def _legalize_args(self, func_ir, args, kwargs, loc, func_globals,
func_closures):
"""
Legalize arguments to the context-manager
Parameters
----------
func_ir: FunctionIR
args: tuple
Positional arguments to the with-context call as IR nodes.
kwargs: dict
Keyword arguments to the with-context call as IR nodes.
loc: numba.core.ir.Loc
Source location of the with-context call.
func_globals: dict
The globals dictionary of the calling function.
func_closures: dict
The resolved closure variables of the calling function.
"""
if args:
raise errors.CompilerError(
"objectmode context doesn't take any positional arguments",
)
typeanns = {}
def report_error(varname, msg, loc):
raise errors.CompilerError(
f"Error handling objmode argument {varname!r}. {msg}",
loc=loc,
)
for k, v in kwargs.items():
if isinstance(v, ir.Const) and isinstance(v.value, str):
typeanns[k] = sigutils._parse_signature_string(v.value)
elif isinstance(v, ir.FreeVar):
try:
v = func_closures[v.name]
except KeyError:
report_error(
varname=k,
msg=f"Freevar {v.name!r} is not defined.",
loc=loc,
)
typeanns[k] = v
elif isinstance(v, ir.Global):
try:
v = func_globals[v.name]
except KeyError:
report_error(
varname=k,
msg=f"Global {v.name!r} is not defined.",
loc=loc,
)
typeanns[k] = v
elif isinstance(v, ir.Expr) and v.op == "getattr":
try:
base_obj = func_ir.infer_constant(v.value)
typ = getattr(base_obj, v.attr)
except (errors.ConstantInferenceError, AttributeError):
report_error(
varname=k,
msg="Getattr cannot be resolved at compile-time.",
loc=loc,
)
else:
typeanns[k] = typ
else:
report_error(
varname=k,
msg=("The value must be a compile-time constant either as "
"a non-local variable or a getattr expression that "
"refers to a Numba type."),
loc=loc
)
# Legalize the types for objmode
for name, typ in typeanns.items():
self._legalize_arg_type(name, typ, loc)
return typeanns
def _compile_for_args(self, *args, **kws):
"""
For internal use. Compile a specialized version of the function
for the given *args* and *kws*, and return the resulting callable.
"""
assert not kws
# call any initialisation required for the compilation chain (e.g.
# extension point registration).
self._compilation_chain_init_hook()
def error_rewrite(e, issue_type):
"""
Rewrite and raise Exception `e` with help supplied based on the
specified issue_type.
"""
if config.SHOW_HELP:
help_msg = errors.error_extras[issue_type]
e.patch_message('\n'.join((str(e).rstrip(), help_msg)))
if config.FULL_TRACEBACKS:
raise e
else:
reraise(type(e), e, None)
argtypes = []
for a in args:
if isinstance(a, OmittedArg):
argtypes.append(types.Omitted(a.value))
else:
argtypes.append(self.typeof_pyval(a))
try:
return self.compile(tuple(argtypes))
except errors.ForceLiteralArg as e:
# Received request for compiler re-entry with the list of arguments
# indicated by e.requested_args.
# First, check if any of these args are already Literal-ized
already_lit_pos = [i for i in e.requested_args
if isinstance(args[i], types.Literal)]
if already_lit_pos:
# Abort compilation if any argument is already a Literal.
# Letting this continue will cause infinite compilation loop.
m = ("Repeated literal typing request.\n"
"{}.\n"
"This is likely caused by an error in typing. "
"Please see nested and suppressed exceptions.")
info = ', '.join('Arg #{} is {}'.format(i, args[i])
for i in sorted(already_lit_pos))
raise errors.CompilerError(m.format(info))
# Convert requested arguments into a Literal.
args = [(types.literal
if i in e.requested_args
else lambda x: x)(args[i])
for i, v in enumerate(args)]
# Re-enter compilation with the Literal-ized arguments
return self._compile_for_args(*args)
except errors.TypingError as e:
# Intercept typing error that may be due to an argument
# that failed inferencing as a Numba type
failed_args = []
for i, arg in enumerate(args):
val = arg.value if isinstance(arg, OmittedArg) else arg
try:
tp = typeof(val, Purpose.argument)
except ValueError as typeof_exc:
failed_args.append((i, str(typeof_exc)))
else:
if tp is None:
failed_args.append(
(i,
"cannot determine Numba type of value %r" % (val,)))
if failed_args:
# Patch error message to ease debugging
msg = str(e).rstrip() + (
"\n\nThis error may have been caused by the following argument(s):\n%s\n"
% "\n".join("- argument %d: %s" % (i, err)
for i, err in failed_args))
e.patch_message(msg)
error_rewrite(e, 'typing')
except errors.UnsupportedError as e:
# Something unsupported is present in the user code, add help info
error_rewrite(e, 'unsupported_error')
except (errors.NotDefinedError, errors.RedefinedError,
errors.VerificationError) as e:
# These errors are probably from an issue with either the code supplied
# being syntactically or otherwise invalid
error_rewrite(e, 'interpreter')
except errors.ConstantInferenceError as e:
# this is from trying to infer something as constant when it isn't
# or isn't supported as a constant
error_rewrite(e, 'constant_inference')
except Exception as e:
if config.SHOW_HELP:
if hasattr(e, 'patch_message'):
help_msg = errors.error_extras['reportable']
e.patch_message('\n'.join((str(e).rstrip(), help_msg)))
# ignore the FULL_TRACEBACKS config, this needs reporting!
raise e
def find_setupwiths(func_ir):
"""Find all top-level with.
Returns a list of ranges for the with-regions.
"""
def find_ranges(blocks):
cfg = compute_cfg_from_blocks(blocks)
sus_setups, sus_pops = set(), set()
# traverse the cfg and collect all suspected SETUP_WITH and POP_BLOCK
# statements so that we can iterate over them
for label, block in blocks.items():
for stmt in block.body:
if ir_utils.is_setup_with(stmt):
sus_setups.add(label)
if ir_utils.is_pop_block(stmt):
sus_pops.add(label)
# now that we do have the statements, iterate through them in reverse
# topo order and from each start looking for pop_blocks
setup_with_to_pop_blocks_map = defaultdict(set)
for setup_block in cfg.topo_sort(sus_setups, reverse=True):
# begin pop_block, search
to_visit, seen = [], []
to_visit.append(setup_block)
while to_visit:
# get whatever is next and record that we have seen it
block = to_visit.pop()
seen.append(block)
# go through the body of the block, looking for statements
for stmt in blocks[block].body:
# raise detected before pop_block
if ir_utils.is_raise(stmt):
raise errors.CompilerError(
'unsupported control flow due to raise '
'statements inside with block')
# special case 3.7, return before POP_BLOCK
if PYVERSION < (3, 8) and ir_utils.is_return(stmt):
raise errors.CompilerError(
'unsupported control flow: due to return '
'statements inside with block')
# if a pop_block, process it
if ir_utils.is_pop_block(stmt) and block in sus_pops:
# record the jump target of this block belonging to this setup
setup_with_to_pop_blocks_map[setup_block].add(block)
# remove the block from blocks to be matched
sus_pops.remove(block)
# stop looking, we have reached the frontier
break
# if we are still here, by the block terminator,
# add all its targets to the to_visit stack, unless we
# have seen them already
if ir_utils.is_terminator(stmt):
for t in stmt.get_targets():
if t not in seen:
to_visit.append(t)
return setup_with_to_pop_blocks_map
blocks = func_ir.blocks
# initial find, will return a dictionary, mapping indices of blocks
# containing SETUP_WITH statements to a set of indices of blocks containing
# POP_BLOCK statements
with_ranges_dict = find_ranges(blocks)
# rewrite the CFG in case there are multiple POP_BLOCK statements for one
# with
func_ir = consolidate_multi_exit_withs(with_ranges_dict, blocks, func_ir)
# here we need to turn the withs back into a list of tuples so that the
# rest of the code can cope
with_ranges_tuple = [(s, list(p)[0])
for (s, p) in with_ranges_dict.items()]
# check for POP_BLOCKS with multiple outgoing edges and reject
for (_, p) in with_ranges_tuple:
targets = blocks[p].terminator.get_targets()
if len(targets) != 1:
raise errors.CompilerError(
"unsupported control flow: with-context contains branches "
"(i.e. break/return/raise) that can leave the block ")
# now we check for returns inside with and reject them
for (_, p) in with_ranges_tuple:
target_block = blocks[p]
if ir_utils.is_return(func_ir.blocks[
target_block.terminator.get_targets()[0]].terminator):
if PYVERSION == (3, 8):
# 3.8 needs to bail here, if this is the case, because the
# later code can't handle it.
raise errors.CompilerError(
"unsupported control flow: due to return statements "
"inside with block")
_rewrite_return(func_ir, p)
# now we need to rewrite the tuple such that we have SETUP_WITH matching the
# successor of the block that contains the POP_BLOCK.
with_ranges_tuple = [(s, func_ir.blocks[p].terminator.get_targets()[0])
for (s, p) in with_ranges_tuple]
# finally we check for nested with statements and reject them
with_ranges_tuple = _eliminate_nested_withs(with_ranges_tuple)
return with_ranges_tuple, func_ir
def _runPass(self, index, pss, internal_state):
mutated = False
def check(func, compiler_state):
mangled = func(compiler_state)
if mangled not in (True, False):
msg = (
"CompilerPass implementations should return True/False. "
"CompilerPass with name '%s' did not.")
raise ValueError(msg % pss.name())
return mangled
def debug_print(pass_name, print_condition, printable_condition):
if pass_name in print_condition:
fid = internal_state.func_id
args = (fid.modname, fid.func_qualname, self.pipeline_name,
printable_condition, pass_name)
print(("%s.%s: %s: %s %s" % args).center(120, '-'))
if internal_state.func_ir is not None:
internal_state.func_ir.dump()
else:
print("func_ir is None")
# debug print before this pass?
debug_print(pss.name(), self._print_before + self._print_wrap,
"BEFORE")
# wire in the analysis info so it's accessible
pss.analysis = self._analysis
with SimpleTimer() as init_time:
mutated |= check(pss.run_initialization, internal_state)
with SimpleTimer() as pass_time:
mutated |= check(pss.run_pass, internal_state)
with SimpleTimer() as finalize_time:
mutated |= check(pss.run_finalizer, internal_state)
# Check that if the pass is an instance of a FunctionPass that it hasn't
# emitted ir.Dels.
if isinstance(pss, FunctionPass):
enforce_no_dels(internal_state.func_ir)
if self._ENFORCING:
# TODO: Add in self consistency enforcement for
# `func_ir._definitions` etc
if _pass_registry.get(pss.__class__).mutates_CFG:
if mutated: # block level changes, rebuild all
PostProcessor(internal_state.func_ir).run()
else: # CFG level changes rebuild CFG
internal_state.func_ir.blocks = transforms.canonicalize_cfg(
internal_state.func_ir.blocks)
# Check the func_ir has exactly one Scope instance
if not legalize_single_scope(internal_state.func_ir.blocks):
raise errors.CompilerError(
f"multiple scope in func_ir detected in {pss}", )
# inject runtimes
pt = pass_timings(init_time.elapsed, pass_time.elapsed,
finalize_time.elapsed)
self.exec_times["%s_%s" % (index, pss.name())] = pt
# debug print after this pass?
debug_print(pss.name(), self._print_after + self._print_wrap, "AFTER")