def typer(val):
if isinstance(val, (types.BaseTuple, types.Sequence)):
# Array constructor, e.g. np.int32([1, 2])
fnty = self.context.resolve_value_type(np.array)
sig = fnty.get_call_type(self.context, (val, types.DType(ty)),
{})
return sig.return_type
elif isinstance(val, (types.Number, types.Boolean, types.IntEnumMember)):
# Scalar constructor, e.g. np.int32(42)
return ty
elif isinstance(val, (types.NPDatetime, types.NPTimedelta)):
# Constructor cast from datetime-like, e.g.
# > np.int64(np.datetime64("2000-01-01"))
if ty.bitwidth == 64:
return ty
else:
msg = (f"Cannot cast {val} to {ty} as {ty} is not 64 bits "
"wide.")
raise errors.TypingError(msg)
else:
if (isinstance(val, types.Array) and val.ndim == 0 and
val.dtype == ty):
# This is 0d array -> scalar degrading
return ty
else:
# unsupported
msg = f"Casting {val} to {ty} directly is unsupported."
if isinstance(val, types.Array):
# array casts are supported a different way.
msg += f" Try doing '<array>.astype(np.{ty})' instead"
raise errors.TypingError(msg)
def _sort_check_key(key):
if isinstance(key, types.Optional):
msg = ("Key must concretely be None or a Numba JIT compiled function, "
"an Optional (union of None and a value) was found")
raise errors.TypingError(msg)
if not (cgutils.is_nonelike(key) or isinstance(key, types.Dispatcher)):
msg = "Key must be None or a Numba JIT compiled function"
raise errors.TypingError(msg)
def _legalize_arg_types(self, args):
for i, a in enumerate(args, start=1):
if isinstance(a, types.List):
msg = "Does not support list type inputs into with-context for arg {}"
raise errors.TypingError(msg.format(i))
elif isinstance(a, types.Dispatcher):
msg = (
"Does not support function type inputs into with-context for arg {}"
)
raise errors.TypingError(msg.format(i))
def literal_list_index(lst, x, start=0, end=_index_end):
# TODO: To make this work, need consts as slice for start/end so as to
# be able to statically analyse the bounds, then its a just loop body
# versioning based iteration along with enumerate to find the item
if isinstance(lst, types.LiteralList):
msg = "list.index is unsupported for literal lists"
raise errors.TypingError(msg)
def get_call_type(self, context, args, kws):
template = self.template(context)
literal_e = None
nonliteral_e = None
# Try with Literal
try:
out = template.apply(args, kws)
except Exception as exc:
if isinstance(exc, errors.ForceLiteralArg):
raise exc
literal_e = exc
out = None
# if the unliteral_args and unliteral_kws are the same as the literal
# ones, set up to not bother retrying
unliteral_args = tuple([unliteral(a) for a in args])
unliteral_kws = {k: unliteral(v) for k, v in kws.items()}
skip = unliteral_args == args and kws == unliteral_kws
# If the above template application failed and the non-literal args are
# different to the literal ones, try again with literals rewritten as
# non-literals
if not skip and out is None:
try:
out = template.apply(unliteral_args, unliteral_kws)
except Exception as exc:
if isinstance(exc, errors.ForceLiteralArg):
raise exc
nonliteral_e = exc
if out is None and (nonliteral_e is not None or literal_e is not None):
header = "- Resolution failure for {} arguments:\n{}\n"
tmplt = _termcolor.highlight(header)
if config.DEVELOPER_MODE:
indent = ' ' * 4
def add_bt(error):
if isinstance(error, BaseException):
# if the error is an actual exception instance, trace it
bt = traceback.format_exception(type(error), error,
error.__traceback__)
else:
bt = [""]
nd2indent = '\n{}'.format(2 * indent)
errstr += _termcolor.reset(nd2indent +
nd2indent.join(bt_as_lines))
return _termcolor.reset(errstr)
else:
add_bt = lambda X: ''
def nested_msg(literalness, e):
estr = str(e)
estr = estr if estr else (str(repr(e)) + add_bt(e))
new_e = errors.TypingError(textwrap.dedent(estr))
return tmplt.format(literalness, str(new_e))
raise errors.TypingError(nested_msg('literal', literal_e) +
nested_msg('non-literal', nonliteral_e))
return out
def _builtin_infer(self, py_type):
# The type hierarchy of python typing library changes in 3.7.
generic_type_check = _py_version_switch(
(3, 7),
lambda x: isinstance(x, py_typing._GenericAlias),
lambda _: True,
)
if not generic_type_check(py_type):
return
list_origin = _py_version_switch((3, 7), list, py_typing.List)
dict_origin = _py_version_switch((3, 7), dict, py_typing.Dict)
set_origin = _py_version_switch((3, 7), set, py_typing.Set)
tuple_origin = _py_version_switch((3, 7), tuple, py_typing.Tuple)
if getattr(py_type, "__origin__", None) is py_typing.Union:
if len(py_type.__args__) != 2:
raise errors.TypingError(
"Cannot type Union of more than two types")
(arg_1_py, arg_2_py) = py_type.__args__
if arg_2_py is type(None): # noqa: E721
return types.Optional(self.infer(arg_1_py))
elif arg_1_py is type(None): # noqa: E721
return types.Optional(self.infer(arg_2_py))
else:
raise errors.TypingError(
"Cannot type Union that is not an Optional "
f"(neither type type {arg_2_py} is not NoneType")
if getattr(py_type, "__origin__", None) is list_origin:
(element_py, ) = py_type.__args__
return types.ListType(self.infer(element_py))
if getattr(py_type, "__origin__", None) is dict_origin:
key_py, value_py = py_type.__args__
return types.DictType(self.infer(key_py), self.infer(value_py))
if getattr(py_type, "__origin__", None) is set_origin:
(element_py, ) = py_type.__args__
return types.Set(self.infer(element_py))
if getattr(py_type, "__origin__", None) is tuple_origin:
tys = tuple(map(self.infer, py_type.__args__))
return types.BaseTuple.from_types(tys)
def _unlit_non_poison(ty):
"""Apply unliteral(ty) and raise a TypingError if type is Poison.
"""
out = unliteral(ty)
if isinstance(out, types.Poison):
m = f"Poison type used in arguments; got {out}"
raise errors.TypingError(m)
return out
def get_type_limits(eltype):
np_dtype = numpy_support.as_dtype(eltype)
if isinstance(eltype, types.Integer):
return np.iinfo(np_dtype)
elif isinstance(eltype, types.Float):
return np.finfo(np_dtype)
else:
msg = 'Type {} not supported'.format(eltype)
raise errors.TypingError(msg)
def ol_set_num_threads(n):
_launch_threads()
if not isinstance(n, types.Integer):
msg = "The number of threads specified must be an integer"
raise errors.TypingError(msg)
def impl(n):
snt_check(n)
_set_num_threads(n)
return impl
def _sort_check_reverse(reverse):
if isinstance(reverse, types.Omitted):
rty = reverse.value
elif isinstance(reverse, types.Optional):
rty = reverse.type
else:
rty = reverse
if not isinstance(rty, (types.Boolean, types.Integer, int, bool)):
msg = "an integer is required for 'reverse' (got type %s)" % reverse
raise errors.TypingError(msg)
return rty
def add_return_type(self, return_type):
"""Add *return_type* to the list of inferred return-types.
If there are too many, raise `TypingError`.
"""
# The maximum limit is picked arbitrarily.
# Don't think that this needs to be user configurable.
RETTY_LIMIT = 16
self._inferred_retty.add(return_type)
if len(self._inferred_retty) >= RETTY_LIMIT:
m = "Return type of recursive function does not converge"
raise errors.TypingError(m)
def length_of_iterator(typingctx, val):
"""
An implementation of len(iter) for internal use.
Primary use is for array comprehensions (see inline_closurecall).
"""
if isinstance(val, types.RangeIteratorType):
val_type = val.yield_type
def codegen(context, builder, sig, args):
(value,) = args
iter_type = range_impl_map[val_type][1]
iterobj = cgutils.create_struct_proxy(iter_type)(context, builder, value)
int_type = iterobj.count.type
return impl_ret_untracked(context, builder, int_type, builder.load(iterobj.count))
return signature(val_type, val), codegen
elif isinstance(val, types.ListIter):
def codegen(context, builder, sig, args):
(value,) = args
intp_t = context.get_value_type(types.intp)
iterobj = ListIterInstance(context, builder, sig.args[0], value)
return impl_ret_untracked(context, builder, intp_t, iterobj.size)
return signature(types.intp, val), codegen
elif isinstance(val, types.ArrayIterator):
def codegen(context, builder, sig, args):
(iterty,) = sig.args
(value,) = args
intp_t = context.get_value_type(types.intp)
iterobj = context.make_helper(builder, iterty, value=value)
arrayty = iterty.array_type
ary = make_array(arrayty)(context, builder, value=iterobj.array)
shape = cgutils.unpack_tuple(builder, ary.shape)
# array iterates along the outer dimension
return impl_ret_untracked(context, builder, intp_t, shape[0])
return signature(types.intp, val), codegen
elif isinstance(val, types.UniTupleIter):
def codegen(context, builder, sig, args):
(iterty,) = sig.args
tuplety = iterty.container
intp_t = context.get_value_type(types.intp)
count_const = intp_t(tuplety.count)
return impl_ret_untracked(context, builder, intp_t, count_const)
return signature(types.intp, val), codegen
elif isinstance(val, types.ListTypeIteratorType):
def codegen(context, builder, sig, args):
(value,) = args
intp_t = context.get_value_type(types.intp)
from numba.typed.listobject import ListIterInstance
iterobj = ListIterInstance(context, builder, sig.args[0], value)
return impl_ret_untracked(context, builder, intp_t, iterobj.size)
return signature(types.intp, val), codegen
else:
msg = ('Unsupported iterator found in array comprehension, try '
'preallocating the array and filling manually.')
raise errors.TypingError(msg)
def ol_set_parallel_chunksize(n):
_launch_threads()
if not isinstance(n, types.Integer):
msg = "The parallel chunksize must be an integer"
raise errors.TypingError(msg)
def impl(n):
if n < 0:
raise ValueError("chunksize must be greater than or equal to zero")
return _set_parallel_chunksize(n)
return impl
def scalar_view(scalar, viewty):
""" Typing for the np scalar 'view' method. """
if (isinstance(scalar, (types.Float, types.Integer))
and isinstance(viewty, types.abstract.DTypeSpec)):
if scalar.bitwidth != viewty.dtype.bitwidth:
raise errors.TypingError(
"Changing the dtype of a 0d array is only supported if the "
"itemsize is unchanged")
def impl(scalar, viewty):
return viewer(scalar, viewty)
return impl
def generic(self, args, kws):
assert not kws
[obj] = args
if isinstance(obj, types.IterableType):
# Raise this here to provide a very specific message about this
# common issue, delaying the error until later leads to something
# less specific being noted as the problem (e.g. no support for
# getiter on array(<>, 2, 'C')).
if isinstance(obj, types.Array) and obj.ndim > 1:
msg = ("Direct iteration is not supported for arrays with "
"dimension > 1. Try using indexing instead.")
raise errors.TypingError(msg)
else:
return signature(obj.iterator_type, obj)
def try_infer(self, py_type):
"""
Try to determine the numba type of a given python type.
We first consider the lookup dictionary. If py_type is not there, we
iterate through the registered functions until one returns a numba type.
If type inference fails, return None.
"""
result = self.lookup.get(py_type, None)
for func in self.functions:
if result is not None:
break
result = func(py_type)
if result is not None and not isinstance(result, types.Type):
raise errors.TypingError(
f"as_numba_type should return a numba type, got {result}")
return result
def overload_where_arrays(cond, x, y):
"""
Implement where() for arrays.
"""
# Choose implementation based on argument types.
if isinstance(cond, types.Array):
if x.dtype != y.dtype:
raise errors.TypingError("x and y should have the same dtype")
# Array where() => return an array of the same shape
if all(ty.layout == "C" for ty in (cond, x, y)):
def where_impl(cond, x, y):
"""
Fast implementation for C-contiguous arrays
"""
shape = cond.shape
if x.shape != shape or y.shape != shape:
raise ValueError("all inputs should have the same shape")
res = np.empty_like(x)
cf = cond.flat
xf = x.flat
yf = y.flat
rf = res.flat
for i in range(cond.size):
rf[i] = xf[i] if cf[i] else yf[i]
return res
else:
def where_impl(cond, x, y):
"""
Generic implementation for other arrays
"""
shape = cond.shape
if x.shape != shape or y.shape != shape:
raise ValueError("all inputs should have the same shape")
res = np.empty_like(x)
for idx, c in np.ndenumerate(cond):
res[idx] = x[idx] if c else y[idx]
return res
return where_impl
def overload_where_scalars(cond, x, y):
"""
Implement where() for scalars.
"""
if not isinstance(cond, types.Array):
if x != y:
raise errors.TypingError("x and y should have the same type")
def where_impl(cond, x, y):
"""
Scalar where() => return a 0-dim array
"""
scal = x if cond else y
# Can't use full_like() on Numpy < 1.8
arr = np.empty_like(scal)
arr[()] = scal
return arr
return where_impl
def _inject_hashsecret_read(tyctx, name):
"""Emit code to load the hashsecret."""
if not isinstance(name, types.StringLiteral):
raise errors.TypingError("requires literal string")
sym = _hashsecret[name.literal_value].symbol
resty = types.uint64
sig = resty(name)
def impl(cgctx, builder, sig, args):
mod = builder.module
try:
# Search for existing global
gv = mod.get_global(sym)
except KeyError:
# Inject the symbol if not already exist.
gv = ir.GlobalVariable(mod, ir.IntType(64), name=sym)
v = builder.load(gv)
return v
return sig, impl
def resolve_output_type(context, inputs, formal_output):
"""
Given the array-compatible input types to an operation (e.g. ufunc),
and the operation's formal output type (a types.Array instance),
resolve the actual output type using the typing *context*.
This uses a mechanism compatible with Numpy's __array_priority__ /
__array_wrap__.
"""
selected_input = inputs[select_array_wrapper(inputs)]
args = selected_input, formal_output
sig = context.resolve_function_type('__array_wrap__', args, {})
if sig is None:
if selected_input.array_priority == types.Array.array_priority:
# If it's the same priority as a regular array, assume we
# should return the output unchanged.
# (we can't define __array_wrap__ explicitly for types.Buffer,
# as that would be inherited by most array-compatible objects)
return formal_output
raise errors.TypingError("__array_wrap__ failed for %s" % (args, ))
return sig.return_type
def typer(val):
if isinstance(val, (types.BaseTuple, types.Sequence)):
# Array constructor, e.g. np.int32([1, 2])
fnty = self.context.resolve_value_type(np.array)
sig = fnty.get_call_type(self.context, (val, types.DType(ty)),
{})
return sig.return_type
elif isinstance(val, (types.Number, types.Boolean)):
# Scalar constructor, e.g. np.int32(42)
return ty
else:
if (isinstance(val, types.Array) and val.ndim == 0
and val.dtype == ty):
# This is 0d array -> scalar degrading
return ty
else:
# unsupported
msg = f"Casting {val} to {ty} directly is unsupported."
if isinstance(val, types.Array):
# array casts are supported a different way.
msg += f" Try doing '<array>.astype(np.{ty})' instead"
raise errors.TypingError(msg)
def literal_list_setitem(lst, index, value):
if isinstance(lst, types.LiteralList):
raise errors.TypingError("Cannot mutate a literal list")
def nested_msg(literalness, e):
estr = str(e)
estr = estr if estr else (str(repr(e)) + add_bt(e))
new_e = errors.TypingError(textwrap.dedent(estr))
return tmplt.format(literalness, str(new_e))
def get_call_type(self, context, args, kws):
template = self.template(context)
literal_e = None
nonliteral_e = None
out = None
choice = [True, False] if template.prefer_literal else [False, True]
for uselit in choice:
if uselit:
# Try with Literal
try:
out = template.apply(args, kws)
except Exception as exc:
if (utils.use_new_style_errors()
and not isinstance(exc, errors.NumbaError)):
raise exc
if isinstance(exc, errors.ForceLiteralArg):
raise exc
literal_e = exc
out = None
else:
break
else:
# if the unliteral_args and unliteral_kws are the same as the
# literal ones, set up to not bother retrying
unliteral_args = tuple([_unlit_non_poison(a) for a in args])
unliteral_kws = {
k: _unlit_non_poison(v)
for k, v in kws.items()
}
skip = unliteral_args == args and kws == unliteral_kws
# If the above template application failed and the non-literal
# args are different to the literal ones, try again with
# literals rewritten as non-literals
if not skip and out is None:
try:
out = template.apply(unliteral_args, unliteral_kws)
except Exception as exc:
if isinstance(exc, errors.ForceLiteralArg):
if template.prefer_literal:
# For template that prefers literal types,
# reaching here means that the literal types
# have failed typing as well.
raise exc
nonliteral_e = exc
else:
break
if out is None and (nonliteral_e is not None or literal_e is not None):
header = "- Resolution failure for {} arguments:\n{}\n"
tmplt = _termcolor.highlight(header)
if config.DEVELOPER_MODE:
indent = ' ' * 4
def add_bt(error):
if isinstance(error, BaseException):
# if the error is an actual exception instance, trace it
bt = traceback.format_exception(
type(error), error, error.__traceback__)
else:
bt = [""]
nd2indent = '\n{}'.format(2 * indent)
errstr = _termcolor.reset(nd2indent +
nd2indent.join(_bt_as_lines(bt)))
return _termcolor.reset(errstr)
else:
add_bt = lambda X: ''
def nested_msg(literalness, e):
estr = str(e)
estr = estr if estr else (str(repr(e)) + add_bt(e))
new_e = errors.TypingError(textwrap.dedent(estr))
return tmplt.format(literalness, str(new_e))
raise errors.TypingError(
nested_msg('literal', literal_e) +
nested_msg('non-literal', nonliteral_e))
return out
def raise_error(self):
for faillist in self._failures.values():
for fail in faillist:
if isinstance(fail.error, errors.ForceLiteralArg):
raise fail.error
raise errors.TypingError(self.format())
def mutate_with_body(self, func_ir, blocks, blk_start, blk_end,
body_blocks, dispatcher_factory, extra):
typeanns = self._legalize_args(extra, loc=blocks[blk_start].loc)
vlt = func_ir.variable_lifetime
inputs, outputs = find_region_inout_vars(
blocks=blocks,
livemap=vlt.livemap,
callfrom=blk_start,
returnto=blk_end,
body_block_ids=set(body_blocks),
)
# Determine types in the output tuple
def strip_var_ver(x):
return x.split('.', 1)[0]
stripped_outs = list(map(strip_var_ver, outputs))
# Verify that only outputs are annotated
extra_annotated = set(typeanns) - set(stripped_outs)
if extra_annotated:
msg = ('Invalid type annotation on non-outgoing variables: {}.'
'Suggestion: remove annotation of the listed variables')
raise errors.TypingError(msg.format(extra_annotated))
# Verify that all outputs are annotated
not_annotated = set(stripped_outs) - set(typeanns)
if not_annotated:
msg = ('missing type annotation on outgoing variables: {0}'
'Example code: with objmode({0}=<add_type_as_str_here>):')
raise errors.TypingError(msg.format(not_annotated))
# Get output types
outtup = types.Tuple([typeanns[v] for v in stripped_outs])
lifted_blks = {k: blocks[k] for k in body_blocks}
_mutate_with_block_callee(lifted_blks, blk_start, blk_end, inputs,
outputs)
lifted_ir = func_ir.derive(
blocks=lifted_blks,
arg_names=tuple(inputs),
arg_count=len(inputs),
force_non_generator=True,
)
dispatcher = dispatcher_factory(lifted_ir,
objectmode=True,
output_types=outtup)
newblk = _mutate_with_block_caller(
dispatcher,
blocks,
blk_start,
blk_end,
inputs,
outputs,
)
blocks[blk_start] = newblk
_clear_blocks(blocks, body_blocks)
return dispatcher
def raise_error(self):
for _tempcls, e in self._failures:
if isinstance(e, errors.ForceLiteralArg):
raise e
raise errors.TypingError(self.format())
# -----------------------------------------------------------------------------
# Implicit casting
@lower_cast(types.List, types.List)
def list_to_list(context, builder, fromty, toty, val):
# Casting from non-reflected to reflected
assert fromty.dtype == toty.dtype
return val
# -----------------------------------------------------------------------------
# Implementations for types.LiteralList
# -----------------------------------------------------------------------------
_banned_error = errors.TypingError("Cannot mutate a literal list")
# Things that mutate literal lists are banned
@overload_method(types.LiteralList, 'append')
def literal_list_banned_append(lst, obj):
raise _banned_error
@overload_method(types.LiteralList, 'extend')
def literal_list_banned_extend(lst, iterable):
raise _banned_error
@overload_method(types.LiteralList, 'insert')
def literal_list_banned_insert(lst, index, obj):
def mutate_with_body(self, func_ir, blocks, blk_start, blk_end,
body_blocks, dispatcher_factory, extra):
cellnames = func_ir.func_id.func.__code__.co_freevars
closures = func_ir.func_id.func.__closure__
func_globals = func_ir.func_id.func.__globals__
if closures is not None:
# Resolve free variables
func_closures = {}
for cellname, closure in zip(cellnames, closures):
try:
cellval = closure.cell_contents
except ValueError as e:
# empty cell will raise
if str(e) != "Cell is empty":
raise
else:
func_closures[cellname] = cellval
else:
# Missing closure object
func_closures = {}
args = extra['args'] if extra else ()
kwargs = extra['kwargs'] if extra else {}
typeanns = self._legalize_args(func_ir=func_ir,
args=args,
kwargs=kwargs,
loc=blocks[blk_start].loc,
func_globals=func_globals,
func_closures=func_closures,
)
vlt = func_ir.variable_lifetime
inputs, outputs = find_region_inout_vars(
blocks=blocks,
livemap=vlt.livemap,
callfrom=blk_start,
returnto=blk_end,
body_block_ids=set(body_blocks),
)
# Determine types in the output tuple
def strip_var_ver(x):
return x.split('.', 1)[0]
stripped_outs = list(map(strip_var_ver, outputs))
# Verify that only outputs are annotated
extra_annotated = set(typeanns) - set(stripped_outs)
if extra_annotated:
msg = (
'Invalid type annotation on non-outgoing variables: {}.'
'Suggestion: remove annotation of the listed variables'
)
raise errors.TypingError(msg.format(extra_annotated))
# Verify that all outputs are annotated
# Note on "$cp" variable:
# ``transforms.consolidate_multi_exit_withs()`` introduces the variable
# for the control-point to determine the correct exit block. This
# variable crosses the with-region boundary. Thus, it will be consider
# an output variable leaving the lifted with-region.
typeanns["$cp"] = types.int32
not_annotated = set(stripped_outs) - set(typeanns)
if not_annotated:
msg = (
'Missing type annotation on outgoing variable(s): {0}\n\n'
'Example code: with objmode({1}=\'<'
'add_type_as_string_here>\')\n'
)
stable_ann = sorted(not_annotated)
raise errors.TypingError(msg.format(stable_ann, stable_ann[0]))
# Get output types
outtup = types.Tuple([typeanns[v] for v in stripped_outs])
lifted_blks = {k: blocks[k] for k in body_blocks}
_mutate_with_block_callee(lifted_blks, blk_start, blk_end,
inputs, outputs)
lifted_ir = func_ir.derive(
blocks=lifted_blks,
arg_names=tuple(inputs),
arg_count=len(inputs),
force_non_generator=True,
)
dispatcher = dispatcher_factory(lifted_ir, objectmode=True,
output_types=outtup)
newblk = _mutate_with_block_caller(
dispatcher, blocks, blk_start, blk_end, inputs, outputs,
)
blocks[blk_start] = newblk
_clear_blocks(blocks, body_blocks)
return dispatcher
def literal_list_getitem(lst, *args):
if not isinstance(lst, types.LiteralList):
return
msg = ("Cannot __getitem__ on a literal list, return type cannot be "
"statically determined.")
raise errors.TypingError(msg)