def _copy_payload(self, src_payload):
"""
Raw-copy the given payload into self.
"""
context = self._context
builder = self._builder
ok = cgutils.alloca_once_value(builder, cgutils.true_bit)
intp_t = context.get_value_type(types.intp)
zero = ir.Constant(intp_t, 0)
one = ir.Constant(intp_t, 1)
payload_type = context.get_data_type(types.SetPayload(self._ty))
payload_size = context.get_abi_sizeof(payload_type)
entry_size = self._entrysize
# Account for the fact that the payload struct already contains an entry
payload_size -= entry_size
mask = src_payload.mask
nentries = builder.add(one, mask)
# Total allocation size = <payload header size> + nentries * entry_size
# (note there can't be any overflow since we're reusing an existing
# payload's parameters)
allocsize = builder.add(
ir.Constant(intp_t, payload_size),
builder.mul(ir.Constant(intp_t, entry_size), nentries))
with builder.if_then(builder.load(ok), likely=True):
meminfo = context.nrt.meminfo_new_varsize(builder, size=allocsize)
alloc_ok = cgutils.is_null(builder, meminfo)
with builder.if_else(cgutils.is_null(builder, meminfo),
likely=False) as (if_error, if_ok):
with if_error:
builder.store(cgutils.false_bit, ok)
with if_ok:
self._set.meminfo = meminfo
payload = self.payload
payload.used = src_payload.used
payload.fill = src_payload.fill
payload.finger = zero
payload.mask = mask
cgutils.raw_memcpy(builder, payload.entries,
src_payload.entries, nentries,
entry_size)
if DEBUG_ALLOCS:
context.printf(
builder,
"allocated %zd bytes for set at %p: mask = %zd\n",
allocsize, payload.ptr, mask)
return builder.load(ok)
def _copy_payload(self, src_payload):
"""
Raw-copy the given payload into self.
"""
context = self._context
builder = self._builder
ok = cgutils.alloca_once_value(builder, cgutils.true_bit)
intp_t = context.get_value_type(types.intp)
zero = ir.Constant(intp_t, 0)
one = ir.Constant(intp_t, 1)
payload_type = context.get_data_type(types.SetPayload(self._ty))
payload_size = context.get_abi_sizeof(payload_type)
entry_size = self._entrysize
# Account for the fact that the payload struct already contains an entry
payload_size -= entry_size
mask = src_payload.mask
nentries = builder.add(one, mask)
# Total allocation size = <payload header size> + nentries * entry_size
# (note there can't be any overflow since we're reusing an existing
# payload's parameters)
allocsize = builder.add(ir.Constant(intp_t, payload_size),
builder.mul(ir.Constant(intp_t, entry_size),
nentries))
with builder.if_then(builder.load(ok), likely=True):
meminfo = context.nrt.meminfo_new_varsize(builder, size=allocsize)
alloc_ok = cgutils.is_null(builder, meminfo)
with builder.if_else(cgutils.is_null(builder, meminfo),
likely=False) as (if_error, if_ok):
with if_error:
builder.store(cgutils.false_bit, ok)
with if_ok:
self._set.meminfo = meminfo
payload = self.payload
payload.used = src_payload.used
payload.fill = src_payload.fill
payload.finger = zero
payload.mask = mask
cgutils.raw_memcpy(builder, payload.entries,
src_payload.entries, nentries,
entry_size)
if DEBUG_ALLOCS:
context.printf(builder,
"allocated %zd bytes for set at %p: mask = %zd\n",
allocsize, payload.ptr, mask)
return builder.load(ok)
def is_leap_year(builder, year_val):
"""
Return a predicate indicating whether *year_val* (offset by 1970) is a
leap year.
"""
actual_year = builder.add(year_val, Constant.int(DATETIME64, 1970))
multiple_of_4 = cgutils.is_null(
builder, builder.and_(actual_year, Constant.int(DATETIME64, 3)))
not_multiple_of_100 = cgutils.is_not_null(
builder, builder.srem(actual_year, Constant.int(DATETIME64, 100)))
multiple_of_400 = cgutils.is_null(
builder, builder.srem(actual_year, Constant.int(DATETIME64, 400)))
return builder.and_(multiple_of_4,
builder.or_(not_multiple_of_100, multiple_of_400))
def is_leap_year(builder, year_val):
"""
Return a predicate indicating whether *year_val* (offset by 1970) is a
leap year.
"""
actual_year = builder.add(year_val, Constant.int(DATETIME64, 1970))
multiple_of_4 = cgutils.is_null(
builder, builder.and_(actual_year, Constant.int(DATETIME64, 3)))
not_multiple_of_100 = cgutils.is_not_null(
builder, builder.srem(actual_year, Constant.int(DATETIME64, 100)))
multiple_of_400 = cgutils.is_null(
builder, builder.srem(actual_year, Constant.int(DATETIME64, 400)))
return builder.and_(multiple_of_4,
builder.or_(not_multiple_of_100, multiple_of_400))
def to_native_arg(self, obj, typ):
if isinstance(typ, types.Record):
# Generate a dummy integer type that has the size of Py_buffer
dummy_py_buffer_type = Type.int(_helperlib.py_buffer_size * 8)
# Allocate the Py_buffer
py_buffer = cgutils.alloca_once(self.builder, dummy_py_buffer_type)
# Zero-fill the py_buffer. where the obj field in Py_buffer is NULL
# PyBuffer_Release has no effect.
zeroed_buffer = lc.Constant.null(dummy_py_buffer_type)
self.builder.store(zeroed_buffer, py_buffer)
buf_as_voidptr = self.builder.bitcast(py_buffer, self.voidptr)
ptr = self.extract_record_data(obj, buf_as_voidptr)
with cgutils.if_unlikely(self.builder,
cgutils.is_null(self.builder, ptr)):
self.builder.ret(ptr)
ltyp = self.context.get_value_type(typ)
val = cgutils.init_record_by_ptr(self.builder, ltyp, ptr)
def dtor():
self.release_record_buffer(buf_as_voidptr)
else:
val = self.to_native_value(obj, typ)
def dtor():
pass
return val, dtor
def _unbox_native_field(typ, obj, field_name: str, c):
ret_ptr = cgutils.alloca_once(c.builder, c.context.get_value_type(typ))
is_error_ptr = cgutils.alloca_once_value(c.builder, cgutils.false_bit)
fail_obj = c.context.get_constant_null(typ)
with local_return(c.builder) as ret:
fail_blk = c.builder.append_basic_block("fail")
with c.builder.goto_block(fail_blk):
c.builder.store(cgutils.true_bit, is_error_ptr)
c.builder.store(fail_obj, ret_ptr)
ret()
field_obj = c.pyapi.object_getattr_string(obj, field_name)
with cgutils.if_unlikely(c.builder,
cgutils.is_null(c.builder, field_obj)):
c.builder.branch(fail_blk)
field_native = c.unbox(typ, field_obj)
c.pyapi.decref(field_obj)
with cgutils.if_unlikely(c.builder, field_native.is_error):
c.builder.branch(fail_blk)
c.builder.store(cgutils.false_bit, is_error_ptr)
c.builder.store(field_native.value, ret_ptr)
return NativeValue(c.builder.load(ret_ptr),
is_error=c.builder.load(is_error_ptr))
def to_native_arg(self, obj, typ):
if isinstance(typ, types.Record):
# Generate a dummy integer type that has the size of Py_buffer
dummy_py_buffer_type = Type.int(_helperlib.py_buffer_size * 8)
# Allocate the Py_buffer
py_buffer = cgutils.alloca_once(self.builder, dummy_py_buffer_type)
# Zero-fill the py_buffer. where the obj field in Py_buffer is NULL
# PyBuffer_Release has no effect.
zeroed_buffer = lc.Constant.null(dummy_py_buffer_type)
self.builder.store(zeroed_buffer, py_buffer)
buf_as_voidptr = self.builder.bitcast(py_buffer, self.voidptr)
ptr = self.extract_record_data(obj, buf_as_voidptr)
with cgutils.if_unlikely(self.builder,
cgutils.is_null(self.builder, ptr)):
self.builder.ret(ptr)
ltyp = self.context.get_value_type(typ)
val = cgutils.init_record_by_ptr(self.builder, ltyp, ptr)
def dtor():
self.release_record_buffer(buf_as_voidptr)
else:
val = self.to_native_value(obj, typ)
def dtor():
pass
return val, dtor
def set_pop(context, builder, sig, args):
inst = SetInstance(context, builder, sig.args[0], args[0])
used = inst.payload.used
with builder.if_then(cgutils.is_null(builder, used), likely=False):
context.call_conv.return_user_exc(builder, KeyError,
("set.pop(): empty set", ))
return inst.pop()
def set_pop(context, builder, sig, args):
inst = SetInstance(context, builder, sig.args[0], args[0])
used = inst.payload.used
with builder.if_then(cgutils.is_null(builder, used), likely=False):
context.call_conv.return_user_exc(builder, KeyError,
("set.pop(): empty set",))
return inst.pop()
def _emit_python_wrapper(self, llvm_module):
# Figure out the Python C API module creation function, and
# get a LLVM function for it.
create_module_fn = llvm_module.add_function(
*self.module_create_definition)
create_module_fn.linkage = lc.LINKAGE_EXTERNAL
# Define a constant string for the module name.
mod_name_init = lc.Constant.stringz(self.module_name)
mod_name_const = llvm_module.add_global_variable(
mod_name_init.type, '.module_name')
mod_name_const.initializer = mod_name_init
mod_name_const.linkage = lc.LINKAGE_INTERNAL
mod_def_base_init = lc.Constant.struct((
lt._pyobject_head_init, # PyObject_HEAD
lc.Constant.null(self.m_init_ty), # m_init
lc.Constant.null(lt._llvm_py_ssize_t), # m_index
lc.Constant.null(lt._pyobject_head_p), # m_copy
))
mod_def_base = llvm_module.add_global_variable(mod_def_base_init.type,
'.module_def_base')
mod_def_base.initializer = mod_def_base_init
mod_def_base.linkage = lc.LINKAGE_INTERNAL
mod_def_init = lc.Constant.struct((
mod_def_base_init, # m_base
lc.Constant.gep(mod_name_const, [ZERO, ZERO]), # m_name
lc.Constant.null(self._char_star), # m_doc
lc.Constant.int(lt._llvm_py_ssize_t, -1), # m_size
self._emit_method_array(llvm_module), # m_methods
lc.Constant.null(self.inquiry_ty), # m_reload
lc.Constant.null(self.traverseproc_ty), # m_traverse
lc.Constant.null(self.inquiry_ty), # m_clear
lc.Constant.null(self.freefunc_ty) # m_free
))
# Define a constant string for the module name.
mod_def = llvm_module.add_global_variable(mod_def_init.type,
'.module_def')
mod_def.initializer = mod_def_init
mod_def.linkage = lc.LINKAGE_INTERNAL
# Define the module initialization function.
mod_init_fn = llvm_module.add_function(*self.module_init_definition)
entry = mod_init_fn.append_basic_block('Entry')
builder = lc.Builder.new(entry)
mod = builder.call(
create_module_fn,
(mod_def, lc.Constant.int(lt._int32, sys.api_version)))
# Test if module has been created correctly.
# (XXX for some reason comparing with the NULL constant fails llvm
# with an assertion in pydebug mode)
with builder.if_then(cgutils.is_null(builder, mod)):
builder.ret(NULL.bitcast(mod_init_fn.type.pointee.return_type))
builder.ret(mod)
def _emit_python_wrapper(self, llvm_module):
# Figure out the Python C API module creation function, and
# get a LLVM function for it.
create_module_fn = llvm_module.add_function(*self.module_create_definition)
create_module_fn.linkage = lc.LINKAGE_EXTERNAL
# Define a constant string for the module name.
mod_name_init = lc.Constant.stringz(self.module_name)
mod_name_const = llvm_module.add_global_variable(mod_name_init.type, '.module_name')
mod_name_const.initializer = mod_name_init
mod_name_const.linkage = lc.LINKAGE_INTERNAL
mod_def_base_init = lc.Constant.struct(
(lt._pyobject_head_init, # PyObject_HEAD
lc.Constant.null(self.m_init_ty), # m_init
lc.Constant.null(lt._llvm_py_ssize_t), # m_index
lc.Constant.null(lt._pyobject_head_p), # m_copy
)
)
mod_def_base = llvm_module.add_global_variable(mod_def_base_init.type, '.module_def_base')
mod_def_base.initializer = mod_def_base_init
mod_def_base.linkage = lc.LINKAGE_INTERNAL
mod_def_init = lc.Constant.struct(
(mod_def_base_init, # m_base
lc.Constant.gep(mod_name_const, [ZERO, ZERO]), # m_name
lc.Constant.null(self._char_star), # m_doc
lc.Constant.int(lt._llvm_py_ssize_t, -1), # m_size
self._emit_method_array(llvm_module), # m_methods
lc.Constant.null(self.inquiry_ty), # m_reload
lc.Constant.null(self.traverseproc_ty), # m_traverse
lc.Constant.null(self.inquiry_ty), # m_clear
lc.Constant.null(self.freefunc_ty) # m_free
)
)
# Define a constant string for the module name.
mod_def = llvm_module.add_global_variable(mod_def_init.type, '.module_def')
mod_def.initializer = mod_def_init
mod_def.linkage = lc.LINKAGE_INTERNAL
# Define the module initialization function.
mod_init_fn = llvm_module.add_function(*self.module_init_definition)
entry = mod_init_fn.append_basic_block('Entry')
builder = lc.Builder.new(entry)
mod = builder.call(create_module_fn,
(mod_def,
lc.Constant.int(lt._int32, sys.api_version)))
# Test if module has been created correctly.
# (XXX for some reason comparing with the NULL constant fails llvm
# with an assertion in pydebug mode)
with builder.if_then(cgutils.is_null(builder, mod)):
builder.ret(NULL.bitcast(mod_init_fn.type.pointee.return_type))
builder.ret(mod)
self.dll_exports.append(mod_init_fn.name)
def imp(context, builder, typ, value):
api = context.get_python_api(builder)
aval = api.object_getattr_string(value, attr)
with cgutils.ifthen(builder, cgutils.is_null(builder, aval)):
context.return_exc(builder)
if isinstance(atyp, types.Method):
return aval
else:
nativevalue = api.to_native_value(aval, atyp)
api.decref(aval)
return nativevalue
def imp(context, builder, typ, value):
api = context.get_python_api(builder)
aval = api.object_getattr_string(value, attr)
with cgutils.ifthen(builder, cgutils.is_null(builder, aval)):
context.return_exc(builder)
if isinstance(atyp, types.Method):
return aval
else:
nativevalue = api.to_native_value(aval, atyp)
api.decref(aval)
return nativevalue
def eh_check(self, builder):
"""Check if an exception is raised
"""
ctx = self._context
cc = ctx.call_conv
# Inspect the excinfo argument on the function
trystatus = cc.check_try_status(builder)
excinfo = trystatus.excinfo
has_raised = builder.not_(cgutils.is_null(builder, excinfo))
with builder.if_then(has_raised):
self.eh_end_try(builder)
return has_raised
def get_env_from_closure(self, builder, clo):
"""
From the pointer *clo* to a _dynfunc.Closure, get a pointer
to the enclosed _dynfunc.Environment.
"""
with cgutils.if_unlikely(builder, cgutils.is_null(builder, clo)):
self.debug_print(builder, "Fatal error: missing _dynfunc.Closure")
builder.unreachable()
clo_body_ptr = cgutils.pointer_add(
builder, clo, _dynfunc._impl_info['offsetof_closure_body'])
clo_body = ClosureBody(self, builder, ref=clo_body_ptr, cast_ref=True)
return clo_body.env
def get_env_from_closure(self, builder, clo):
"""
From the pointer *clo* to a _dynfunc.Closure, get a pointer
to the enclosed _dynfunc.Environment.
"""
with cgutils.if_unlikely(builder, cgutils.is_null(builder, clo)):
self.debug_print(builder, "Fatal error: missing _dynfunc.Closure")
builder.unreachable()
clo_body_ptr = cgutils.pointer_add(
builder, clo, _dynfunc._impl_info['offsetof_closure_body'])
clo_body = ClosureBody(self, builder, ref=clo_body_ptr, cast_ref=True)
return clo_body.env
def emit_environment_sentry(self, envptr, return_pyobject=False):
"""Emits LLVM code to ensure the `envptr` is not NULL
"""
is_null = cgutils.is_null(self.builder, envptr)
with cgutils.if_unlikely(self.builder, is_null):
if return_pyobject:
fnty = self.builder.function.type.pointee
assert fnty.return_type == self.pyobj
self.err_set_string("PyExc_RuntimeError",
"missing Environment")
self.builder.ret(self.get_null_object())
else:
self.context.call_conv.return_user_exc(
self.builder, RuntimeError, ("missing Environment", ))
def store(retval):
is_error = cgutils.is_null(builder, retval)
with cgutils.ifelse(builder, is_error) as (if_error, if_ok):
with if_error:
msg = context.insert_const_string(pyapi.module,
"object mode ufunc")
msgobj = pyapi.string_from_string(msg)
pyapi.err_write_unraisable(msgobj)
pyapi.decref(msgobj)
with if_ok:
# Unbox
retval = pyapi.to_native_value(retval, signature.return_type)
# Store
out.store_direct(retval, builder.load(store_offset))
def store(retval):
is_error = cgutils.is_null(builder, retval)
with cgutils.ifelse(builder, is_error) as (if_error, if_ok):
with if_error:
msg = context.insert_const_string(pyapi.module,
"object mode ufunc")
msgobj = pyapi.string_from_string(msg)
pyapi.err_write_unraisable(msgobj)
pyapi.decref(msgobj)
with if_ok:
# Unbox
retval = pyapi.to_native_value(retval, signature.return_type)
# Store
out.store_direct(retval, builder.load(store_offset))
def emit_environment_sentry(self, envptr, return_pyobject=False):
"""Emits LLVM code to ensure the `envptr` is not NULL
"""
is_null = cgutils.is_null(self.builder, envptr)
with cgutils.if_unlikely(self.builder, is_null):
if return_pyobject:
fnty = self.builder.function.type.pointee
assert fnty.return_type == self.pyobj
self.err_set_string("PyExc_RuntimeError",
"missing Environment")
self.builder.ret(self.get_null_object())
else:
self.context.call_conv.return_user_exc(self.builder,
RuntimeError,
("missing Environment",))
def allocate_ex(cls, context, builder, list_type, nitems):
"""
Allocate a ListInstance with its storage.
Return a (ok, instance) tuple where *ok* is a LLVM boolean and
*instance* is a ListInstance object (the object's contents are
only valid when *ok* is true).
"""
intp_t = context.get_value_type(types.intp)
if isinstance(nitems, int):
nitems = ir.Constant(intp_t, nitems)
payload_type = context.get_data_type(types.ListPayload(list_type))
payload_size = context.get_abi_sizeof(payload_type)
itemsize = get_itemsize(context, list_type)
# Account for the fact that the payload struct contains one entry
payload_size -= itemsize
ok = cgutils.alloca_once_value(builder, cgutils.true_bit)
self = cls(context, builder, list_type, None)
# Total allocation size = <payload header size> + nitems * itemsize
allocsize, ovf = cgutils.muladd_with_overflow(
builder, nitems, ir.Constant(intp_t, itemsize),
ir.Constant(intp_t, payload_size))
with builder.if_then(ovf, likely=False):
builder.store(cgutils.false_bit, ok)
with builder.if_then(builder.load(ok), likely=True):
meminfo = context.nrt.meminfo_new_varsize_dtor(
builder, size=allocsize, dtor=self.get_dtor())
with builder.if_else(cgutils.is_null(builder, meminfo),
likely=False) as (if_error, if_ok):
with if_error:
builder.store(cgutils.false_bit, ok)
with if_ok:
self._list.meminfo = meminfo
self._list.parent = context.get_constant_null(
types.pyobject)
self._payload.allocated = nitems
self._payload.size = ir.Constant(intp_t, 0) # for safety
self._payload.dirty = cgutils.false_bit
# Zero the allocated region
self.zfill(self.size.type(0), nitems)
return builder.load(ok), self
def allocate_ex(cls, context, builder, list_type, nitems):
"""
Allocate a ListInstance with its storage.
Return a (ok, instance) tuple where *ok* is a LLVM boolean and
*instance* is a ListInstance object (the object's contents are
only valid when *ok* is true).
"""
intp_t = context.get_value_type(types.intp)
if isinstance(nitems, int):
nitems = ir.Constant(intp_t, nitems)
payload_type = context.get_data_type(types.ListPayload(list_type))
payload_size = context.get_abi_sizeof(payload_type)
itemsize = get_itemsize(context, list_type)
# Account for the fact that the payload struct contains one entry
payload_size -= itemsize
ok = cgutils.alloca_once_value(builder, cgutils.true_bit)
self = cls(context, builder, list_type, None)
# Total allocation size = <payload header size> + nitems * itemsize
allocsize, ovf = cgutils.muladd_with_overflow(builder, nitems,
ir.Constant(intp_t, itemsize),
ir.Constant(intp_t, payload_size))
with builder.if_then(ovf, likely=False):
builder.store(cgutils.false_bit, ok)
with builder.if_then(builder.load(ok), likely=True):
meminfo = context.nrt.meminfo_new_varsize_dtor(
builder, size=allocsize, dtor=self.get_dtor())
with builder.if_else(cgutils.is_null(builder, meminfo),
likely=False) as (if_error, if_ok):
with if_error:
builder.store(cgutils.false_bit, ok)
with if_ok:
self._list.meminfo = meminfo
self._list.parent = context.get_constant_null(types.pyobject)
self._payload.allocated = nitems
self._payload.size = ir.Constant(intp_t, 0) # for safety
self._payload.dirty = cgutils.false_bit
# Zero the allocated region
self.zfill(self.size.type(0), nitems)
return builder.load(ok), self
def check_element_type(nth, itemobj, expected_typobj):
typobj = nth.typeof(itemobj)
# Check if *typobj* is NULL
with c.builder.if_then(
cgutils.is_null(c.builder, typobj),
likely=False,
):
c.builder.store(cgutils.true_bit, errorptr)
loop.do_break()
# Mandate that objects all have the same exact type
type_mismatch = c.builder.icmp_signed('!=', typobj, expected_typobj)
with c.builder.if_then(type_mismatch, likely=False):
c.builder.store(cgutils.true_bit, errorptr)
c.pyapi.err_format(
"PyExc_TypeError",
"can't unbox heterogeneous list: %S != %S",
expected_typobj, typobj,
)
c.pyapi.decref(typobj)
loop.do_break()
c.pyapi.decref(typobj)
def call_class_method(self, builder, func, signature, args):
api = self.get_python_api(builder)
tys = signature.args
retty = signature.return_type
pyargs = [api.from_native_value(av, at) for av, at in zip(args, tys)]
res = api.call_function_objargs(func, pyargs)
# clean up
api.decref(func)
for obj in pyargs:
api.decref(obj)
with cgutils.ifthen(builder, cgutils.is_null(builder, res)):
self.return_exc(builder)
if retty == types.none:
api.decref(res)
return self.get_dummy_value()
else:
nativeresult = api.to_native_value(res, retty)
api.decref(res)
return nativeresult
def _prepare_call_to_object_mode(context, builder, func, signature, args, env):
mod = cgutils.get_module(builder)
thisfunc = cgutils.get_function(builder)
bb_core_return = thisfunc.append_basic_block('ufunc.core.return')
pyapi = context.get_python_api(builder)
# Call to
# PyObject* ndarray_new(int nd,
# npy_intp *dims, /* shape */
# npy_intp *strides,
# void* data,
# int type_num,
# int itemsize)
ll_int = context.get_value_type(types.int32)
ll_intp = context.get_value_type(types.intp)
ll_intp_ptr = Type.pointer(ll_intp)
ll_voidptr = context.get_value_type(types.voidptr)
ll_pyobj = context.get_value_type(types.pyobject)
fnty = Type.function(
ll_pyobj,
[ll_int, ll_intp_ptr, ll_intp_ptr, ll_voidptr, ll_int, ll_int])
fn_array_new = mod.get_or_insert_function(fnty, name="numba_ndarray_new")
# Convert each llarray into pyobject
error_pointer = cgutils.alloca_once(builder, Type.int(1), name='error')
builder.store(cgutils.true_bit, error_pointer)
ndarray_pointers = []
ndarray_objects = []
for i, (arr, arrtype) in enumerate(zip(args, signature.args)):
ptr = cgutils.alloca_once(builder, ll_pyobj)
ndarray_pointers.append(ptr)
builder.store(Constant.null(ll_pyobj), ptr) # initialize to NULL
arycls = context.make_array(arrtype)
array = arycls(context, builder, ref=arr)
zero = Constant.int(ll_int, 0)
# Extract members of the llarray
nd = Constant.int(ll_int, arrtype.ndim)
dims = builder.gep(array._get_ptr_by_name('shape'), [zero, zero])
strides = builder.gep(array._get_ptr_by_name('strides'), [zero, zero])
data = builder.bitcast(array.data, ll_voidptr)
dtype = np.dtype(str(arrtype.dtype))
# Prepare other info for reconstruction of the PyArray
type_num = Constant.int(ll_int, dtype.num)
itemsize = Constant.int(ll_int, dtype.itemsize)
# Call helper to reconstruct PyArray objects
obj = builder.call(fn_array_new,
[nd, dims, strides, data, type_num, itemsize])
builder.store(obj, ptr)
ndarray_objects.append(obj)
obj_is_null = cgutils.is_null(builder, obj)
builder.store(obj_is_null, error_pointer)
cgutils.cbranch_or_continue(builder, obj_is_null, bb_core_return)
# Call ufunc core function
object_sig = [types.pyobject] * len(ndarray_objects)
status, retval = context.call_conv.call_function(builder,
func,
ll_pyobj,
object_sig,
ndarray_objects,
env=env)
builder.store(status.is_error, error_pointer)
# Release returned object
pyapi.decref(retval)
builder.branch(bb_core_return)
# At return block
builder.position_at_end(bb_core_return)
# Release argument object
for ndary_ptr in ndarray_pointers:
pyapi.decref(builder.load(ndary_ptr))
innercall = status.code
return innercall, builder.load(error_pointer)
def box_COO(typ: COOType, val: "some LLVM thing", c) -> COO:
ret_ptr = cgutils.alloca_once(c.builder, c.pyapi.pyobj)
fail_obj = c.pyapi.get_null_object()
coo = cgutils.create_struct_proxy(typ)(c.context, c.builder, value=val)
with local_return(c.builder) as ret:
data_obj = c.box(typ.data_type, coo.data)
with cgutils.if_unlikely(c.builder,
cgutils.is_null(c.builder, data_obj)):
c.builder.store(fail_obj, ret_ptr)
ret()
coords_obj = c.box(typ.coords_type, coo.coords)
with cgutils.if_unlikely(c.builder,
cgutils.is_null(c.builder, coords_obj)):
c.pyapi.decref(data_obj)
c.builder.store(fail_obj, ret_ptr)
ret()
shape_obj = c.box(typ.shape_type, coo.shape)
with cgutils.if_unlikely(c.builder,
cgutils.is_null(c.builder, shape_obj)):
c.pyapi.decref(coords_obj)
c.pyapi.decref(data_obj)
c.builder.store(fail_obj, ret_ptr)
ret()
fill_value_obj = c.box(typ.fill_value_type, coo.fill_value)
with cgutils.if_unlikely(c.builder,
cgutils.is_null(c.builder, fill_value_obj)):
c.pyapi.decref(shape_obj)
c.pyapi.decref(coords_obj)
c.pyapi.decref(data_obj)
c.builder.store(fail_obj, ret_ptr)
ret()
class_obj = c.pyapi.unserialize(c.pyapi.serialize_object(COO))
with cgutils.if_unlikely(c.builder,
cgutils.is_null(c.builder, class_obj)):
c.pyapi.decref(shape_obj)
c.pyapi.decref(coords_obj)
c.pyapi.decref(data_obj)
c.pyapi.decref(fill_value_obj)
c.builder.store(fail_obj, ret_ptr)
ret()
args = c.pyapi.tuple_pack([coords_obj, data_obj, shape_obj])
c.pyapi.decref(shape_obj)
c.pyapi.decref(coords_obj)
c.pyapi.decref(data_obj)
with cgutils.if_unlikely(c.builder, cgutils.is_null(c.builder, args)):
c.pyapi.decref(fill_value_obj)
c.pyapi.decref(class_obj)
c.builder.store(fail_obj, ret_ptr)
ret()
kwargs = c.pyapi.dict_pack([("fill_value", fill_value_obj)])
c.pyapi.decref(fill_value_obj)
with cgutils.if_unlikely(c.builder, cgutils.is_null(c.builder,
kwargs)):
c.pyapi.decref(class_obj)
c.builder.store(fail_obj, ret_ptr)
ret()
c.builder.store(c.pyapi.call(class_obj, args, kwargs), ret_ptr)
c.pyapi.decref(class_obj)
c.pyapi.decref(args)
c.pyapi.decref(kwargs)
return c.builder.load(ret_ptr)
def early_exit_if_null(builder, stack, obj):
return early_exit_if(builder, stack, cgutils.is_null(builder, obj))
def is_null(self, obj):
return cgutils.is_null(self.builder, obj)
def _allocate_payload(self, nentries, realloc=False):
"""
Allocate and initialize payload for the given number of entries.
If *realloc* is True, the existing meminfo is reused.
CAUTION: *nentries* must be a power of 2!
"""
context = self._context
builder = self._builder
ok = cgutils.alloca_once_value(builder, cgutils.true_bit)
intp_t = context.get_value_type(types.intp)
zero = ir.Constant(intp_t, 0)
one = ir.Constant(intp_t, 1)
payload_type = context.get_data_type(types.SetPayload(self._ty))
payload_size = context.get_abi_sizeof(payload_type)
entry_size = self._entrysize
# Account for the fact that the payload struct already contains an entry
payload_size -= entry_size
# Total allocation size = <payload header size> + nentries * entry_size
allocsize, ovf = cgutils.muladd_with_overflow(
builder, nentries, ir.Constant(intp_t, entry_size),
ir.Constant(intp_t, payload_size))
with builder.if_then(ovf, likely=False):
builder.store(cgutils.false_bit, ok)
with builder.if_then(builder.load(ok), likely=True):
if realloc:
meminfo = self._set.meminfo
ptr = context.nrt.meminfo_varsize_alloc(builder,
meminfo,
size=allocsize)
alloc_ok = cgutils.is_null(builder, ptr)
else:
meminfo = context.nrt.meminfo_new_varsize(builder,
size=allocsize)
alloc_ok = cgutils.is_null(builder, meminfo)
with builder.if_else(cgutils.is_null(builder, meminfo),
likely=False) as (if_error, if_ok):
with if_error:
builder.store(cgutils.false_bit, ok)
with if_ok:
if not realloc:
self._set.meminfo = meminfo
self._set.parent = context.get_constant_null(
types.pyobject)
payload = self.payload
# Initialize entries to 0xff (EMPTY)
cgutils.memset(builder, payload.ptr, allocsize, 0xFF)
payload.used = zero
payload.fill = zero
payload.finger = zero
new_mask = builder.sub(nentries, one)
payload.mask = new_mask
if DEBUG_ALLOCS:
context.printf(
builder,
"allocated %zd bytes for set at %p: mask = %zd\n",
allocsize, payload.ptr, new_mask)
return builder.load(ok)
def _prepare_call_to_object_mode(context, builder, pyapi, func,
signature, args, env):
mod = builder.module
bb_core_return = builder.append_basic_block('ufunc.core.return')
# Call to
# PyObject* ndarray_new(int nd,
# npy_intp *dims, /* shape */
# npy_intp *strides,
# void* data,
# int type_num,
# int itemsize)
ll_int = context.get_value_type(types.int32)
ll_intp = context.get_value_type(types.intp)
ll_intp_ptr = Type.pointer(ll_intp)
ll_voidptr = context.get_value_type(types.voidptr)
ll_pyobj = context.get_value_type(types.pyobject)
fnty = Type.function(ll_pyobj, [ll_int, ll_intp_ptr,
ll_intp_ptr, ll_voidptr,
ll_int, ll_int])
fn_array_new = mod.get_or_insert_function(fnty, name="numba_ndarray_new")
# Convert each llarray into pyobject
error_pointer = cgutils.alloca_once(builder, Type.int(1), name='error')
builder.store(cgutils.true_bit, error_pointer)
ndarray_pointers = []
ndarray_objects = []
for i, (arr, arrtype) in enumerate(zip(args, signature.args)):
ptr = cgutils.alloca_once(builder, ll_pyobj)
ndarray_pointers.append(ptr)
builder.store(Constant.null(ll_pyobj), ptr) # initialize to NULL
arycls = context.make_array(arrtype)
array = arycls(context, builder, value=arr)
zero = Constant.int(ll_int, 0)
# Extract members of the llarray
nd = Constant.int(ll_int, arrtype.ndim)
dims = builder.gep(array._get_ptr_by_name('shape'), [zero, zero])
strides = builder.gep(array._get_ptr_by_name('strides'), [zero, zero])
data = builder.bitcast(array.data, ll_voidptr)
dtype = np.dtype(str(arrtype.dtype))
# Prepare other info for reconstruction of the PyArray
type_num = Constant.int(ll_int, dtype.num)
itemsize = Constant.int(ll_int, dtype.itemsize)
# Call helper to reconstruct PyArray objects
obj = builder.call(fn_array_new, [nd, dims, strides, data,
type_num, itemsize])
builder.store(obj, ptr)
ndarray_objects.append(obj)
obj_is_null = cgutils.is_null(builder, obj)
builder.store(obj_is_null, error_pointer)
cgutils.cbranch_or_continue(builder, obj_is_null, bb_core_return)
# Call ufunc core function
object_sig = [types.pyobject] * len(ndarray_objects)
status, retval = context.call_conv.call_function(
builder, func, types.pyobject, object_sig,
ndarray_objects, env=env)
builder.store(status.is_error, error_pointer)
# Release returned object
pyapi.decref(retval)
builder.branch(bb_core_return)
# At return block
builder.position_at_end(bb_core_return)
# Release argument object
for ndary_ptr in ndarray_pointers:
pyapi.decref(builder.load(ndary_ptr))
innercall = status.code
return innercall, builder.load(error_pointer)
def to_native_value(self, obj, typ):
builder = self.builder
def c_api_error():
return cgutils.is_not_null(builder, self.err_occurred())
if isinstance(typ, types.Object) or typ == types.pyobject:
return NativeValue(obj)
elif typ == types.boolean:
istrue = self.object_istrue(obj)
zero = Constant.null(istrue.type)
val = builder.icmp(lc.ICMP_NE, istrue, zero)
return NativeValue(val, is_error=c_api_error())
elif isinstance(typ, types.Integer):
val = self.to_native_int(obj, typ)
return NativeValue(val, is_error=c_api_error())
elif typ == types.float32:
fobj = self.number_float(obj)
fval = self.float_as_double(fobj)
self.decref(fobj)
val = builder.fptrunc(fval,
self.context.get_argument_type(typ))
return NativeValue(val, is_error=c_api_error())
elif typ == types.float64:
fobj = self.number_float(obj)
val = self.float_as_double(fobj)
self.decref(fobj)
return NativeValue(val, is_error=c_api_error())
elif typ in (types.complex128, types.complex64):
cplxcls = self.context.make_complex(types.complex128)
cplx = cplxcls(self.context, builder)
pcplx = cplx._getpointer()
ok = self.complex_adaptor(obj, pcplx)
failed = cgutils.is_false(builder, ok)
with cgutils.if_unlikely(builder, failed):
self.err_set_string("PyExc_TypeError",
"conversion to %s failed" % (typ,))
if typ == types.complex64:
c64cls = self.context.make_complex(typ)
c64 = c64cls(self.context, builder)
freal = self.context.cast(builder, cplx.real,
types.float64, types.float32)
fimag = self.context.cast(builder, cplx.imag,
types.float64, types.float32)
c64.real = freal
c64.imag = fimag
return NativeValue(c64._getvalue(), is_error=failed)
else:
return NativeValue(cplx._getvalue(), is_error=failed)
elif isinstance(typ, types.NPDatetime):
val = self.extract_np_datetime(obj)
return NativeValue(val, is_error=c_api_error())
elif isinstance(typ, types.NPTimedelta):
val = self.extract_np_timedelta(obj)
return NativeValue(val, is_error=c_api_error())
elif isinstance(typ, types.Record):
buf = self.alloca_buffer()
ptr = self.extract_record_data(obj, buf)
is_error = cgutils.is_null(self.builder, ptr)
ltyp = self.context.get_value_type(typ)
val = builder.bitcast(ptr, ltyp)
def cleanup():
self.release_buffer(buf)
return NativeValue(val, cleanup=cleanup, is_error=is_error)
elif isinstance(typ, types.Array):
val, failed = self.to_native_array(obj, typ)
return NativeValue(val, is_error=failed)
elif isinstance(typ, types.Buffer):
return self.to_native_buffer(obj, typ)
elif isinstance(typ, types.Optional):
return self.to_native_optional(obj, typ)
elif isinstance(typ, (types.Tuple, types.UniTuple)):
return self.to_native_tuple(obj, typ)
elif isinstance(typ, types.Generator):
return self.to_native_generator(obj, typ)
elif isinstance(typ, types.ExternalFunctionPointer):
if typ.get_pointer is not None:
# Call get_pointer() on the object to get the raw pointer value
ptrty = self.context.get_function_pointer_type(typ)
ret = cgutils.alloca_once_value(builder,
Constant.null(ptrty),
name='fnptr')
ser = self.serialize_object(typ.get_pointer)
get_pointer = self.unserialize(ser)
with cgutils.if_likely(builder,
cgutils.is_not_null(builder, get_pointer)):
intobj = self.call_function_objargs(get_pointer, (obj,))
self.decref(get_pointer)
with cgutils.if_likely(builder,
cgutils.is_not_null(builder, intobj)):
ptr = self.long_as_voidptr(intobj)
self.decref(intobj)
builder.store(builder.bitcast(ptr, ptrty), ret)
return NativeValue(builder.load(ret), is_error=c_api_error())
raise NotImplementedError("cannot convert %s to native value" % (typ,))
def _prepare_call_to_object_mode(context, builder, pyapi, func,
signature, args, env):
mod = builder.module
bb_core_return = builder.append_basic_block('ufunc.core.return')
# Call to
# PyObject* ndarray_new(int nd,
# npy_intp *dims, /* shape */
# npy_intp *strides,
# void* data,
# int type_num,
# int itemsize)
ll_int = context.get_value_type(types.int32)
ll_intp = context.get_value_type(types.intp)
ll_intp_ptr = Type.pointer(ll_intp)
ll_voidptr = context.get_value_type(types.voidptr)
ll_pyobj = context.get_value_type(types.pyobject)
fnty = Type.function(ll_pyobj, [ll_int, ll_intp_ptr,
ll_intp_ptr, ll_voidptr,
ll_int, ll_int])
fn_array_new = mod.get_or_insert_function(fnty, name="numba_ndarray_new")
# Convert each llarray into pyobject
error_pointer = cgutils.alloca_once(builder, Type.int(1), name='error')
builder.store(cgutils.true_bit, error_pointer)
# The PyObject* arguments to the kernel function
object_args = []
object_pointers = []
for i, (arg, argty) in enumerate(zip(args, signature.args)):
# Allocate NULL-initialized slot for this argument
objptr = cgutils.alloca_once(builder, ll_pyobj, zfill=True)
object_pointers.append(objptr)
if isinstance(argty, types.Array):
# Special case arrays: we don't need full-blown NRT reflection
# since the argument will be gone at the end of the kernel
arycls = context.make_array(argty)
array = arycls(context, builder, value=arg)
zero = Constant.int(ll_int, 0)
# Extract members of the llarray
nd = Constant.int(ll_int, argty.ndim)
dims = builder.gep(array._get_ptr_by_name('shape'), [zero, zero])
strides = builder.gep(array._get_ptr_by_name('strides'), [zero, zero])
data = builder.bitcast(array.data, ll_voidptr)
dtype = np.dtype(str(argty.dtype))
# Prepare other info for reconstruction of the PyArray
type_num = Constant.int(ll_int, dtype.num)
itemsize = Constant.int(ll_int, dtype.itemsize)
# Call helper to reconstruct PyArray objects
obj = builder.call(fn_array_new, [nd, dims, strides, data,
type_num, itemsize])
else:
# Other argument types => use generic boxing
obj = pyapi.from_native_value(argty, arg)
builder.store(obj, objptr)
object_args.append(obj)
obj_is_null = cgutils.is_null(builder, obj)
builder.store(obj_is_null, error_pointer)
cgutils.cbranch_or_continue(builder, obj_is_null, bb_core_return)
# Call ufunc core function
object_sig = [types.pyobject] * len(object_args)
status, retval = context.call_conv.call_function(
builder, func, types.pyobject, object_sig,
object_args, env=env)
builder.store(status.is_error, error_pointer)
# Release returned object
pyapi.decref(retval)
builder.branch(bb_core_return)
# At return block
builder.position_at_end(bb_core_return)
# Release argument objects
for objptr in object_pointers:
pyapi.decref(builder.load(objptr))
innercall = status.code
return innercall, builder.load(error_pointer)
def _allocate_payload(self, nentries, realloc=False):
"""
Allocate and initialize payload for the given number of entries.
If *realloc* is True, the existing meminfo is reused.
CAUTION: *nentries* must be a power of 2!
"""
context = self._context
builder = self._builder
ok = cgutils.alloca_once_value(builder, cgutils.true_bit)
intp_t = context.get_value_type(types.intp)
zero = ir.Constant(intp_t, 0)
one = ir.Constant(intp_t, 1)
payload_type = context.get_data_type(types.SetPayload(self._ty))
payload_size = context.get_abi_sizeof(payload_type)
entry_size = self._entrysize
# Account for the fact that the payload struct already contains an entry
payload_size -= entry_size
# Total allocation size = <payload header size> + nentries * entry_size
allocsize, ovf = cgutils.muladd_with_overflow(builder, nentries,
ir.Constant(intp_t, entry_size),
ir.Constant(intp_t, payload_size))
with builder.if_then(ovf, likely=False):
builder.store(cgutils.false_bit, ok)
with builder.if_then(builder.load(ok), likely=True):
if realloc:
meminfo = self._set.meminfo
ptr = context.nrt.meminfo_varsize_alloc(builder, meminfo,
size=allocsize)
alloc_ok = cgutils.is_null(builder, ptr)
else:
meminfo = context.nrt.meminfo_new_varsize(builder, size=allocsize)
alloc_ok = cgutils.is_null(builder, meminfo)
with builder.if_else(cgutils.is_null(builder, meminfo),
likely=False) as (if_error, if_ok):
with if_error:
builder.store(cgutils.false_bit, ok)
with if_ok:
if not realloc:
self._set.meminfo = meminfo
self._set.parent = context.get_constant_null(types.pyobject)
payload = self.payload
# Initialize entries to 0xff (EMPTY)
cgutils.memset(builder, payload.ptr, allocsize, 0xFF)
payload.used = zero
payload.fill = zero
payload.finger = zero
new_mask = builder.sub(nentries, one)
payload.mask = new_mask
if DEBUG_ALLOCS:
context.printf(builder,
"allocated %zd bytes for set at %p: mask = %zd\n",
allocsize, payload.ptr, new_mask)
return builder.load(ok)
def _prepare_call_to_object_mode(context, builder, pyapi, func, signature,
args, env):
mod = builder.module
bb_core_return = builder.append_basic_block('ufunc.core.return')
# Call to
# PyObject* ndarray_new(int nd,
# npy_intp *dims, /* shape */
# npy_intp *strides,
# void* data,
# int type_num,
# int itemsize)
ll_int = context.get_value_type(types.int32)
ll_intp = context.get_value_type(types.intp)
ll_intp_ptr = Type.pointer(ll_intp)
ll_voidptr = context.get_value_type(types.voidptr)
ll_pyobj = context.get_value_type(types.pyobject)
fnty = Type.function(
ll_pyobj,
[ll_int, ll_intp_ptr, ll_intp_ptr, ll_voidptr, ll_int, ll_int])
fn_array_new = mod.get_or_insert_function(fnty, name="numba_ndarray_new")
# Convert each llarray into pyobject
error_pointer = cgutils.alloca_once(builder, Type.int(1), name='error')
builder.store(cgutils.true_bit, error_pointer)
# The PyObject* arguments to the kernel function
object_args = []
object_pointers = []
for i, (arg, argty) in enumerate(zip(args, signature.args)):
# Allocate NULL-initialized slot for this argument
objptr = cgutils.alloca_once(builder, ll_pyobj, zfill=True)
object_pointers.append(objptr)
if isinstance(argty, types.Array):
# Special case arrays: we don't need full-blown NRT reflection
# since the argument will be gone at the end of the kernel
arycls = context.make_array(argty)
array = arycls(context, builder, value=arg)
zero = Constant.int(ll_int, 0)
# Extract members of the llarray
nd = Constant.int(ll_int, argty.ndim)
dims = builder.gep(array._get_ptr_by_name('shape'), [zero, zero])
strides = builder.gep(array._get_ptr_by_name('strides'),
[zero, zero])
data = builder.bitcast(array.data, ll_voidptr)
dtype = np.dtype(str(argty.dtype))
# Prepare other info for reconstruction of the PyArray
type_num = Constant.int(ll_int, dtype.num)
itemsize = Constant.int(ll_int, dtype.itemsize)
# Call helper to reconstruct PyArray objects
obj = builder.call(fn_array_new,
[nd, dims, strides, data, type_num, itemsize])
else:
# Other argument types => use generic boxing
obj = pyapi.from_native_value(argty, arg)
builder.store(obj, objptr)
object_args.append(obj)
obj_is_null = cgutils.is_null(builder, obj)
builder.store(obj_is_null, error_pointer)
cgutils.cbranch_or_continue(builder, obj_is_null, bb_core_return)
# Call ufunc core function
object_sig = [types.pyobject] * len(object_args)
status, retval = context.call_conv.call_function(builder,
func,
types.pyobject,
object_sig,
object_args,
env=env)
builder.store(status.is_error, error_pointer)
# Release returned object
pyapi.decref(retval)
builder.branch(bb_core_return)
# At return block
builder.position_at_end(bb_core_return)
# Release argument objects
for objptr in object_pointers:
pyapi.decref(builder.load(objptr))
innercall = status.code
return innercall, builder.load(error_pointer)
def is_null(self, obj):
return cgutils.is_null(self.builder, obj)
