def unbox_range_index(typ, val, c):
start_obj = c.pyapi.object_getattr_string(val, "start")
stop_obj = c.pyapi.object_getattr_string(val, "stop")
step_obj = c.pyapi.object_getattr_string(val, "step")
# TODO: support range unboxing with reference to parent in Numba?
range_index = cgutils.create_struct_proxy(typ)(c.context, c.builder)
range_index_data = cgutils.create_struct_proxy(RangeIndexDataType)(
c.context, c.builder)
range_index_data.start = c.pyapi.long_as_longlong(start_obj)
range_index_data.stop = c.pyapi.long_as_longlong(stop_obj)
range_index_data.step = c.pyapi.long_as_longlong(step_obj)
range_index.data = range_index_data._getvalue()
if typ.is_named:
name_obj = c.pyapi.object_getattr_string(val, "name")
range_index.name = numba.cpython.unicode.unbox_unicode_str(
types.unicode_type, name_obj, c).value
c.pyapi.decref(name_obj)
c.pyapi.decref(start_obj)
c.pyapi.decref(stop_obj)
c.pyapi.decref(step_obj)
is_error = cgutils.is_not_null(c.builder, c.pyapi.err_occurred())
return NativeValue(range_index._getvalue(), is_error=is_error)
def lower_constant_function_type(context, builder, typ, pyval):
typ = typ.get_precise()
if isinstance(pyval, CFunc):
addr = pyval._wrapper_address
sfunc = cgutils.create_struct_proxy(typ)(context, builder)
sfunc.addr = context.add_dynamic_addr(builder, addr, info=str(typ))
sfunc.pyaddr = context.add_dynamic_addr(builder,
id(pyval),
info=type(pyval).__name__)
return sfunc._getvalue()
if isinstance(pyval, Dispatcher):
sfunc = cgutils.create_struct_proxy(typ)(context, builder)
sfunc.pyaddr = context.add_dynamic_addr(builder,
id(pyval),
info=type(pyval).__name__)
return sfunc._getvalue()
if isinstance(pyval, WrapperAddressProtocol):
addr = pyval.__wrapper_address__()
assert typ.check_signature(pyval.signature())
sfunc = cgutils.create_struct_proxy(typ)(context, builder)
sfunc.addr = context.add_dynamic_addr(builder, addr, info=str(typ))
sfunc.pyaddr = context.add_dynamic_addr(builder,
id(pyval),
info=type(pyval).__name__)
return sfunc._getvalue()
# TODO: implement support for pytypes.FunctionType, ctypes.CFUNCTYPE
raise NotImplementedError(
'lower_constant_struct_function_type({}, {}, {}, {})'.format(
context, builder, typ, pyval))
def lowering(context, builder, sig, actual_args):
# A list of elements to assign from
source_list = []
# Convert the list of argument types to a list of load IRs.
for argidx, fml_arg in enumerate(fml_arg_list):
if isinstance(fml_arg, VectorType):
pxy = cgutils.create_struct_proxy(fml_arg)(
context, builder, actual_args[argidx])
source_list += [
getattr(pxy, attr) for attr in fml_arg.attr_names
]
else:
# assumed primitive type
source_list.append(actual_args[argidx])
if len(source_list) != vector_type.num_elements:
raise CudaLoweringError(
f"Unmatched number of source elements ({len(source_list)}) "
"and target elements ({vector_type.num_elements}).")
out = cgutils.create_struct_proxy(vector_type)(context, builder)
for attr_name, source in zip(vector_type.attr_names, source_list):
setattr(out, attr_name, source)
return out._getvalue()
def masked_scalar_unary_op_impl(context, builder, sig, args):
"""
Implement <op> `MaskedType`
"""
# MaskedType(...)
masked_type_1 = sig.args[0]
# MaskedType(...)
masked_return_type = sig.return_type
m1 = cgutils.create_struct_proxy(masked_type_1)(context,
builder,
value=args[0])
# we will return an output struct
result = cgutils.create_struct_proxy(masked_return_type)(context,
builder)
# compute output validity
result.valid = m1.valid
with builder.if_then(m1.valid):
# Let numba handle generating the extra IR needed to perform
# operations on mixed types, by compiling the final core op between
# the two primitive values as a separate function and calling it
result.value = context.compile_internal(
builder,
lambda x: op(x),
nb_signature(
masked_return_type.value_type,
masked_type_1.value_type,
),
(m1.value, ),
)
return result._getvalue()
def masked_scalar_const_op_impl(context, builder, sig, args):
return_type = sig.return_type
result = cgutils.create_struct_proxy(return_type)(context, builder)
result.valid = context.get_constant(types.boolean, 0)
if isinstance(sig.args[0], MaskedType):
masked_type, const_type = sig.args
masked_value, const_value = args
indata = cgutils.create_struct_proxy(masked_type)(
context, builder, value=masked_value)
nb_sig = nb_signature(return_type.value_type,
masked_type.value_type, const_type)
compile_args = (indata.value, const_value)
else:
const_type, masked_type = sig.args
const_value, masked_value = args
indata = cgutils.create_struct_proxy(masked_type)(
context, builder, value=masked_value)
nb_sig = nb_signature(return_type.value_type, const_type,
masked_type.value_type)
compile_args = (const_value, indata.value)
with builder.if_then(indata.valid):
result.value = context.compile_internal(builder,
lambda x, y: op(x, y),
nb_sig, compile_args)
result.valid = context.get_constant(types.boolean, 1)
return result._getvalue()
def box_range_index(typ, val, c):
mod_name = c.context.insert_const_string(c.builder.module, "pandas")
pd_class_obj = c.pyapi.import_module_noblock(mod_name)
range_index = cgutils.create_struct_proxy(typ)(c.context, c.builder, val)
range_index_data = cgutils.create_struct_proxy(RangeIndexDataType)(
c.context, c.builder, range_index.data)
start = c.pyapi.from_native_value(types.int64, range_index_data.start)
stop = c.pyapi.from_native_value(types.int64, range_index_data.stop)
step = c.pyapi.from_native_value(types.int64, range_index_data.step)
# dtype and copy params are not stored so use default values
dtype = c.pyapi.make_none()
copy = c.pyapi.bool_from_bool(c.context.get_constant(types.bool_, False))
if typ.is_named:
name = c.pyapi.from_native_value(types.unicode_type, range_index.name)
else:
name = c.pyapi.make_none()
res = c.pyapi.call_method(pd_class_obj, "RangeIndex",
(start, stop, step, dtype, copy, name))
c.pyapi.decref(start)
c.pyapi.decref(stop)
c.pyapi.decref(step)
c.pyapi.decref(dtype)
c.pyapi.decref(copy)
c.pyapi.decref(name)
c.pyapi.decref(pd_class_obj)
return res
def impl_ctor_ts_ts(context, builder, sig, args):
typ = sig.return_type
rhs = args[0]
ts = cgutils.create_struct_proxy(typ)(context, builder)
rhsproxy = cgutils.create_struct_proxy(typ)(context, builder)
rhsproxy._setvalue(rhs)
cgutils.copy_struct(ts, rhsproxy)
return ts._getvalue()
def pd_positional_index_getiter(context, builder, sig, args):
""" Returns a new iterator object for PositionalIndexType by delegating to range.__iter__ """
(value, ) = args
positional_index = cgutils.create_struct_proxy(sig.args[0])(context,
builder, value)
range_index = cgutils.create_struct_proxy(sig.args[0].data)(
context, builder, positional_index.data)
res = call_getiter(context, builder, RangeIndexDataType, range_index.data)
return impl_ret_untracked(context, builder, PositionalIndexType, res)
def cuda_quaternion_add(context, builder, sig, args):
typ = sig.return_type
q1 = cgutils.create_struct_proxy(typ)(context, builder, value=args[0])
q2 = cgutils.create_struct_proxy(typ)(context, builder, value=args[1])
q3 = cgutils.create_struct_proxy(typ)(context, builder)
q3.a = builder.fadd(q1.a, q2.a)
q3.b = builder.fadd(q1.b, q2.b)
q3.c = builder.fadd(q1.c, q2.c)
q3.d = builder.fadd(q1.d, q2.d)
return q3._getvalue()
def codegen(context, builder, sig, args):
_,d = args
ctor = cgutils.create_struct_proxy(inst_type)
dstruct = ctor(context, builder, value=d)
meminfo = dstruct.meminfo
context.nrt.incref(builder, types.MemInfoPointer(types.voidptr), meminfo)
st = cgutils.create_struct_proxy(cast_type)(context, builder)
st.meminfo = meminfo
return st._getvalue()
def unbox_listtype(typ, val, c):
context = c.context
builder = c.builder
miptr = c.pyapi.object_getattr_string(val, '_opaque')
native = c.unbox(types.MemInfoPointer(types.voidptr), miptr)
mi = native.value
ctor = cgutils.create_struct_proxy(typ)
lstruct = ctor(context, builder)
data_pointer = context.nrt.meminfo_data(builder, mi)
data_pointer = builder.bitcast(
data_pointer,
listobject.ll_list_type.as_pointer(),
)
lstruct.data = builder.load(data_pointer)
lstruct.meminfo = mi
lstobj = lstruct._getvalue()
c.pyapi.decref(miptr)
return NativeValue(lstobj)
def box_lsttype(typ, val, c):
context = c.context
builder = c.builder
# XXX deduplicate
ctor = cgutils.create_struct_proxy(typ)
lstruct = ctor(context, builder, value=val)
# Returns the plain MemInfo
boxed_meminfo = c.box(
types.MemInfoPointer(types.voidptr),
lstruct.meminfo,
)
modname = c.context.insert_const_string(
c.builder.module, 'numba.typed.typedlist',
)
typedlist_mod = c.pyapi.import_module_noblock(modname)
fmp_fn = c.pyapi.object_getattr_string(typedlist_mod, '_from_meminfo_ptr')
lsttype_obj = c.pyapi.unserialize(c.pyapi.serialize_object(typ))
result_var = builder.alloca(c.pyapi.pyobj)
builder.store(cgutils.get_null_value(c.pyapi.pyobj), result_var)
with builder.if_then(cgutils.is_not_null(builder, lsttype_obj)):
res = c.pyapi.call_function_objargs(
fmp_fn, (boxed_meminfo, lsttype_obj),
)
c.pyapi.decref(fmp_fn)
c.pyapi.decref(typedlist_mod)
c.pyapi.decref(boxed_meminfo)
builder.store(res, result_var)
return builder.load(result_var)
def impl_index_value(context, builder, sig, args):
typ = sig.return_type
index, value = args
index_value = cgutils.create_struct_proxy(typ)(context, builder)
index_value.index = index
index_value.value = value
return index_value._getvalue()
def codegen(context, builder, sig, args):
table_pyobject = args[1]
nrt_table = context.nrt.get_nrt_api(builder)
arrow_table = cgutils.create_struct_proxy(sig.return_type)(context,
builder)
fnty = lir.FunctionType(
lir.VoidType(),
[
lir.IntType(8).as_pointer(), # ptr to pyarrow table
arrow_table.meminfo.type.as_pointer(), # meminfo to fill
lir.IntType(8).as_pointer(),
]) # NRT API func table
func_name = f"create_arrow_table"
fn = cgutils.get_or_insert_function(builder.module,
fnty,
name=func_name)
builder.call(fn, [
table_pyobject,
arrow_table._get_ptr_by_name('meminfo'), nrt_table
])
arrow_table.table_ptr = context.nrt.meminfo_data(
builder, arrow_table.meminfo)
return arrow_table._getvalue()
def box_series(typ, val, c):
"""
"""
mod_name = c.context.insert_const_string(c.builder.module, "pandas")
pd_class_obj = c.pyapi.import_module_noblock(mod_name)
dtype = typ.dtype
series = cgutils.create_struct_proxy(
typ)(c.context, c.builder, val)
arr = _box_series_data(dtype, typ.data, series.data, c)
if typ.index is types.none:
index = c.pyapi.make_none()
else:
index = _box_index_data(typ.index, series.index, c)
if typ.is_named:
name = c.pyapi.from_native_value(string_type, series.name)
else:
name = c.pyapi.make_none()
dtype = c.pyapi.make_none() # TODO: dtype
res = c.pyapi.call_method(
pd_class_obj, "Series", (arr, index, dtype, name))
c.pyapi.decref(arr)
c.pyapi.decref(index)
c.pyapi.decref(dtype)
c.pyapi.decref(name)
c.pyapi.decref(pd_class_obj)
return res
def codegen(context, builder, sig, args):
d, = args
ctor = cgutils.create_struct_proxy(inst_type)
dstruct = ctor(context, builder, value=d)
context.nrt.incref(builder, types.MemInfoPointer(types.voidptr),
dstruct.meminfo)
def unbox_(typ, obj, c):
ptr_obj = c.pyapi.object_getattr_string(obj, "address")
ctx = cgutils.create_struct_proxy(typ)(c.context, c.builder)
ctx.ptr = c.pyapi.long_as_voidptr(ptr_obj)
c.pyapi.decref(ptr_obj)
is_error = cgutils.is_not_null(c.builder, c.pyapi.err_occurred())
return NativeValue(ctx._getvalue(), is_error=is_error)
def _unbox_class_instance(typ, val, c):
def access_member(member_offset):
# Access member by byte offset
offset = c.context.get_constant(types.uintp, member_offset)
llvoidptr = ir.IntType(8).as_pointer()
ptr = cgutils.pointer_add(c.builder, val, offset)
casted = c.builder.bitcast(ptr, llvoidptr.as_pointer())
return c.builder.load(casted)
struct_cls = cgutils.create_struct_proxy(typ)
inst = struct_cls(c.context, c.builder)
# load from Python object
ptr_meminfo = access_member(_box.box_meminfoptr_offset)
ptr_dataptr = access_member(_box.box_dataptr_offset)
# store to native structure
inst.meminfo = c.builder.bitcast(ptr_meminfo, inst.meminfo.type)
inst.data = c.builder.bitcast(ptr_dataptr, inst.data.type)
ret = inst._getvalue()
c.context.nrt.incref(c.builder, typ, ret)
return NativeValue(ret, is_error=c.pyapi.c_api_error())
def box_dicttype(typ, val, c):
context = c.context
builder = c.builder
# XXX deduplicate
ctor = cgutils.create_struct_proxy(typ)
dstruct = ctor(context, builder, value=val)
# Returns the plain MemInfo
boxed_meminfo = c.box(
types.MemInfoPointer(types.voidptr),
dstruct.meminfo,
)
modname = c.context.insert_const_string(
c.builder.module,
'numba.typed.typeddict',
)
typeddict_mod = c.pyapi.import_module_noblock(modname)
fmp_fn = c.pyapi.object_getattr_string(typeddict_mod, '_from_meminfo_ptr')
dicttype_obj = c.pyapi.unserialize(c.pyapi.serialize_object(typ))
res = c.pyapi.call_function_objargs(fmp_fn, (boxed_meminfo, dicttype_obj))
c.pyapi.decref(fmp_fn)
c.pyapi.decref(typeddict_mod)
c.pyapi.decref(boxed_meminfo)
return res
def impl_MultiVector(context, builder, sig, args):
typ = sig.return_type
layout, value = args
mv = cgutils.create_struct_proxy(typ)(context, builder)
mv.layout = layout
mv.value = value
return impl_ret_borrowed(context, builder, sig.return_type, mv._getvalue())
def hpat_pandas_dataframe_box(typ, val, c):
"""
This method is to copy data from JITted region data structure
to new Python object data structure.
Python object data structure has creating in this procedure.
"""
dataframe = cgutils.create_struct_proxy(typ)(c.context, c.builder, value=val)
ir_ptr_data = c.box(typ.data, dataframe.data)
dataframe_ctor_args = c.pyapi.tuple_pack([ir_ptr_data, ])
# dataframe_ctor_kwargs = c.pyapi.dict_pack([("data", ir_ptr_data), ])
"""
It is better to use kwargs but it fails into SIGSEGV
"""
dataframe_ctor_fn = c.pyapi.unserialize(c.pyapi.serialize_object(pandas.DataFrame))
"""
Create a pandas.DataFrame ctor() function pointer
"""
df_obj = c.pyapi.call(dataframe_ctor_fn, dataframe_ctor_args) # kws=dataframe_ctor_kwargs)
"""
Call pandas.DataFrame function pointer with parameters
"""
c.pyapi.decref(ir_ptr_data)
c.pyapi.decref(dataframe_ctor_args)
c.pyapi.decref(dataframe_ctor_fn)
return df_obj
def impl_doubledouble(context, builder, sig, args):
typ = sig.return_type
x, y = args
doubledouble = cgutils.create_struct_proxy(typ)(context, builder)
doubledouble.x = x
doubledouble.y = y
return doubledouble._getvalue()
def codegen(context, builder, sig, args):
pyapi = context.get_python_api(builder)
c = numba.pythonapi._UnboxContext(context, builder, pyapi)
df_typ = sig.args[0]
col_ind = sig.args[1].literal_value
data_typ = df_typ.data[col_ind]
col_name = df_typ.columns[col_ind]
# TODO: refcounts?
dataframe = cgutils.create_struct_proxy(
sig.args[0])(context, builder, value=args[0])
series_obj = c.pyapi.object_getattr_string(dataframe.parent, col_name)
arr_obj = c.pyapi.object_getattr_string(series_obj, "values")
# TODO: support column of tuples?
native_val = _unbox_series_data(
data_typ.dtype, data_typ, arr_obj, c)
c.pyapi.decref(series_obj)
c.pyapi.decref(arr_obj)
c.context.nrt.incref(builder, df_typ.index, dataframe.index)
# assign array and set unboxed flag
dataframe.data = builder.insert_value(
dataframe.data, native_val.value, col_ind)
return dataframe._getvalue()
def pd_int64_index_getiter(context, builder, sig, args):
""" Returns a new iterator object for Int64IndexType by delegating to array __iter__ """
(value, ) = args
int64_index = cgutils.create_struct_proxy(sig.args[0])(context, builder,
value)
res = call_getiter(context, builder, sig.args[0].data, int64_index.data)
return impl_ret_untracked(context, builder, Int64IndexType, res)
def codegen(context, builder, signature, args):
parent_val, column_id_val, data_val, sort_val, target_columns = args
# create series struct and store values
groupby_obj = cgutils.create_struct_proxy(signature.return_type)(
context, builder)
groupby_obj.parent = parent_val
groupby_obj.col_id = column_id_val
groupby_obj.data = data_val
groupby_obj.sort = sort_val
groupby_obj.target_default = context.get_constant(
types.bool_, target_not_specified)
column_strs = [
numba.cpython.unicode.make_string_from_constant(
context, builder, string_type, c) for c in selected_col_names
]
column_tup = context.make_tuple(
builder, types.UniTuple(string_type, n_target_cols), column_strs)
groupby_obj.target_columns = column_tup
# increase refcount of stored values
if context.enable_nrt:
context.nrt.incref(builder, signature.args[0], parent_val)
context.nrt.incref(builder, signature.args[2], data_val)
for var in column_strs:
context.nrt.incref(builder, string_type, var)
return groupby_obj._getvalue()
def box_int64_index(typ, val, c):
mod_name = c.context.insert_const_string(c.builder.module, "pandas")
pd_class_obj = c.pyapi.import_module_noblock(mod_name)
int64_index = cgutils.create_struct_proxy(typ)(c.context, c.builder, val)
data = box_array(typ.data, int64_index.data, c)
# dtype and copy params are not stored so use default values
dtype = c.pyapi.make_none()
copy = c.pyapi.bool_from_bool(c.context.get_constant(types.bool_, False))
if typ.is_named:
name = c.pyapi.from_native_value(types.unicode_type, int64_index.name)
else:
name = c.pyapi.make_none()
res = c.pyapi.call_method(pd_class_obj, "Int64Index",
(data, dtype, copy, name))
c.pyapi.decref(data)
c.pyapi.decref(dtype)
c.pyapi.decref(copy)
c.pyapi.decref(name)
c.pyapi.decref(pd_class_obj)
return res
def impl_interval(context, builder, sig, args):
typ = sig.return_type
lo, hi = args
interval = cgutils.create_struct_proxy(typ)(context, builder)
interval.lo = lo
interval.hi = hi
return interval._getvalue()
def codegen(context, builder, signature, args):
ptr = args[1]
ctor = cgutils.create_struct_proxy(list_ty)
lstruct = ctor(context, builder)
lstruct.data = ptr
_add_meminfo(context, builder, lstruct)
return lstruct._getvalue()
def get_helper_class(self, typ, kind='value'):
"""
Get a helper class for the given *typ*.
"""
# XXX handle all types: complex, array, etc.
# XXX should it be a method on the model instead? this would allow a default kind...
return cgutils.create_struct_proxy(typ, kind)
def pq_read_string_parallel_lower(context, builder, sig, args):
typ = sig.return_type
dtype = StringArrayPayloadType()
meminfo, meminfo_data_ptr = construct_string_array(context, builder)
str_arr_payload = cgutils.create_struct_proxy(dtype)(context, builder)
string_array = context.make_helper(builder, typ)
string_array.num_items = args[3]
fnty = lir.FunctionType(lir.IntType(32),
[lir.IntType(8).as_pointer(), lir.IntType(64),
lir.IntType(32).as_pointer().as_pointer(),
lir.IntType(8).as_pointer().as_pointer(),
lir.IntType(8).as_pointer().as_pointer(),
lir.IntType(64), lir.IntType(64)])
fn = builder.module.get_or_insert_function(
fnty, name="pq_read_string_parallel")
res = builder.call(fn, [args[0], args[1],
str_arr_payload._get_ptr_by_name('offsets'),
str_arr_payload._get_ptr_by_name('data'),
str_arr_payload._get_ptr_by_name('null_bitmap'),
args[2],
args[3]])
builder.store(str_arr_payload._getvalue(), meminfo_data_ptr)
string_array.meminfo = meminfo
string_array.offsets = str_arr_payload.offsets
string_array.data = str_arr_payload.data
string_array.null_bitmap = str_arr_payload.null_bitmap
string_array.num_total_chars = builder.zext(builder.load(
builder.gep(string_array.offsets, [string_array.num_items])), lir.IntType(64))
ret = string_array._getvalue()
return impl_ret_new_ref(context, builder, typ, ret)
