def base_dummy_type_factory(name): class DynType(object): pass class DynTypeType(types.Opaque): pass dyn_type_type = DynTypeType(name) @typeof_impl.register(DynType) def typeof_mydummy(val, c): return dyn_type_type register_model(DynTypeType)(models.OpaqueModel) return DynTypeType, DynType, dyn_type_type
def test_parametrized_types(self): """https://github.com/numba/numba/issues/6401""" register_model(ParametrizedType)(UniTupleModel) @typeof_impl.register(Parametrized) def typeof_unit(val, c): return ParametrizedType(val) @unbox(ParametrizedType) def unbox_parametrized(typ, obj, context): return context.unbox(types.UniTuple(typ.dtype, len(typ)), obj) @generated_jit def dict_vs_cache_vs_parametrized(v): typ = v def objmode_vs_cache_vs_parametrized_impl(v): # typed.List shows same behaviour after fix for #6397 d = typed.Dict.empty(types.unicode_type, typ) d['data'] = v return objmode_vs_cache_vs_parametrized_impl @jit(nopython=True, cache=True) def set_parametrized_data(x, y): # Has had a tendency to segfault when the compiled function # was loaded from cache in a different process than the one # it was originally compiled in. # The new process is simulated below by resetting the dispatchers # and the target context dict_vs_cache_vs_parametrized(x) dict_vs_cache_vs_parametrized(y) x, y = Parametrized(('a', 'b')), Parametrized(('a',)) set_parametrized_data(x, y) # reset dispatchers and targetctx to force re-load from cache as if a # new process would jit the function set_parametrized_data._make_finalizer()() set_parametrized_data._reset_overloads() set_parametrized_data.targetctx.init() for ii in range(50): # <- sometimes works a few times self.assertIsNone(set_parametrized_data(x, y))
def setUp(self): # Use test_id to makesure no collision is possible. test_id = self.id() DummyType = type('DummyTypeFor{}'.format(test_id), (types.Opaque, ), {}) dummy_type = DummyType("my_dummy") register_model(DummyType)(OpaqueModel) class Dummy(object): pass @typeof_impl.register(Dummy) def typeof_Dummy(val, c): return dummy_type @unbox(DummyType) def unbox_index(typ, obj, c): return NativeValue(c.context.get_dummy_value()) self.Dummy = Dummy self.DummyType = DummyType
def test_mixin_against_real_example(self): # See issue #4970, this checks that unicode eq/ne now ignores extension # types. with self.create_temp_module(self.source_lines) as test_module: FooType = test_module.FooType self.assertFalse(FooType().is_internal) # set up an extension type class Foo(object): pass register_model(FooType)(models.OpaqueModel) @typeof_impl.register(Foo) def _typ_foo(val, c): return FooType() @unbox(FooType) def unbox_foo(typ, obj, c): return NativeValue(c.context.get_dummy_value()) @overload(operator.eq) def foo_eq(a, b): if a == FooType(): return lambda a, b: "RAN CUSTOM EQ OVERLOAD" @overload(operator.ne) def foo_ne(a, b): if a == FooType(): return lambda a, b: "RAN CUSTOM NE OVERLOAD" @njit def f(a): return a == "A", a != "A" self.assertEqual(("RAN CUSTOM EQ OVERLOAD", "RAN CUSTOM NE OVERLOAD"), f(Foo()))
def mk_type(self, spec): '''Make numba type and register opaque model''' assert 'pyclass' in spec, "Missing required attribute 'pyclass' in daal4py spec: " + str(spec) self.name = spec['pyclass'].__name__ def mk_simple(name): class NbType(types.Opaque): '''Our numba type for given algo class''' def __init__(self): super(NbType, self).__init__(name=name) return NbType # make type and type instance for algo, its result or possibly model # also register their opaque data model self.NbType = mk_simple(self.name + '_nbtype') self.all_nbtypes[self.name] = self.NbType() register_model(self.NbType)(models.OpaqueModel) # some of the classes can be parameters to others and have a default NULL/None # We need to cast Python None to C NULL @lower_cast(types.none, self.NbType()) def none_to_nbtype(context, builder, fromty, toty, val): zero = context.get_constant(types.intp, 0) return builder.inttoptr(zero, context.get_value_type(toty))
class DatetimeDateType(types.Type): def __init__(self): super(DatetimeDateType, self).__init__( name='DatetimeDateType()') self.bitwidth = 64 datetime_date_type = DatetimeDateType() @typeof_impl.register(datetime.date) def typeof_pd_timestamp(val, c): return datetime_date_type register_model(DatetimeDateType)(models.IntegerModel) @infer_getattr class DatetimeAttribute(AttributeTemplate): key = DatetimeDateType def generic_resolve(self, typ, attr): return types.int64 @lower_getattr(DatetimeDateType, 'year') def datetime_get_year(context, builder, typ, val): return builder.lshr(val, lir.Constant(lir.IntType(64), 32))
# @infer_global(dist_setitem) # class DistSetitem(AbstractTemplate): # def generic(self, args, kws): # assert not kws # assert len(args)==5 # return signature(types.int32, *unliteral_all(args)) class ReqArrayType(types.Type): def __init__(self): super(ReqArrayType, self).__init__(name='ReqArrayType()') req_array_type = ReqArrayType() register_model(ReqArrayType)(models.OpaqueModel) def comm_req_alloc(): return 0 def comm_req_dealloc(): return 0 @infer_global(comm_req_alloc) class DistCommReqAlloc(AbstractTemplate): def generic(self, args, kws): assert not kws assert len(args) == 1 and args[0] == types.int32
from llvmlite import ir as lir import llvmlite.binding as ll from . import hdict_ext from sdc.utils import unliteral_all ll_voidp = lir.IntType(8).as_pointer() class ByteVecType(types.Opaque): def __init__(self): super(ByteVecType, self).__init__( name='byte_vec') byte_vec_type = ByteVecType() register_model(ByteVecType)(models.OpaqueModel) class DictType(types.Opaque): def __init__(self, key_typ, val_typ): self.key_typ = key_typ self.val_typ = val_typ super(DictType, self).__init__( name='DictType{}{}'.format(key_typ, val_typ)) @property def key(self): return self.key_typ, self.val_typ @property def iterator_type(self):
def get_element_type(dtype): out = repr(dtype) if out == 'bool': out = 'bool_' return out class XeConnectType(types.Opaque): def __init__(self): super(XeConnectType, self).__init__(name='XeConnectType') xe_connect_type = XeConnectType() register_model(XeConnectType)(models.OpaqueModel) class XeDSetType(types.Opaque): def __init__(self): super(XeDSetType, self).__init__(name='XeDSetType') xe_dset_type = XeDSetType() register_model(XeDSetType)(models.OpaqueModel) get_column_size_xenon = types.ExternalFunction( "get_column_size_xenon", types.int64(xe_connect_type, xe_dset_type, types.intp)) # read_xenon_col = types.ExternalFunction( # "c_read_xenon",
class PandasDataFrameType(types.Type): def __init__(self, col_names, col_types): self.col_names = col_names self.col_types = col_types super(PandasDataFrameType, self).__init__( name='PandasDataFrameType({}, {})'.format(col_names, col_types)) @typeof_impl.register(pd.DataFrame) def typeof_pd_dataframe(val, c): col_names = val.columns.tolist() # TODO: support other types like string and timestamp col_types = get_hiframes_dtypes(val) return PandasDataFrameType(col_names, col_types) register_model(PandasDataFrameType)(models.OpaqueModel) @unbox(PandasDataFrameType) def unbox_df(typ, val, c): """unbox dataframe to an Opaque pointer columns will be extracted later if necessary. """ # XXX: refcount? return NativeValue(val) def get_hiframes_dtypes(df): """get hiframe data types for a pandas dataframe """ pd_typ_list = df.dtypes.tolist() col_names = df.columns.tolist() hi_typs = []
# Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** from numba.extending import models from numba.extending import register_model from .types import ( CategoricalDtypeType, Categorical, ) register_model(CategoricalDtypeType)(models.OpaqueModel) register_model(Categorical)(models.ArrayModel)
name = "USM:ndarray(%s, %sd, %s)" % (dtype, ndim, layout) super(USMNdArrayType, self).__init__( dtype, ndim, layout, readonly=readonly, name=name, addrspace=addrspace, ) def copy(self, *args, **kwargs): return super(USMNdArrayType, self).copy(*args, **kwargs) # This tells Numba to use the DPPYArray data layout for object of type USMNdArrayType. register_model(USMNdArrayType)(DPPYArrayModel) dppy_target.spirv_data_model_manager.register(USMNdArrayType, DPPYArrayModel) @typeof_impl.register(usm_ndarray) def typeof_usm_ndarray(val, c): """ This function creates the Numba type (USMNdArrayType) when a usm_ndarray is passed. """ try: dtype = numpy_support.from_dtype(val.dtype) except NotImplementedError: raise ValueError("Unsupported array dtype: %s" % (val.dtype,)) layout = "C" readonly = False return USMNdArrayType(
replace_arg_nodes(f_block, [fname]) nodes = f_block.body[:-3] # remove none return A_var = nodes[-2].value.args[0] #A_var = nodes[-1].target nodes.append(ir.Assign(A_var, lhs, lhs.loc)) return nodes class BagFileType(types.Opaque): def __init__(self): super(BagFileType, self).__init__(name='BagFileType') bag_file_type = BagFileType() register_model(BagFileType)(models.OpaqueModel) @infer_global(read_ros_images) class ReadMsgImageTyper(AbstractTemplate): def generic(self, args, kws): assert not kws assert len(args) == 1 return signature(types.Array(types.uint8, 4, 'C'), *args) @infer_global(open_bag) class BagOpenTyper(AbstractTemplate): def generic(self, args, kws): assert not kws assert len(args) == 1
# @property # def iterator_type(self): # return ArrowTableIteratorType(self).iterator_type @register_model(ArrowTableType) class ArrowTableModel(models.StructModel): def __init__(self, dmm, fe_type): members = [ ('table_ptr', types.CPointer(types.uint8)), ('meminfo', types.MemInfoPointer(types.voidptr)), ] models.StructModel.__init__(self, dmm, fe_type, members) make_attribute_wrapper(ArrowTableType, 'table_ptr', '_table_ptr') class PyarrowTableType(types.Type): def __init__(self): super(PyarrowTableType, self).__init__(name="PyarrowTableType()") register_model(PyarrowTableType)(models.OpaqueModel) # FIXME_Numba#3372: add into numba.types to allow returning from objmode types.PyarrowTableType = PyarrowTableType
self.explicit_select = explicit_select super(RollingType, self).__init__(name="RollingType({}, {}, {}, {})".format( df_type, on, selection, explicit_select)) def copy(self): # XXX is copy necessary? # TODO: key attribute? return RollingType(self.df_type, self.on, self.selection, self.explicit_select) # dummy model since info is kept in type # TODO: add df object and win/center vals to allow control flow? register_model(RollingType)(models.OpaqueModel) @overload_method(DataFrameType, 'rolling') def df_rolling_overload(df, window, min_periods=None, center=False, win_type=None, on=None, axis=0, closed=None): def _impl(df, window, min_periods=None, center=False,
def test_externally_defined_type_is_external(self): with self.create_temp_module(self.source_lines) as test_module: FooType = test_module.FooType self.assertFalse(FooType().is_internal) # set up an extension type class Foo(object): pass register_model(FooType)(models.OpaqueModel) @typeof_impl.register(Foo) def _typ_foo(val, c): return FooType() @unbox(FooType) def unbox_foo(typ, obj, c): return NativeValue(c.context.get_dummy_value()) # function to overload def false_if_not_array(a): pass # Set up an overload which will accept all types irrespective of # whether they are from Numba's closed type system @overload(false_if_not_array) def ol_false_if_not_array(a): if isinstance(a, types.Array): return lambda a : True else: return lambda a : False @njit def call_false_if_not_array(a): return false_if_not_array(a) self.assertTrue(call_false_if_not_array(np.zeros(10))) self.assertFalse(call_false_if_not_array(10)) # The extension type was accepted self.assertFalse(call_false_if_not_array(Foo())) # Now do the same sort of overload but put in a guard based on the # use of internal types def false_if_not_array_closed_system(a): pass @overload(false_if_not_array_closed_system) def ol_false_if_not_array_closed_system(a): if a.is_internal: # guard if isinstance(a, types.Array): return lambda a : True else: return lambda a : False @njit def call_false_if_not_array_closed_system(a): return false_if_not_array_closed_system(a) self.assertTrue(call_false_if_not_array_closed_system(np.zeros(10))) self.assertFalse(call_false_if_not_array_closed_system(10)) with self.assertRaises(errors.TypingError) as raises: call_false_if_not_array_closed_system(Foo()) estr = str(raises.exception) self.assertIn("Invalid use of Function", estr) self.assertIn("false_if_not_array_closed_system", estr) self.assertIn("(Foo)", estr)
super(SeriesModel, self).__init__(dmm, fe_type, members) class BoxedSeriesType(types.Type): """Series type before unboxing. Using a different type to avoid data model issues and confusion. """ def __init__(self, dtype): self.dtype = dtype name = "BoxedSeriesType({})".format(dtype) super(BoxedSeriesType, self).__init__(name) # register_model(BoxedSeriesType)(models.OpaqueModel) register_model(BoxedSeriesType)(SeriesModel) class UnBoxedSeriesType(types.Type): """Series type before boxing. Using a different type to avoid data model issues and confusion. """ def __init__(self, dtype): self.dtype = dtype name = "UnBoxedSeriesType({})".format(dtype) super(UnBoxedSeriesType, self).__init__(name) register_model(UnBoxedSeriesType)(SeriesModel)
# TODO: move to Numba class BooleanLiteral(types.Literal, types.Boolean): def can_convert_to(self, typingctx, other): # similar to IntegerLiteral conv = typingctx.can_convert(self.literal_type, other) if conv is not None: return max(conv, types.Conversion.promote) types.Literal.ctor_map[bool] = BooleanLiteral register_model(BooleanLiteral)(numba.extending.models.BooleanModel) @lower_cast(BooleanLiteral, types.Boolean) def literal_bool_cast(context, builder, fromty, toty, val): lit = context.get_constant_generic( builder, fromty.literal_type, fromty.literal_value, ) return context.cast(builder, lit, fromty.literal_type, toty) def get_constant(func_ir, var, default=NOT_CONSTANT): def_node = guard(get_definition, func_ir, var) if def_node is None:
class StringType(types.Opaque): def __init__(self): super(StringType, self).__init__(name='StringType') string_type = StringType() @typeof_impl.register(str) def _typeof_str(val, c): return string_type register_model(StringType)(models.OpaqueModel) # XXX: should be subtype of StringType? class CharType(types.Type): def __init__(self): super(CharType, self).__init__(name='CharType') self.bitwidth = 8 char_type = CharType() register_model(CharType)(models.IntegerModel) @overload(operator.getitem) def char_getitem_overload(_str, ind):
from numba.typing.templates import (signature, AbstractTemplate, infer_global, infer) from numba.extending import (register_model, models, lower_builtin) from numba import cgutils # a native buffer pointer managed explicity (e.g. deleted manually) class CBufferType(types.Opaque): def __init__(self): super(CBufferType, self).__init__(name='CBufferType') c_buffer_type = CBufferType() register_model(CBufferType)(models.OpaqueModel) def get_sendrecv_counts(): return 0 def shuffle_data(): return 0 def sort(): return 0 def local_merge():
class StringType(types.Opaque): def __init__(self): super(StringType, self).__init__(name='StringType') string_type = StringType() @typeof_impl.register(str) def _typeof_str(val, c): return string_type register_model(StringType)(models.OpaqueModel) @infer class StringAdd(ConcreteTemplate): key = "+" cases = [signature(string_type, string_type, string_type)] @infer class StringOpEq(AbstractTemplate): key = '==' def generic(self, args, kws): assert not kws (arg1, arg2) = args
def box_type(self): return MyArray @typeof_impl.register(MyArray) def typeof_ta_ndarray(val, c): try: dtype = numpy_support.from_dtype(val.dtype) except NotImplementedError: raise ValueError("Unsupported array dtype: %s" % (val.dtype, )) layout = numpy_support.map_layout(val) readonly = not val.flags.writeable return MyArrayType(dtype, val.ndim, layout, readonly=readonly) register_model(MyArrayType)(numba.core.datamodel.models.ArrayModel) @type_callable(MyArray) def type_myarray(context): def typer(shape, dtype, buf): out = MyArrayType(dtype=buf.dtype, ndim=len(shape), layout=buf.layout) return out return typer @lower_builtin(MyArray, types.UniTuple, types.DType, types.Array) def impl_myarray(context, builder, sig, args): from numba.np.arrayobj import make_array, populate_array
self.as_index = as_index self.explicit_select = explicit_select super(DataFrameGroupByType, self).__init__( name="DataFrameGroupBy({}, {}, {}, {}, {})".format( df_type, keys, selection, as_index, explicit_select)) def copy(self): # XXX is copy necessary? return DataFrameGroupByType(self.df_type, self.keys, self.selection, self.as_index, self.explicit_select) # dummy model since info is kept in type # TODO: add df object to allow control flow? register_model(DataFrameGroupByType)(models.OpaqueModel) @overload_method(DataFrameType, 'groupby') def df_groupby_overload(df, by=None, axis=0, level=None, as_index=True, sort=True, group_keys=True, squeeze=False, observed=False): if by is None: raise ValueError("groupby 'by' argument required")
@typeof_impl.register(MyDummy) def typeof_mydummy(val, c): return mydummy_type @lower_cast(MyDummyType, types.Number) def mydummy_to_number(context, builder, fromty, toty, val): """ Implicit conversion from MyDummy to int. """ return context.get_constant(toty, 42) def get_dummy(): return mydummy register_model(MyDummyType)(models.OpaqueModel) @unbox(MyDummyType) def unbox_index(typ, obj, c): return NativeValue(c.context.get_dummy_value()) # ----------------------------------------------------------------------- # Define a second custom type but w/o implicit cast to Number class MyDummy2(object): pass class MyDummyType2(types.Opaque): pass
dist_pass._array_counts[arr.name] = [end_var] nodes += out return nodes distributed.distributed_run_extensions[CsvReader] = csv_distributed_run class StreamReaderType(types.Opaque): def __init__(self): super(StreamReaderType, self).__init__(name='StreamReaderType') stream_reader_type = StreamReaderType() register_model(StreamReaderType)(models.OpaqueModel) @box(StreamReaderType) def box_stream_reader(typ, val, c): return val csv_file_chunk_reader = types.ExternalFunction( "csv_file_chunk_reader", stream_reader_type(types.voidptr, types.bool_, types.int64, types.int64)) def _get_dtype_str(t): dtype = t.dtype if isinstance(dtype, PDCategoricalDtype):
assert not kws assert len(args) == 2 return signature(args[1], *args) @bound_function("dict.pop") def resolve_pop(self, dict, args, kws): assert not kws return signature(dict.val_typ, *args) @bound_function("dict.keys") def resolve_keys(self, dict, args, kws): assert not kws return signature(DictKeyIteratorType(dict.key_typ, dict.val_typ)) register_model(DictType)(models.OpaqueModel) @box(DictType) def box_dict(typ, val, c): """ """ # interval = cgutils.create_struct_proxy(typ)(c.context, c.builder, value=val) # lo_obj = c.pyapi.float_from_double(interval.lo) # hi_obj = c.pyapi.float_from_double(interval.hi) class_obj = c.pyapi.unserialize(c.pyapi.serialize_object(DictIntInt)) res = c.pyapi.call_function_objargs(class_obj, (val, )) # c.pyapi.decref(lo_obj) # c.pyapi.decref(hi_obj) c.pyapi.decref(class_obj) return res
@property def iterator_type(self): return SetIterType(self) def is_precise(self): return self.dtype.is_precise() set_string_type = SetType(string_type) class SetIterType(types.BaseContainerIterator): container_class = SetType register_model(SetType)(models.OpaqueModel) _init_set_string = types.ExternalFunction("init_set_string", set_string_type()) def init_set_string(): return set() @overload(init_set_string) def init_set_overload(): return lambda: _init_set_string()
@typeof_impl.register(MyDummy) def typeof_mydummy(val, c): return mydummy_type @lower_cast(MyDummyType, types.Number) def mydummy_to_number(context, builder, fromty, toty, val): """ Implicit conversion from MyDummy to int. """ return context.get_constant(toty, 42) def get_dummy(): return mydummy register_model(MyDummyType)(models.OpaqueModel) @unbox(MyDummyType) def unbox_index(typ, obj, c): return NativeValue(c.context.get_dummy_value()) # ----------------------------------------------------------------------- # Define a second custom type but w/o implicit cast to Number class MyDummy2(object): pass class MyDummyType2(types.Opaque): pass
# into a njit function. @typeof_impl.register(ndarray) def typeof_ta_ndarray(val, c): try: dtype = numpy_support.from_dtype(val.dtype) except NotImplementedError: raise ValueError("Unsupported array dtype: %s" % (val.dtype,)) layout = numpy_support.map_layout(val) readonly = not val.flags.writeable return UsmSharedArrayType(dtype, val.ndim, layout, readonly=readonly) # This tells Numba to use the default Numpy ndarray data layout for # object of type UsmArray. # register_model(UsmSharedArrayType)(DPPYArrayModel) register_model(UsmSharedArrayType)(numba.core.datamodel.models.ArrayModel) # dppy_target.spirv_data_model_manager.register(UsmSharedArrayType, DPPYArrayModel) dppy_target.spirv_data_model_manager.register( UsmSharedArrayType, numba.core.datamodel.models.ArrayModel ) # This tells Numba how to convert from its native representation # of a UsmArray in a njit function back to a Python UsmArray. @box(UsmSharedArrayType) def box_array(typ, val, c): nativearycls = c.context.make_array(typ) nativeary = nativearycls(c.context, c.builder, value=val) if c.context.enable_nrt: np_dtype = numpy_support.as_dtype(typ.dtype) dtypeptr = c.env_manager.read_const(c.env_manager.add_const(np_dtype)) # Steals NRT ref