def test_validity_add(dtype, nelem):
expect_fn = np.add
test_fn = libgdf.gdf_add_generic
# data
h_lhs = gen_rand(dtype, nelem)
h_rhs = gen_rand(dtype, nelem)
d_lhs = rmm.to_device(h_lhs)
d_rhs = rmm.to_device(h_rhs)
d_result = rmm.device_array_like(d_lhs)
# valids
h_lhs_valids = gen_rand(np.int8, (nelem + 8 - 1) // 8)
h_rhs_valids = gen_rand(np.int8, (nelem + 8 - 1) // 8)
d_lhs_valids = rmm.to_device(h_lhs_valids)
d_rhs_valids = rmm.to_device(h_rhs_valids)
d_result_valids = rmm.device_array_like(d_lhs_valids)
# columns
col_lhs = new_column()
col_rhs = new_column()
col_result = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs),
unwrap_devary(d_lhs_valids), nelem, gdf_dtype)
libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs),
unwrap_devary(d_rhs_valids), nelem, gdf_dtype)
libgdf.gdf_column_view(col_result, unwrap_devary(d_result),
unwrap_devary(d_result_valids), nelem, gdf_dtype)
libgdf.gdf_validity_and(col_lhs, col_rhs, col_result)
expect = expect_fn(h_lhs, h_rhs)
test_fn(col_lhs, col_rhs, col_result)
got = d_result.copy_to_host()
# Ensure validity mask is matching
expect_valids = h_lhs_valids & h_rhs_valids
got_valids = d_result_valids.copy_to_host()
np.testing.assert_array_equal(expect_valids, got_valids)
# Masked data
mask = buffer_as_bits(expect_valids.data)[:expect.size]
expect_masked = expect[mask]
got_masked = got[mask]
print('expect')
print(expect_masked)
print('got')
print(got_masked)
np.testing.assert_array_equal(expect_masked, got_masked)
def test_lhs_rhs_dtype_mismatch():
lhs_dtype = np.int32
rhs_dtype = np.float32
nelem = 5
h_lhs = np.arange(nelem, dtype=lhs_dtype)
h_rhs = np.arange(nelem, dtype=rhs_dtype)
d_lhs = rmm.to_device(h_lhs)
d_rhs = rmm.to_device(h_rhs)
d_result = rmm.device_array_like(d_lhs)
col_lhs = new_column()
col_rhs = new_column()
col_result = new_column()
libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem,
get_dtype(lhs_dtype))
libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem,
get_dtype(rhs_dtype))
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, get_dtype(lhs_dtype))
with pytest.raises(GDFError) as raises:
libgdf.gdf_add_generic(col_lhs, col_rhs, col_result)
raises.match("GDF_UNSUPPORTED_DTYPE")
with pytest.raises(GDFError) as raises:
libgdf.gdf_eq_generic(col_lhs, col_rhs, col_result)
raises.match("GDF_UNSUPPORTED_DTYPE")
with pytest.raises(GDFError) as raises:
libgdf.gdf_bitwise_and_generic(col_lhs, col_rhs, col_result)
raises.match("GDF_UNSUPPORTED_DTYPE")
def deserialize(cls, deserialize, header, frames):
"""Called when dask.distributed is performing a deserialization for
data of this class.
Do not use this directly. It is invoked by dask.distributed.
Parameters
----------
deserialize : callable
Used to deserialize data that needs further deserialization .
header, frames : dict
See custom serialization documentation in dask.distributed.
Returns
-------
obj : Buffer
Returns an instance of Buffer.
"""
# Using IPC?
if header['kind'] == 'ipc':
ipch = deserialize(header['mem'], frames)
# Open IPC handle
with ipch as data:
# Copy remote data over
mem = rmm.device_array_like(data)
mem.copy_to_device(data)
# Not using IPC
else:
# Deserialize the numpy array
mem = deserialize(header['mem'], frames)
mem.flags['WRITEABLE'] = True # XXX: hack for numba to work
return Buffer(mem)
def find_last(arr, val, compare="eq"):
"""
Returns the index of the last occurrence of *val* in *arr*.
Or the last occurence of *arr* *compare* *val*, if *compare* is not eq
Otherwise, returns -1.
Parameters
----------
arr : device array
val : scalar
compare: str ('gt', 'lt', or 'eq' (default))
"""
found = rmm.device_array_like(arr)
if found.size > 0:
if compare == "gt":
gpu_mark_gt.forall(found.size)(arr, val, found, -1)
elif compare == "lt":
gpu_mark_lt.forall(found.size)(arr, val, found, -1)
else:
if arr.dtype in ("float32", "float64"):
gpu_mark_found_float.forall(found.size)(arr, val, found, -1)
else:
gpu_mark_found_int.forall(found.size)(arr, val, found, -1)
from cudf.dataframe.columnops import as_column
found_col = as_column(found)
max_index = found_col.max()
return max_index
def fillna(data, mask, value):
out = rmm.device_array_like(data)
out.copy_to_device(data)
if data.size > 0:
configured = gpu_fill_masked.forall(data.size)
configured(value, mask, out)
return out
def bitwise_op_test(dtype, expect_fn, test_fn, nelem=128):
h_lhs = gen_rand(dtype, nelem)
h_rhs = gen_rand(dtype, nelem)
d_lhs = rmm.to_device(h_lhs)
d_rhs = rmm.to_device(h_rhs)
d_result = rmm.device_array_like(d_lhs)
col_lhs = new_column()
col_rhs = new_column()
col_result = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, gdf_dtype)
expect = expect_fn(h_lhs, h_rhs)
test_fn(col_lhs, col_rhs, col_result)
got = d_result.copy_to_host()
print('got')
print(got)
print('expect')
print(expect)
np.testing.assert_array_equal(expect, got)
def math_op_test(dtype,
ulp,
expect_fn,
test_fn,
nelem=128,
scale=1,
positive_only=False):
randvals = gen_rand(dtype, nelem, positive_only=positive_only)
h_data = (randvals * scale).astype(dtype)
d_data = rmm.to_device(h_data)
d_result = rmm.device_array_like(d_data)
col_data = new_column()
col_result = new_column()
gdf_dtype = get_dtype(dtype)
# data column
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
gdf_dtype)
# result column
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, gdf_dtype)
expect = expect_fn(h_data)
test_fn(col_data, col_result)
got = d_result.copy_to_host()
print('got')
print(got)
print('expect')
print(expect)
np.testing.assert_array_max_ulp(expect, got, maxulp=ulp)
def arith_op_test(dtype,
ulp,
expect_fn,
test_fn,
nelem=128,
non_zero_rhs=False):
h_lhs = gen_rand(dtype, nelem)
h_rhs = gen_rand(dtype, nelem)
if non_zero_rhs:
fix_zeros(h_rhs)
d_lhs = rmm.to_device(h_lhs)
d_rhs = rmm.to_device(h_rhs)
d_result = rmm.device_array_like(d_lhs)
col_lhs = new_column()
col_rhs = new_column()
col_result = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, gdf_dtype)
expect = expect_fn(h_lhs, h_rhs)
test_fn(col_lhs, col_rhs, col_result)
got = d_result.copy_to_host()
print('got')
print(got)
print('expect')
print(expect)
np.testing.assert_array_max_ulp(expect, got, maxulp=ulp)
def _request_transfer(key, remoteinfo):
logger.info("rebuild from: %s for %r", remoteinfo, key)
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect("tcp://{0}:{1}".format(*remoteinfo))
myaddr = _global_addr[0]
theiraddr = remoteinfo[0]
if myaddr == theiraddr:
# Same machine go by IPC
logger.info("request by IPC")
socket.send(pickle.dumps(("IPC", key)))
rcv = socket.recv()
ipch = pickle.loads(rcv)
# Open IPC and copy to local context
with ipch as data:
copied = rmm.device_array_like(data)
copied.copy_to_device(data)
# Release
_request_drop(socket, key)
return copied
else:
# Different machine go by NET
logger.info("request by NET: %s->%s", theiraddr, myaddr)
socket.send(pickle.dumps(("NET", key)))
rcv = socket.recv()
output = rmm.to_device(pickle.loads(rcv))
# Release
_request_drop(socket, key)
return output
def copy_array(arr, out=None):
if out is None:
out = rmm.device_array_like(arr)
assert out.size == arr.size
if arr.is_c_contiguous() and out.is_c_contiguous():
out.copy_to_device(arr)
else:
gpu_copy.forall(out.size)(arr, out)
return out
def recode(data, recode_table, na_value):
"""Recode data with the given recode table.
And setting out-of-range values to *na_value*
"""
newdata = rmm.device_array_like(data)
recode_table = to_device(recode_table)
blksz = 32 * 4
blkct = min(16, max(1, data.size // blksz))
gpu_recode[blkct, blksz](newdata, data, recode_table, na_value)
return newdata
def run(self, arr, k):
if k >= MAX_FAST_UNIQUE_K:
raise NotImplementedError('k >= {}'.format(MAX_FAST_UNIQUE_K))
# setup mem
outsz_ptr = rmm.device_array(shape=1, dtype=np.intp)
out = rmm.device_array_like(arr)
# kernel
self._kernel[1, 64](arr, k, out, outsz_ptr)
# copy to host
unique_ct = outsz_ptr.copy_to_host()[0]
if unique_ct < 0:
raise ValueError('too many unique value (hint: increase k)')
else:
hout = out.copy_to_host()
return hout[:unique_ct]
def test_rmm_alloc(dtype, nelem):
# data
h_in = gen_rand(dtype, nelem)
h_result = gen_rand(dtype, nelem)
d_in = rmm.to_device(h_in)
d_result = rmm.device_array_like(d_in)
d_result.copy_to_device(d_in)
h_result = d_result.copy_to_host()
print('expect')
print(h_in)
print('got')
print(h_result)
np.testing.assert_array_equal(h_result, h_in)
def array_tester(dtype, nelem):
# data
h_in = np.full(nelem, 3.2, dtype)
h_result = np.empty(nelem, dtype)
d_in = rmm.to_device(h_in)
d_result = rmm.device_array_like(d_in)
d_result.copy_to_device(d_in)
h_result = d_result.copy_to_host()
print('expect')
print(h_in)
print('got')
print(h_result)
np.testing.assert_array_equal(h_result, h_in)
def array_tester(dtype, nelem):
# data
h_in = np.full(nelem, 3.2, dtype)
h_result = np.empty(nelem, dtype)
d_in = rmm.to_device(h_in)
d_result = rmm.device_array_like(d_in)
d_result.copy_to_device(d_in)
h_result = d_result.copy_to_host()
print('expect')
print(h_in)
print('got')
print(h_result)
np.testing.assert_array_equal(h_result, h_in)
def test_rmm_csv_log():
dtype = np.int32
nelem=1024
# data
h_in = gen_rand(dtype, nelem)
h_result = gen_rand(dtype, nelem)
d_in = rmm.to_device(h_in)
d_result = rmm.device_array_like(d_in)
d_result.copy_to_device(d_in)
h_result = d_result.copy_to_host()
csv=rmm.csv_log()
print(csv[:1000])
assert(csv.find("Event Type,Device ID,Address,Stream,Size (bytes),Free Memory,Total Memory,Current Allocs,Start,End,Elapsed") >= 0)
def test_col_mismatch_error():
nelem = 128
h_data = np.random.random(nelem).astype(np.float32)
d_data = rmm.to_device(h_data)
d_result = rmm.device_array_like(d_data)
col_data = new_column()
col_result = new_column()
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
libgdf.GDF_FLOAT32)
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem + 10, libgdf.GDF_FLOAT32)
with pytest.raises(GDFError) as excinfo:
libgdf.gdf_sin_generic(col_data, col_result)
assert 'GDF_COLUMN_SIZE_MISMATCH' == str(excinfo.value)
def test_rmm_csv_log():
dtype = np.int32
nelem = 1024
# data
h_in = np.full(nelem, 3.2, dtype)
d_in = rmm.to_device(h_in)
d_result = rmm.device_array_like(d_in)
d_result.copy_to_device(d_in)
csv = rmm.csv_log()
print(csv[:1000])
assert (csv.find("Event Type,Device ID,Address,Stream,Size (bytes),"
"Free Memory,Total Memory,Current Allocs,Start,End,"
"Elapsed,Location") >= 0)
def test_unsupported_dtype_error():
nelem = 128
h_data = np.random.random(nelem).astype(np.float32)
d_data = rmm.to_device(h_data)
d_result = rmm.device_array_like(d_data)
col_data = new_column()
col_result = new_column()
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
libgdf.GDF_INT32)
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem + 10, libgdf.GDF_FLOAT32)
with pytest.raises(GDFError) as excinfo:
libgdf.gdf_sin_generic(col_data, col_result)
assert 'GDF_UNSUPPORTED_DTYPE' == str(excinfo.value)
def test_rmm_csv_log():
dtype = np.int32
nelem = 1024
# data
h_in = np.full(nelem, 3.2, dtype)
d_in = rmm.to_device(h_in)
d_result = rmm.device_array_like(d_in)
d_result.copy_to_device(d_in)
csv = rmm.csv_log()
print(csv[:1000])
assert(csv.find("Event Type,Device ID,Address,Stream,Size (bytes),"
"Free Memory,Total Memory,Current Allocs,Start,End,"
"Elapsed,Location") >= 0)
def find_last(arr, val):
"""
Returns the index of the last occurrence of *val* in *arr*.
Otherwise, returns -1.
Parameters
----------
arr : device array
val : scalar
"""
found = rmm.device_array_like(arr)
if found.size > 0:
if arr.dtype in ('float32', 'float64'):
gpu_mark_found_float.forall(found.size)(arr, val, found, -1)
else:
gpu_mark_found_int.forall(found.size)(arr, val, found, -1)
from cudf.dataframe.columnops import as_column
found_col = as_column(found)
max_index = found_col.max()
return max_index
def fill_mask(data, mask, value):
"""fill a column with the same value using a custom mask
Parameters
----------
data : device array
data
mask : device array
validity mask
value : scale
fill value
Returns
-------
device array
mask filled column with scalar value
"""
out = rmm.device_array_like(data)
out.copy_to_device(data)
if data.size > 0:
configured = gpu_fill_masked.forall(data.size)
configured(value, mask, out)
return out
def as_column(arbitrary, nan_as_null=True, dtype=None):
"""Create a Column from an arbitrary object
Currently support inputs are:
* ``Column``
* ``Buffer``
* ``Series``
* ``Index``
* numba device array
* cuda array interface
* numpy array
* pyarrow array
* pandas.Categorical
Returns
-------
result : subclass of TypedColumnBase
- CategoricalColumn for pandas.Categorical input.
- DatetimeColumn for datetime input
- NumericalColumn for all other inputs.
"""
from cudf.dataframe import numerical, categorical, datetime, string
from cudf.dataframe.series import Series
from cudf.dataframe.index import Index
if isinstance(arbitrary, Column):
categories = None
if hasattr(arbitrary, "categories"):
categories = arbitrary.categories
data = build_column(arbitrary.data,
arbitrary.dtype,
mask=arbitrary.mask,
categories=categories)
elif isinstance(arbitrary, Series):
data = arbitrary._column
elif isinstance(arbitrary, Index):
data = arbitrary._values
elif isinstance(arbitrary, Buffer):
data = numerical.NumericalColumn(data=arbitrary, dtype=arbitrary.dtype)
elif isinstance(arbitrary, nvstrings.nvstrings):
data = string.StringColumn(data=arbitrary)
elif cuda.devicearray.is_cuda_ndarray(arbitrary):
data = as_column(Buffer(arbitrary))
if (data.dtype in [np.float16, np.float32, np.float64]
and arbitrary.size > 0):
if nan_as_null:
mask = cudautils.mask_from_devary(arbitrary)
data = data.set_mask(mask)
elif cuda.is_cuda_array(arbitrary):
# Use cuda array interface to do create a numba device array by
# reference
new_dev_array = cuda.as_cuda_array(arbitrary)
# Allocate new output array using rmm and copy the numba device array
# to an rmm owned device array
out_dev_array = rmm.device_array_like(new_dev_array)
out_dev_array.copy_to_device(new_dev_array)
data = as_column(out_dev_array)
elif isinstance(arbitrary, np.ndarray):
# CUDF assumes values are always contiguous
if not arbitrary.flags['C_CONTIGUOUS']:
arbitrary = np.ascontiguousarray(arbitrary)
if arbitrary.dtype.kind == 'M':
data = datetime.DatetimeColumn.from_numpy(arbitrary)
elif arbitrary.dtype.kind in ('O', 'U'):
data = as_column(pa.Array.from_pandas(arbitrary))
else:
data = as_column(rmm.to_device(arbitrary), nan_as_null=nan_as_null)
elif isinstance(arbitrary, pa.Array):
if isinstance(arbitrary, pa.StringArray):
count = len(arbitrary)
null_count = arbitrary.null_count
buffers = arbitrary.buffers()
# Buffer of actual strings values
if buffers[2] is not None:
sbuf = np.frombuffer(buffers[2], dtype='int8')
else:
sbuf = np.empty(0, dtype='int8')
# Buffer of offsets values
obuf = np.frombuffer(buffers[1], dtype='int32')
# Buffer of null bitmask
nbuf = None
if null_count > 0:
nbuf = np.frombuffer(buffers[0], dtype='int8')
data = as_column(
nvstrings.from_offsets(sbuf,
obuf,
count,
nbuf=nbuf,
ncount=null_count))
elif isinstance(arbitrary, pa.NullArray):
new_dtype = dtype
if (type(dtype) == str and dtype == 'empty') or dtype is None:
new_dtype = np.dtype(arbitrary.type.to_pandas_dtype())
if pd.api.types.is_categorical_dtype(new_dtype):
arbitrary = arbitrary.dictionary_encode()
else:
if nan_as_null:
arbitrary = arbitrary.cast(_gdf.np_to_pa_dtype(new_dtype))
else:
# casting a null array doesn't make nans valid
# so we create one with valid nans from scratch:
if new_dtype == np.dtype("object"):
arbitrary = utils.scalar_broadcast_to(
None, (len(arbitrary), ), dtype=new_dtype)
else:
arbitrary = utils.scalar_broadcast_to(
np.nan, (len(arbitrary), ), dtype=new_dtype)
data = as_column(arbitrary, nan_as_null=nan_as_null)
elif isinstance(arbitrary, pa.DictionaryArray):
pamask, padata = buffers_from_pyarrow(arbitrary)
data = categorical.CategoricalColumn(
data=padata,
mask=pamask,
null_count=arbitrary.null_count,
categories=arbitrary.dictionary.to_pylist(),
ordered=arbitrary.type.ordered,
)
elif isinstance(arbitrary, pa.TimestampArray):
arbitrary = arbitrary.cast(pa.timestamp('ms'))
pamask, padata = buffers_from_pyarrow(arbitrary, dtype='M8[ms]')
data = datetime.DatetimeColumn(data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=np.dtype('M8[ms]'))
elif isinstance(arbitrary, pa.Date64Array):
pamask, padata = buffers_from_pyarrow(arbitrary, dtype='M8[ms]')
data = datetime.DatetimeColumn(data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=np.dtype('M8[ms]'))
elif isinstance(arbitrary, pa.Date32Array):
# No equivalent np dtype and not yet supported
warnings.warn(
"Date32 values are not yet supported so this will "
"be typecast to a Date64 value", UserWarning)
arbitrary = arbitrary.cast(pa.date64())
data = as_column(arbitrary)
elif isinstance(arbitrary, pa.BooleanArray):
# Arrow uses 1 bit per value while we use int8
dtype = np.dtype(np.bool)
# Needed because of bug in PyArrow
# https://issues.apache.org/jira/browse/ARROW-4766
if len(arbitrary) > 0:
arbitrary = arbitrary.cast(pa.int8())
else:
arbitrary = pa.array([], type=pa.int8())
pamask, padata = buffers_from_pyarrow(arbitrary, dtype=dtype)
data = numerical.NumericalColumn(data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=dtype)
else:
pamask, padata = buffers_from_pyarrow(arbitrary)
data = numerical.NumericalColumn(
data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=np.dtype(arbitrary.type.to_pandas_dtype()))
elif isinstance(arbitrary, pa.ChunkedArray):
gpu_cols = [
as_column(chunk, dtype=dtype) for chunk in arbitrary.chunks
]
if dtype and dtype != 'empty':
new_dtype = dtype
else:
pa_type = arbitrary.type
if pa.types.is_dictionary(pa_type):
new_dtype = 'category'
else:
new_dtype = np.dtype(pa_type.to_pandas_dtype())
data = Column._concat(gpu_cols, dtype=new_dtype)
elif isinstance(arbitrary, (pd.Series, pd.Categorical)):
if pd.api.types.is_categorical_dtype(arbitrary):
data = as_column(pa.array(arbitrary, from_pandas=True))
elif arbitrary.dtype == np.bool:
# Bug in PyArrow or HDF that requires us to do this
data = as_column(pa.array(np.array(arbitrary), from_pandas=True))
else:
data = as_column(pa.array(arbitrary, from_pandas=nan_as_null))
elif isinstance(arbitrary, pd.Timestamp):
# This will always treat NaTs as nulls since it's not technically a
# discrete value like NaN
data = as_column(pa.array(pd.Series([arbitrary]), from_pandas=True))
elif np.isscalar(arbitrary) and not isinstance(arbitrary, memoryview):
if hasattr(arbitrary, 'dtype'):
data_type = _gdf.np_to_pa_dtype(arbitrary.dtype)
if data_type in (pa.date64(), pa.date32()):
# PyArrow can't construct date64 or date32 arrays from np
# datetime types
arbitrary = arbitrary.astype('int64')
data = as_column(pa.array([arbitrary], type=data_type))
else:
data = as_column(pa.array([arbitrary]), nan_as_null=nan_as_null)
elif isinstance(arbitrary, memoryview):
data = as_column(np.array(arbitrary),
dtype=dtype,
nan_as_null=nan_as_null)
else:
try:
data = as_column(memoryview(arbitrary))
except TypeError:
try:
pa_type = None
if dtype is not None:
if pd.api.types.is_categorical_dtype(dtype):
raise TypeError
else:
np_type = np.dtype(dtype).type
if np_type == np.bool_:
pa_type = pa.bool_()
else:
pa_type = _gdf.np_to_pa_dtype(np.dtype(dtype).type)
data = as_column(pa.array(arbitrary,
type=pa_type,
from_pandas=nan_as_null),
nan_as_null=nan_as_null)
except (pa.ArrowInvalid, pa.ArrowTypeError, TypeError):
np_type = None
if pd.api.types.is_categorical_dtype(dtype):
data = as_column(pd.Series(arbitrary, dtype='category'),
nan_as_null=nan_as_null)
else:
if dtype is None:
np_type = None
else:
np_type = np.dtype(dtype)
data = as_column(np.array(arbitrary, dtype=np_type),
nan_as_null=nan_as_null)
return data
def apply_round(data, decimal):
output_dary = rmm.device_array_like(data)
if output_dary.size > 0:
gpu_round.forall(output_dary.size)(data, output_dary, decimal)
return output_dary
def input_to_dev_array(X,
order='F',
deepcopy=False,
check_dtype=False,
convert_to_dtype=False,
check_cols=False,
check_rows=False,
fail_on_order=False):
"""
Convert input X to device array suitable for C++ methods
Acceptable input formats:
* cuDF Dataframe - returns a deep copy always
* cuDF Series - returns by reference or a deep copy depending on
`deepcopy`
* Numpy array - returns a copy in device always
* cuda array interface compliant array (like Cupy) - returns a
reference unless deepcopy=True
* numba device array - returns a reference unless deepcopy=True
Returns: namedtuple('dev_array', 'array pointer n_rows n_cols dtype')
`dev_array` is a new device array if the input was not a numba device
array. It is a reference to the input X if it was a numba device array
or cuda array interface compliant (like cupy)
"""
if convert_to_dtype:
X = convert_dtype(X, to_dtype=convert_to_dtype)
check_dtype = False
if isinstance(X, cudf.DataFrame):
dtype = np.dtype(X[X.columns[0]]._column.dtype)
if order == 'F':
X_m = X.as_gpu_matrix(order='F')
elif order == 'C':
X_m = cuml.utils.numba_utils.row_matrix(X)
elif (isinstance(X, cudf.Series)):
if deepcopy:
X_m = X.to_gpu_array()
else:
if X.null_count == 0:
X_m = X._column._data.mem
else:
raise ValueError("Error: cuDF Series has missing/null values")
elif isinstance(X, np.ndarray):
dtype = X.dtype
X_m = rmm.to_device(np.array(X, order=order, copy=False))
elif cuda.is_cuda_array(X):
# Use cuda array interface to create a device array by reference
X_m = cuda.as_cuda_array(X)
if deepcopy:
out_dev_array = rmm.device_array_like(X_m)
out_dev_array.copy_to_device(X_m)
X_m = out_dev_array
elif cuda.devicearray.is_cuda_ndarray(X):
if deepcopy:
out_dev_array = rmm.device_array_like(X)
out_dev_array.copy_to_device(X)
X_m = out_dev_array
else:
X_m = X
else:
msg = "X matrix format " + str(X.__class__) + " not supported"
raise TypeError(msg)
dtype = X_m.dtype
if check_dtype:
if dtype != check_dtype:
del X_m
raise TypeError("Expected " + str(check_dtype) + "input but got " +
str(dtype) + " instead.")
n_rows = X_m.shape[0]
if len(X_m.shape) > 1:
n_cols = X_m.shape[1]
else:
n_cols = 1
if check_cols:
if n_cols != check_cols:
raise ValueError("Expected " + str(check_cols) +
" columns but got " + str(n_cols) + " columns.")
if check_rows:
if n_rows != check_rows:
raise ValueError("Expected " + str(check_rows) + " rows but got " +
str(n_rows) + " rows.")
if not check_numba_order(X_m, order):
if fail_on_order:
raise ValueError("Expected " + order_to_str(order) +
" major order, but got the opposite.")
else:
warnings.warn("Expected " + order_to_str(order) + " major order, "
"but got the opposite. Converting data, this will "
"result in additional memory utilization.")
cuml.utils.numba_utils.gpu_major_converter(X_m,
n_rows,
n_cols,
dtype,
to_order=order)
X_ptr = get_dev_array_ptr(X_m)
result = namedtuple('dev_array', 'array pointer n_rows n_cols dtype')
return result(array=X_m,
pointer=X_ptr,
n_rows=n_rows,
n_cols=n_cols,
dtype=dtype)
