def test_sum_of_squares(dtype, nelem):
data = gen_rand(dtype, nelem)
d_data = rmm.to_device(data)
d_result = rmm.device_array(libgdf.gdf_reduce_optimal_output_size(),
dtype=d_data.dtype)
col_data = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_sum_of_squares(col_data, unwrap_devary(d_result), d_result.size)
got = d_result.copy_to_host()[0]
expect = (data ** 2).sum()
print('expect:', expect)
print('got:', got)
if np.dtype(dtype).kind == 'i':
if 0 <= expect <= np.iinfo(dtype).max:
np.testing.assert_array_almost_equal(expect, got)
else:
print('overflow, passing')
else:
np.testing.assert_array_almost_equal(expect, got,
decimal=accuracy_for_dtype[dtype])
def test_prefixsum(dtype, nelem):
if dtype == np.int8:
# to keep data in range
data = gen_rand(dtype, nelem, low=-2, high=2)
else:
data = gen_rand(dtype, nelem)
d_data = rmm.to_device(data)
d_result = rmm.device_array(d_data.size, dtype=d_data.dtype)
col_data = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
gdf_dtype)
col_result = new_column()
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, gdf_dtype)
inclusive = True
libgdf.gdf_prefixsum_generic(col_data, col_result, inclusive)
expect = np.cumsum(d_data.copy_to_host())
got = d_result.copy_to_host()
if not inclusive:
expect = expect[:-1]
assert got[0] == 0
got = got[1:]
np.testing.assert_array_equal(expect, got)
def cast_op_test(dtype, to_dtype, test_fn, nelem=128):
h_data = gen_rand(dtype, nelem).astype(dtype)
d_data = cuda.to_device(h_data)
d_result = cuda.device_array(d_data.size, dtype=to_dtype)
assert d_data.dtype == dtype
assert d_result.dtype == to_dtype
col_data = new_column()
col_result = new_column()
# data column
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
get_dtype(dtype))
# result column
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, get_dtype(to_dtype))
expect = h_data.astype(to_dtype)
test_fn(col_data, col_result)
got = d_result.copy_to_host()
print('got')
print(got)
print('expect')
print(expect)
np.testing.assert_equal(expect, got)
def math_op_test(dtype, ulp, expect_fn, test_fn, nelem=128, scale=1):
h_data = (gen_rand(dtype, nelem) * scale).astype(dtype)
d_data = cuda.to_device(h_data)
d_result = cuda.device_array_like(d_data)
col_data = new_column()
col_result = new_column()
gdf_dtype = get_dtype(dtype)
# data column
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
gdf_dtype)
# result column
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, gdf_dtype)
expect = expect_fn(h_data)
test_fn(col_data, col_result)
got = d_result.copy_to_host()
print('got')
print(got)
print('expect')
print(expect)
np.testing.assert_array_max_ulp(expect, got, maxulp=ulp)
def test_product(dtype, nelem):
if np.dtype(dtype).kind == 'i':
data = np.ones(nelem, dtype=dtype)
# Set at most 30 items to [0..2) to keep the value within 2^32
for _ in range(30):
data[random.randrange(nelem)] = random.random() * 2
else:
data = gen_rand(dtype, nelem)
print('max', data.max(), 'min', data.min())
d_data = cuda.to_device(data)
d_result = cuda.device_array(libgdf.gdf_reduce_optimal_output_size(),
dtype=d_data.dtype)
col_data = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_product_generic(col_data, unwrap_devary(d_result),
d_result.size)
got = d_result.copy_to_host()[0]
expect = np.product(data)
print('expect:', expect)
print('got:', got)
np.testing.assert_array_almost_equal(expect, got)
def _call_hash_multi(api, ncols, col_input, magic, nrows):
out_ary = np.zeros(nrows, dtype=np.int32)
d_out = cuda.to_device(out_ary)
col_out = new_column()
libgdf.gdf_column_view(col_out, unwrap_devary(d_out), ffi.NULL,
out_ary.size, get_dtype(d_out.dtype))
api(ncols, col_input, magic, col_out)
dataptr = col_out.data
print(dataptr)
datasize = col_out.size
print(datasize)
addr = ctypes.c_uint64(int(ffi.cast("uintptr_t", dataptr)))
print(hex(addr.value))
memptr = cuda.driver.MemoryPointer(context=cuda.current_context(),
pointer=addr,
size=4 * datasize)
print(memptr)
ary = cuda.devicearray.DeviceNDArray(shape=(datasize, ),
strides=(4, ),
dtype=np.dtype(np.int32),
gpu_data=memptr)
hashed_result = ary.copy_to_host()
print(hashed_result)
return hashed_result
def columnview(size, data, mask=None, dtype=None):
"""
Make a column view.
Parameters
----------
size : int
Data count.
data : Buffer
The data buffer.
mask : Buffer; optional
The mask buffer.
dtype : numpy.dtype; optional
The dtype of the data. Defaults to *data.dtype*.
"""
def unwrap(buffer):
if buffer is None:
return ffi.NULL
assert buffer.mem.is_c_contiguous(), "libGDF expects contiguous memory"
devary = buffer.to_gpu_array()
return unwrap_devary(devary)
dtype = dtype or data.dtype
colview = ffi.new('gdf_column*')
libgdf.gdf_column_view(colview, unwrap(data), unwrap(mask), size,
np_to_gdf_dtype(dtype))
return colview
def test_cuda_error():
dtype = np.float32
col = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col, ffi.NULL, ffi.NULL, 0, gdf_dtype)
def _make_hash_input(hash_input, ncols):
ci = []
for i in range(ncols):
d_input = cuda.to_device(hash_input[i])
col_input = new_column()
libgdf.gdf_column_view(col_input, unwrap_devary(d_input), ffi.NULL,
hash_input[i].size, get_dtype(d_input.dtype))
ci.append(col_input)
yield ci
def test_radixsort(nelem, descending, dtype):
def expected_fn(key):
# Use mergesort for stable sort
# Negate the key for descending
if issubclass(dtype, np.integer):
def negate_values(v):
return ~key
else:
# Note: this doesn't work on the smallest value of integer
# i.e. -((int8)-128) -> -128
def negate_values(v):
return -key
sorted_idx = np.argsort(negate_values(key) if descending else key,
kind='mergesort')
sorted_keys = key[sorted_idx]
# Returns key, vals
return sorted_keys, sorted_idx
# Make data
key = gen_rand(dtype, nelem)
d_key = rmm.to_device(key)
col_key = new_column()
libgdf.gdf_column_view(col_key, unwrap_devary(d_key), ffi.NULL, nelem,
get_dtype(d_key.dtype))
val = np.arange(nelem, dtype=np.int64)
d_val = rmm.to_device(val)
col_val = new_column()
libgdf.gdf_column_view(col_val, unwrap_devary(d_val), ffi.NULL, nelem,
get_dtype(d_val.dtype))
sizeof_key = d_key.dtype.itemsize
sizeof_val = d_val.dtype.itemsize
begin_bit = 0
end_bit = sizeof_key * 8
# Setup plan
plan = libgdf.gdf_radixsort_plan(nelem, descending, begin_bit, end_bit)
libgdf.gdf_radixsort_plan_setup(plan, sizeof_key, sizeof_val)
# Sort
libgdf.gdf_radixsort_generic(plan, col_key, col_val)
# Cleanup
libgdf.gdf_radixsort_plan_free(plan)
# Check
got_keys = d_key.copy_to_host()
got_vals = d_val.copy_to_host()
sorted_keys, sorted_vals = expected_fn(key)
np.testing.assert_array_equal(sorted_keys, got_keys)
np.testing.assert_array_equal(sorted_vals, got_vals)
def test_cuda_error():
dtype = np.float32
col = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col, ffi.NULL, ffi.NULL, 0, gdf_dtype)
with pytest.raises(GDFError) as raises:
libgdf.gdf_add_generic(col, col, col)
raises.match("CUDA ERROR.")
def _make_input(left, right):
d_left = cuda.to_device(left)
col_left = new_column()
libgdf.gdf_column_view(col_left, unwrap_devary(d_left), ffi.NULL,
left.size, get_dtype(d_left.dtype))
d_right = cuda.to_device(right)
col_right = new_column()
libgdf.gdf_column_view(col_right, unwrap_devary(d_right), ffi.NULL,
right.size, get_dtype(d_right.dtype))
yield col_left, col_right
def _call_hash_multi(api, ncols, col_input, magic, nrows):
out_ary = np.zeros(nrows, dtype=np.int32)
d_out = rmm.to_device(out_ary)
col_out = new_column()
libgdf.gdf_column_view(col_out, unwrap_devary(d_out), ffi.NULL,
out_ary.size, get_dtype(d_out.dtype))
api(ncols, col_input, magic, col_out)
hashed_result = d_out.copy_to_host()
print(hashed_result)
return hashed_result
def columnview(size, data, mask=None, dtype=None):
"""
Make a column view.
"""
def unwrap(buffer):
if buffer is None:
return ffi.NULL
devary = buffer.to_gpu_array()
return ffi.cast('void*', devary.device_ctypes_pointer.value)
dtype = dtype or data.dtype
colview = ffi.new('gdf_column*')
libgdf.gdf_column_view(colview, unwrap(data), unwrap(mask), size,
np_to_gdf_dtype(dtype))
return colview
def columnview(size, data, mask=None, dtype=None):
"""
Make a column view.
"""
def unwrap(buffer):
if buffer is None:
return ffi.NULL
devary = buffer.to_gpu_array()
return unwrap_devary(devary)
dtype = dtype or data.dtype
colview = ffi.new('gdf_column*')
libgdf.gdf_column_view(colview, unwrap(data), unwrap(mask), size,
np_to_gdf_dtype(dtype))
return colview
def _make_hash_input(hash_input, ncols):
ci = []
di = []
for i in range(ncols):
di.append(rmm.to_device(hash_input[i]))
for i in range(ncols):
col_input = new_column()
libgdf.gdf_column_view(col_input, unwrap_devary(di[i]), ffi.NULL,
hash_input[i].size,
get_dtype(hash_input[i].dtype))
ci.append(col_input)
initial_hash_values = rmm.to_device(np.arange(ncols, dtype=np.uint32))
yield ci, unwrap_devary(initial_hash_values)
def _make_input_multi(left, right, ncols):
cl = []
cr = []
for i in range(ncols):
d_left = cuda.to_device(left[i])
col_left = new_column()
libgdf.gdf_column_view(col_left, unwrap_devary(d_left), ffi.NULL,
left[i].size, get_dtype(d_left.dtype))
cl.append(col_left)
d_right = cuda.to_device(right[i])
col_right = new_column()
libgdf.gdf_column_view(col_right, unwrap_devary(d_right), ffi.NULL,
right[i].size, get_dtype(d_right.dtype))
cr.append(col_right)
yield cl, cr
def test_unsupported_dtype_error():
nelem = 128
h_data = np.random.random(nelem).astype(np.float32)
d_data = cuda.to_device(h_data)
d_result = cuda.device_array_like(d_data)
col_data = new_column()
col_result = new_column()
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
libgdf.GDF_INT32)
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem + 10, libgdf.GDF_FLOAT32)
with pytest.raises(GDFError) as excinfo:
libgdf.gdf_sin_generic(col_data, col_result)
assert 'GDF_UNSUPPORTED_DTYPE' == str(excinfo.value)
def test_col_mismatch_error():
nelem = 128
h_data = np.random.random(nelem).astype(np.float32)
d_data = cuda.to_device(h_data)
d_result = cuda.device_array_like(d_data)
col_data = new_column()
col_result = new_column()
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
libgdf.GDF_FLOAT32)
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem + 10, libgdf.GDF_FLOAT32)
with pytest.raises(GDFError) as excinfo:
libgdf.gdf_sin_generic(col_data, col_result)
assert 'GDF_COLUMN_SIZE_MISMATCH' == str(excinfo.value)
def test_sum(dtype, nelem):
data = gen_rand(dtype, nelem)
d_data = cuda.to_device(data)
d_result = cuda.device_array(libgdf.gdf_reduce_optimal_output_size(),
dtype=d_data.dtype)
col_data = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_sum_generic(col_data, unwrap_devary(d_result), d_result.size)
got = d_result.copy_to_host()[0]
expect = data.sum()
print('expect:', expect)
print('got:', got)
np.testing.assert_array_almost_equal(expect, got)
def _columnview(size, data, mask, dtype, null_count):
colview = ffi.new('gdf_column*')
if null_count is None:
libgdf.gdf_column_view(
colview,
data,
mask,
size,
np_to_gdf_dtype(dtype),
)
else:
libgdf.gdf_column_view_augmented(
colview,
data,
mask,
size,
np_to_gdf_dtype(dtype),
null_count,
)
return colview
def test_max(dtype, nelem):
data = gen_rand(dtype, nelem)
d_data = rmm.to_device(data)
d_result = rmm.device_array(libgdf.gdf_reduce_optimal_output_size(),
dtype=d_data.dtype)
col_data = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_max(col_data, unwrap_devary(d_result), d_result.size)
got = d_result.copy_to_host()[0]
expect = data.max()
print('expect:', expect)
print('got:', got)
assert expect == got
def test_prefixsum_masked(dtype, nelem):
if dtype == np.int8:
data = gen_rand(dtype, nelem, low=-2, high=2)
else:
data = gen_rand(dtype, nelem)
mask = gen_rand(np.int8, (nelem + 8 - 1) // 8)
dummy_mask = gen_rand(np.int8, (nelem + 8 - 1) // 8)
d_data = rmm.to_device(data)
d_mask = rmm.to_device(mask)
d_result = rmm.device_array(d_data.size, dtype=d_data.dtype)
d_result_mask = rmm.to_device(dummy_mask)
gdf_dtype = get_dtype(dtype)
extra_dtype_info = ffi.new('gdf_dtype_extra_info*')
extra_dtype_info.time_unit = libgdf.TIME_UNIT_NONE
col_data = new_column()
libgdf.gdf_column_view_augmented(col_data, unwrap_devary(d_data),
unwrap_devary(d_mask), nelem, gdf_dtype,
count_nulls(d_mask, nelem),
extra_dtype_info[0])
col_result = new_column()
libgdf.gdf_column_view(col_result, unwrap_devary(d_result),
unwrap_devary(d_result_mask), nelem, gdf_dtype)
inclusive = True
libgdf.gdf_prefixsum(col_data, col_result, inclusive)
boolmask = buffer_as_bits(mask)[:nelem]
expect = np.cumsum(data[boolmask])
got = d_result.copy_to_host()[boolmask]
if not inclusive:
expect = expect[:-1]
assert got[0] == 0
got = got[1:]
decimal = 4 if dtype == np.float32 else 6
np.testing.assert_array_almost_equal(expect, got, decimal=decimal)
def test_sum_masked(nelem):
dtype = np.float64
data = gen_rand(dtype, nelem)
mask = gen_rand(np.int8, (nelem + 8 - 1) // 8)
d_data = cuda.to_device(data)
d_mask = cuda.to_device(mask)
d_result = cuda.device_array(libgdf.gdf_reduce_optimal_output_size(),
dtype=d_data.dtype)
col_data = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_data, unwrap_devary(d_data),
unwrap_devary(d_mask), nelem, gdf_dtype)
libgdf.gdf_sum_generic(col_data, unwrap_devary(d_result), d_result.size)
got = d_result.copy_to_host()[0]
boolmask = buffer_as_bits(mask)[:nelem]
expect = data[boolmask].sum()
np.testing.assert_almost_equal(expect, got)
def arith_op_test(dtype,
ulp,
expect_fn,
test_fn,
nelem=128,
non_zero_rhs=False):
h_lhs = gen_rand(dtype, nelem)
h_rhs = gen_rand(dtype, nelem)
if non_zero_rhs:
fix_zeros(h_rhs)
d_lhs = rmm.to_device(h_lhs)
d_rhs = rmm.to_device(h_rhs)
d_result = rmm.device_array_like(d_lhs)
col_lhs = new_column()
col_rhs = new_column()
col_result = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, gdf_dtype)
expect = expect_fn(h_lhs, h_rhs)
test_fn(col_lhs, col_rhs, col_result)
got = d_result.copy_to_host()
print('got')
print(got)
print('expect')
print(expect)
np.testing.assert_array_max_ulp(expect, got, maxulp=ulp)
def bitwise_op_test(dtype, expect_fn, test_fn, nelem=128):
h_lhs = gen_rand(dtype, nelem)
h_rhs = gen_rand(dtype, nelem)
d_lhs = rmm.to_device(h_lhs)
d_rhs = rmm.to_device(h_rhs)
d_result = rmm.device_array_like(d_lhs)
col_lhs = new_column()
col_rhs = new_column()
col_result = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, gdf_dtype)
expect = expect_fn(h_lhs, h_rhs)
test_fn(col_lhs, col_rhs, col_result)
got = d_result.copy_to_host()
print('got')
print(got)
print('expect')
print(expect)
np.testing.assert_array_equal(expect, got)
def test_output_dtype_mismatch():
lhs_dtype = np.int32
rhs_dtype = np.int32
nelem = 5
h_lhs = np.arange(nelem, dtype=lhs_dtype)
h_rhs = np.arange(nelem, dtype=rhs_dtype)
d_lhs = rmm.to_device(h_lhs)
d_rhs = rmm.to_device(h_rhs)
d_result = rmm.device_array(d_lhs.size, dtype=np.float32)
col_lhs = new_column()
col_rhs = new_column()
col_result = new_column()
libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem,
get_dtype(lhs_dtype))
libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem,
get_dtype(rhs_dtype))
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, get_dtype(d_result.dtype))
with pytest.raises(GDFError) as raises:
libgdf.gdf_add_generic(col_lhs, col_rhs, col_result)
raises.match("GDF_UNSUPPORTED_DTYPE")
with pytest.raises(GDFError) as raises:
libgdf.gdf_eq_generic(col_lhs, col_rhs, col_result)
raises.match("GDF_UNSUPPORTED_DTYPE")
with pytest.raises(GDFError) as raises:
libgdf.gdf_bitwise_and_generic(col_lhs, col_rhs, col_result)
raises.match("GDF_UNSUPPORTED_DTYPE")
def logical_op_test(dtype, expect_fn, test_fn, nelem=128, gdf_dtype=None):
h_lhs = gen_rand(dtype, nelem)
h_rhs = gen_rand(dtype, nelem)
d_lhs = rmm.to_device(h_lhs)
d_rhs = rmm.to_device(h_rhs)
d_result = rmm.device_array(d_lhs.size, dtype=np.bool)
col_lhs = new_column()
col_rhs = new_column()
col_result = new_column()
gdf_dtype = get_dtype(dtype) if gdf_dtype is None else gdf_dtype
libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL,
nelem, libgdf.GDF_INT8)
expect = expect_fn(h_lhs, h_rhs)
test_fn(col_lhs, col_rhs, col_result)
got = d_result.copy_to_host()
print(expect, got)
np.testing.assert_equal(expect, got)
def test_sum(dtype, nelem):
data = gen_rand(dtype, nelem)
d_data = rmm.to_device(data)
d_result = rmm.device_array(
libgdf.gdf_reduction_get_intermediate_output_size(),
dtype=d_data.dtype)
col_data = new_column()
gdf_dtype = get_dtype(dtype)
libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem,
gdf_dtype)
libgdf.gdf_sum(col_data, unwrap_devary(d_result), d_result.size)
got = d_result.copy_to_host()[0]
expect = dtype(data.sum())
print('expect:', expect)
print('got:', got)
significant = 4 if dtype == np.float32 else 6
np.testing.assert_approx_equal(expect, got, significant=significant)
def _columnview(size, data, mask, dtype, null_count):
colview = ffi.new('gdf_column*')
extra_dtype_info = ffi.new('gdf_dtype_extra_info*')
extra_dtype_info.time_unit = libgdf.TIME_UNIT_NONE
if null_count is None:
libgdf.gdf_column_view(
colview,
data,
mask,
size,
np_to_gdf_dtype(dtype),
)
else:
libgdf.gdf_column_view_augmented(
colview,
data,
mask,
size,
np_to_gdf_dtype(dtype),
null_count,
extra_dtype_info[0],
)
return colview
def _columnview(size, data, mask, dtype):
colview = ffi.new('gdf_column*')
libgdf.gdf_column_view(colview, data, mask, size,
np_to_gdf_dtype(dtype))
return colview
