def test_sum_of_squares(dtype, nelem): data = gen_rand(dtype, nelem) d_data = rmm.to_device(data) d_result = rmm.device_array(libgdf.gdf_reduce_optimal_output_size(), dtype=d_data.dtype) col_data = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_sum_of_squares(col_data, unwrap_devary(d_result), d_result.size) got = d_result.copy_to_host()[0] expect = (data ** 2).sum() print('expect:', expect) print('got:', got) if np.dtype(dtype).kind == 'i': if 0 <= expect <= np.iinfo(dtype).max: np.testing.assert_array_almost_equal(expect, got) else: print('overflow, passing') else: np.testing.assert_array_almost_equal(expect, got, decimal=accuracy_for_dtype[dtype])
def test_prefixsum(dtype, nelem): if dtype == np.int8: # to keep data in range data = gen_rand(dtype, nelem, low=-2, high=2) else: data = gen_rand(dtype, nelem) d_data = rmm.to_device(data) d_result = rmm.device_array(d_data.size, dtype=d_data.dtype) col_data = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem, gdf_dtype) col_result = new_column() libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL, nelem, gdf_dtype) inclusive = True libgdf.gdf_prefixsum_generic(col_data, col_result, inclusive) expect = np.cumsum(d_data.copy_to_host()) got = d_result.copy_to_host() if not inclusive: expect = expect[:-1] assert got[0] == 0 got = got[1:] np.testing.assert_array_equal(expect, got)
def cast_op_test(dtype, to_dtype, test_fn, nelem=128): h_data = gen_rand(dtype, nelem).astype(dtype) d_data = cuda.to_device(h_data) d_result = cuda.device_array(d_data.size, dtype=to_dtype) assert d_data.dtype == dtype assert d_result.dtype == to_dtype col_data = new_column() col_result = new_column() # data column libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem, get_dtype(dtype)) # result column libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL, nelem, get_dtype(to_dtype)) expect = h_data.astype(to_dtype) test_fn(col_data, col_result) got = d_result.copy_to_host() print('got') print(got) print('expect') print(expect) np.testing.assert_equal(expect, got)
def math_op_test(dtype, ulp, expect_fn, test_fn, nelem=128, scale=1): h_data = (gen_rand(dtype, nelem) * scale).astype(dtype) d_data = cuda.to_device(h_data) d_result = cuda.device_array_like(d_data) col_data = new_column() col_result = new_column() gdf_dtype = get_dtype(dtype) # data column libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem, gdf_dtype) # result column libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL, nelem, gdf_dtype) expect = expect_fn(h_data) test_fn(col_data, col_result) got = d_result.copy_to_host() print('got') print(got) print('expect') print(expect) np.testing.assert_array_max_ulp(expect, got, maxulp=ulp)
def test_product(dtype, nelem): if np.dtype(dtype).kind == 'i': data = np.ones(nelem, dtype=dtype) # Set at most 30 items to [0..2) to keep the value within 2^32 for _ in range(30): data[random.randrange(nelem)] = random.random() * 2 else: data = gen_rand(dtype, nelem) print('max', data.max(), 'min', data.min()) d_data = cuda.to_device(data) d_result = cuda.device_array(libgdf.gdf_reduce_optimal_output_size(), dtype=d_data.dtype) col_data = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_product_generic(col_data, unwrap_devary(d_result), d_result.size) got = d_result.copy_to_host()[0] expect = np.product(data) print('expect:', expect) print('got:', got) np.testing.assert_array_almost_equal(expect, got)
def _call_hash_multi(api, ncols, col_input, magic, nrows): out_ary = np.zeros(nrows, dtype=np.int32) d_out = cuda.to_device(out_ary) col_out = new_column() libgdf.gdf_column_view(col_out, unwrap_devary(d_out), ffi.NULL, out_ary.size, get_dtype(d_out.dtype)) api(ncols, col_input, magic, col_out) dataptr = col_out.data print(dataptr) datasize = col_out.size print(datasize) addr = ctypes.c_uint64(int(ffi.cast("uintptr_t", dataptr))) print(hex(addr.value)) memptr = cuda.driver.MemoryPointer(context=cuda.current_context(), pointer=addr, size=4 * datasize) print(memptr) ary = cuda.devicearray.DeviceNDArray(shape=(datasize, ), strides=(4, ), dtype=np.dtype(np.int32), gpu_data=memptr) hashed_result = ary.copy_to_host() print(hashed_result) return hashed_result
def columnview(size, data, mask=None, dtype=None): """ Make a column view. Parameters ---------- size : int Data count. data : Buffer The data buffer. mask : Buffer; optional The mask buffer. dtype : numpy.dtype; optional The dtype of the data. Defaults to *data.dtype*. """ def unwrap(buffer): if buffer is None: return ffi.NULL assert buffer.mem.is_c_contiguous(), "libGDF expects contiguous memory" devary = buffer.to_gpu_array() return unwrap_devary(devary) dtype = dtype or data.dtype colview = ffi.new('gdf_column*') libgdf.gdf_column_view(colview, unwrap(data), unwrap(mask), size, np_to_gdf_dtype(dtype)) return colview
def test_cuda_error(): dtype = np.float32 col = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col, ffi.NULL, ffi.NULL, 0, gdf_dtype)
def _make_hash_input(hash_input, ncols): ci = [] for i in range(ncols): d_input = cuda.to_device(hash_input[i]) col_input = new_column() libgdf.gdf_column_view(col_input, unwrap_devary(d_input), ffi.NULL, hash_input[i].size, get_dtype(d_input.dtype)) ci.append(col_input) yield ci
def test_radixsort(nelem, descending, dtype): def expected_fn(key): # Use mergesort for stable sort # Negate the key for descending if issubclass(dtype, np.integer): def negate_values(v): return ~key else: # Note: this doesn't work on the smallest value of integer # i.e. -((int8)-128) -> -128 def negate_values(v): return -key sorted_idx = np.argsort(negate_values(key) if descending else key, kind='mergesort') sorted_keys = key[sorted_idx] # Returns key, vals return sorted_keys, sorted_idx # Make data key = gen_rand(dtype, nelem) d_key = rmm.to_device(key) col_key = new_column() libgdf.gdf_column_view(col_key, unwrap_devary(d_key), ffi.NULL, nelem, get_dtype(d_key.dtype)) val = np.arange(nelem, dtype=np.int64) d_val = rmm.to_device(val) col_val = new_column() libgdf.gdf_column_view(col_val, unwrap_devary(d_val), ffi.NULL, nelem, get_dtype(d_val.dtype)) sizeof_key = d_key.dtype.itemsize sizeof_val = d_val.dtype.itemsize begin_bit = 0 end_bit = sizeof_key * 8 # Setup plan plan = libgdf.gdf_radixsort_plan(nelem, descending, begin_bit, end_bit) libgdf.gdf_radixsort_plan_setup(plan, sizeof_key, sizeof_val) # Sort libgdf.gdf_radixsort_generic(plan, col_key, col_val) # Cleanup libgdf.gdf_radixsort_plan_free(plan) # Check got_keys = d_key.copy_to_host() got_vals = d_val.copy_to_host() sorted_keys, sorted_vals = expected_fn(key) np.testing.assert_array_equal(sorted_keys, got_keys) np.testing.assert_array_equal(sorted_vals, got_vals)
def test_cuda_error(): dtype = np.float32 col = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col, ffi.NULL, ffi.NULL, 0, gdf_dtype) with pytest.raises(GDFError) as raises: libgdf.gdf_add_generic(col, col, col) raises.match("CUDA ERROR.")
def _make_input(left, right): d_left = cuda.to_device(left) col_left = new_column() libgdf.gdf_column_view(col_left, unwrap_devary(d_left), ffi.NULL, left.size, get_dtype(d_left.dtype)) d_right = cuda.to_device(right) col_right = new_column() libgdf.gdf_column_view(col_right, unwrap_devary(d_right), ffi.NULL, right.size, get_dtype(d_right.dtype)) yield col_left, col_right
def _call_hash_multi(api, ncols, col_input, magic, nrows): out_ary = np.zeros(nrows, dtype=np.int32) d_out = rmm.to_device(out_ary) col_out = new_column() libgdf.gdf_column_view(col_out, unwrap_devary(d_out), ffi.NULL, out_ary.size, get_dtype(d_out.dtype)) api(ncols, col_input, magic, col_out) hashed_result = d_out.copy_to_host() print(hashed_result) return hashed_result
def columnview(size, data, mask=None, dtype=None): """ Make a column view. """ def unwrap(buffer): if buffer is None: return ffi.NULL devary = buffer.to_gpu_array() return ffi.cast('void*', devary.device_ctypes_pointer.value) dtype = dtype or data.dtype colview = ffi.new('gdf_column*') libgdf.gdf_column_view(colview, unwrap(data), unwrap(mask), size, np_to_gdf_dtype(dtype)) return colview
def columnview(size, data, mask=None, dtype=None): """ Make a column view. """ def unwrap(buffer): if buffer is None: return ffi.NULL devary = buffer.to_gpu_array() return unwrap_devary(devary) dtype = dtype or data.dtype colview = ffi.new('gdf_column*') libgdf.gdf_column_view(colview, unwrap(data), unwrap(mask), size, np_to_gdf_dtype(dtype)) return colview
def _make_hash_input(hash_input, ncols): ci = [] di = [] for i in range(ncols): di.append(rmm.to_device(hash_input[i])) for i in range(ncols): col_input = new_column() libgdf.gdf_column_view(col_input, unwrap_devary(di[i]), ffi.NULL, hash_input[i].size, get_dtype(hash_input[i].dtype)) ci.append(col_input) initial_hash_values = rmm.to_device(np.arange(ncols, dtype=np.uint32)) yield ci, unwrap_devary(initial_hash_values)
def _make_input_multi(left, right, ncols): cl = [] cr = [] for i in range(ncols): d_left = cuda.to_device(left[i]) col_left = new_column() libgdf.gdf_column_view(col_left, unwrap_devary(d_left), ffi.NULL, left[i].size, get_dtype(d_left.dtype)) cl.append(col_left) d_right = cuda.to_device(right[i]) col_right = new_column() libgdf.gdf_column_view(col_right, unwrap_devary(d_right), ffi.NULL, right[i].size, get_dtype(d_right.dtype)) cr.append(col_right) yield cl, cr
def test_unsupported_dtype_error(): nelem = 128 h_data = np.random.random(nelem).astype(np.float32) d_data = cuda.to_device(h_data) d_result = cuda.device_array_like(d_data) col_data = new_column() col_result = new_column() libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem, libgdf.GDF_INT32) libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL, nelem + 10, libgdf.GDF_FLOAT32) with pytest.raises(GDFError) as excinfo: libgdf.gdf_sin_generic(col_data, col_result) assert 'GDF_UNSUPPORTED_DTYPE' == str(excinfo.value)
def test_col_mismatch_error(): nelem = 128 h_data = np.random.random(nelem).astype(np.float32) d_data = cuda.to_device(h_data) d_result = cuda.device_array_like(d_data) col_data = new_column() col_result = new_column() libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem, libgdf.GDF_FLOAT32) libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL, nelem + 10, libgdf.GDF_FLOAT32) with pytest.raises(GDFError) as excinfo: libgdf.gdf_sin_generic(col_data, col_result) assert 'GDF_COLUMN_SIZE_MISMATCH' == str(excinfo.value)
def test_sum(dtype, nelem): data = gen_rand(dtype, nelem) d_data = cuda.to_device(data) d_result = cuda.device_array(libgdf.gdf_reduce_optimal_output_size(), dtype=d_data.dtype) col_data = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_sum_generic(col_data, unwrap_devary(d_result), d_result.size) got = d_result.copy_to_host()[0] expect = data.sum() print('expect:', expect) print('got:', got) np.testing.assert_array_almost_equal(expect, got)
def _columnview(size, data, mask, dtype, null_count): colview = ffi.new('gdf_column*') if null_count is None: libgdf.gdf_column_view( colview, data, mask, size, np_to_gdf_dtype(dtype), ) else: libgdf.gdf_column_view_augmented( colview, data, mask, size, np_to_gdf_dtype(dtype), null_count, ) return colview
def test_max(dtype, nelem): data = gen_rand(dtype, nelem) d_data = rmm.to_device(data) d_result = rmm.device_array(libgdf.gdf_reduce_optimal_output_size(), dtype=d_data.dtype) col_data = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_max(col_data, unwrap_devary(d_result), d_result.size) got = d_result.copy_to_host()[0] expect = data.max() print('expect:', expect) print('got:', got) assert expect == got
def test_prefixsum_masked(dtype, nelem): if dtype == np.int8: data = gen_rand(dtype, nelem, low=-2, high=2) else: data = gen_rand(dtype, nelem) mask = gen_rand(np.int8, (nelem + 8 - 1) // 8) dummy_mask = gen_rand(np.int8, (nelem + 8 - 1) // 8) d_data = rmm.to_device(data) d_mask = rmm.to_device(mask) d_result = rmm.device_array(d_data.size, dtype=d_data.dtype) d_result_mask = rmm.to_device(dummy_mask) gdf_dtype = get_dtype(dtype) extra_dtype_info = ffi.new('gdf_dtype_extra_info*') extra_dtype_info.time_unit = libgdf.TIME_UNIT_NONE col_data = new_column() libgdf.gdf_column_view_augmented(col_data, unwrap_devary(d_data), unwrap_devary(d_mask), nelem, gdf_dtype, count_nulls(d_mask, nelem), extra_dtype_info[0]) col_result = new_column() libgdf.gdf_column_view(col_result, unwrap_devary(d_result), unwrap_devary(d_result_mask), nelem, gdf_dtype) inclusive = True libgdf.gdf_prefixsum(col_data, col_result, inclusive) boolmask = buffer_as_bits(mask)[:nelem] expect = np.cumsum(data[boolmask]) got = d_result.copy_to_host()[boolmask] if not inclusive: expect = expect[:-1] assert got[0] == 0 got = got[1:] decimal = 4 if dtype == np.float32 else 6 np.testing.assert_array_almost_equal(expect, got, decimal=decimal)
def test_sum_masked(nelem): dtype = np.float64 data = gen_rand(dtype, nelem) mask = gen_rand(np.int8, (nelem + 8 - 1) // 8) d_data = cuda.to_device(data) d_mask = cuda.to_device(mask) d_result = cuda.device_array(libgdf.gdf_reduce_optimal_output_size(), dtype=d_data.dtype) col_data = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col_data, unwrap_devary(d_data), unwrap_devary(d_mask), nelem, gdf_dtype) libgdf.gdf_sum_generic(col_data, unwrap_devary(d_result), d_result.size) got = d_result.copy_to_host()[0] boolmask = buffer_as_bits(mask)[:nelem] expect = data[boolmask].sum() np.testing.assert_almost_equal(expect, got)
def arith_op_test(dtype, ulp, expect_fn, test_fn, nelem=128, non_zero_rhs=False): h_lhs = gen_rand(dtype, nelem) h_rhs = gen_rand(dtype, nelem) if non_zero_rhs: fix_zeros(h_rhs) d_lhs = rmm.to_device(h_lhs) d_rhs = rmm.to_device(h_rhs) d_result = rmm.device_array_like(d_lhs) col_lhs = new_column() col_rhs = new_column() col_result = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL, nelem, gdf_dtype) expect = expect_fn(h_lhs, h_rhs) test_fn(col_lhs, col_rhs, col_result) got = d_result.copy_to_host() print('got') print(got) print('expect') print(expect) np.testing.assert_array_max_ulp(expect, got, maxulp=ulp)
def bitwise_op_test(dtype, expect_fn, test_fn, nelem=128): h_lhs = gen_rand(dtype, nelem) h_rhs = gen_rand(dtype, nelem) d_lhs = rmm.to_device(h_lhs) d_rhs = rmm.to_device(h_rhs) d_result = rmm.device_array_like(d_lhs) col_lhs = new_column() col_rhs = new_column() col_result = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL, nelem, gdf_dtype) expect = expect_fn(h_lhs, h_rhs) test_fn(col_lhs, col_rhs, col_result) got = d_result.copy_to_host() print('got') print(got) print('expect') print(expect) np.testing.assert_array_equal(expect, got)
def test_output_dtype_mismatch(): lhs_dtype = np.int32 rhs_dtype = np.int32 nelem = 5 h_lhs = np.arange(nelem, dtype=lhs_dtype) h_rhs = np.arange(nelem, dtype=rhs_dtype) d_lhs = rmm.to_device(h_lhs) d_rhs = rmm.to_device(h_rhs) d_result = rmm.device_array(d_lhs.size, dtype=np.float32) col_lhs = new_column() col_rhs = new_column() col_result = new_column() libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem, get_dtype(lhs_dtype)) libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem, get_dtype(rhs_dtype)) libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL, nelem, get_dtype(d_result.dtype)) with pytest.raises(GDFError) as raises: libgdf.gdf_add_generic(col_lhs, col_rhs, col_result) raises.match("GDF_UNSUPPORTED_DTYPE") with pytest.raises(GDFError) as raises: libgdf.gdf_eq_generic(col_lhs, col_rhs, col_result) raises.match("GDF_UNSUPPORTED_DTYPE") with pytest.raises(GDFError) as raises: libgdf.gdf_bitwise_and_generic(col_lhs, col_rhs, col_result) raises.match("GDF_UNSUPPORTED_DTYPE")
def logical_op_test(dtype, expect_fn, test_fn, nelem=128, gdf_dtype=None): h_lhs = gen_rand(dtype, nelem) h_rhs = gen_rand(dtype, nelem) d_lhs = rmm.to_device(h_lhs) d_rhs = rmm.to_device(h_rhs) d_result = rmm.device_array(d_lhs.size, dtype=np.bool) col_lhs = new_column() col_rhs = new_column() col_result = new_column() gdf_dtype = get_dtype(dtype) if gdf_dtype is None else gdf_dtype libgdf.gdf_column_view(col_lhs, unwrap_devary(d_lhs), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_column_view(col_rhs, unwrap_devary(d_rhs), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_column_view(col_result, unwrap_devary(d_result), ffi.NULL, nelem, libgdf.GDF_INT8) expect = expect_fn(h_lhs, h_rhs) test_fn(col_lhs, col_rhs, col_result) got = d_result.copy_to_host() print(expect, got) np.testing.assert_equal(expect, got)
def test_sum(dtype, nelem): data = gen_rand(dtype, nelem) d_data = rmm.to_device(data) d_result = rmm.device_array( libgdf.gdf_reduction_get_intermediate_output_size(), dtype=d_data.dtype) col_data = new_column() gdf_dtype = get_dtype(dtype) libgdf.gdf_column_view(col_data, unwrap_devary(d_data), ffi.NULL, nelem, gdf_dtype) libgdf.gdf_sum(col_data, unwrap_devary(d_result), d_result.size) got = d_result.copy_to_host()[0] expect = dtype(data.sum()) print('expect:', expect) print('got:', got) significant = 4 if dtype == np.float32 else 6 np.testing.assert_approx_equal(expect, got, significant=significant)
def _columnview(size, data, mask, dtype, null_count): colview = ffi.new('gdf_column*') extra_dtype_info = ffi.new('gdf_dtype_extra_info*') extra_dtype_info.time_unit = libgdf.TIME_UNIT_NONE if null_count is None: libgdf.gdf_column_view( colview, data, mask, size, np_to_gdf_dtype(dtype), ) else: libgdf.gdf_column_view_augmented( colview, data, mask, size, np_to_gdf_dtype(dtype), null_count, extra_dtype_info[0], ) return colview
def _columnview(size, data, mask, dtype): colview = ffi.new('gdf_column*') libgdf.gdf_column_view(colview, data, mask, size, np_to_gdf_dtype(dtype)) return colview