def _one_hot_encode_column(
column: ColumnBase,
categories: ColumnBase,
prefix: Optional[str],
prefix_sep: Optional[str],
dtype: Optional[Dtype],
) -> Dict[str, ColumnBase]:
"""Encode a single column with one hot encoding. The return dictionary
contains pairs of (category, encodings). The keys may be prefixed with
`prefix`, separated with category name with `prefix_sep`. The encoding
columns maybe coerced into `dtype`.
"""
if isinstance(column, CategoricalColumn):
if column.size == column.null_count:
column = column_empty_like(categories, newsize=column.size)
else:
column = column._get_decategorized_column()
if column.size * categories.size >= np.iinfo("int32").max:
raise ValueError(
"Size limitation exceeded: column.size * category.size < "
"np.iinfo('int32').max. Consider reducing size of category")
data = one_hot_encode(column, categories)
if prefix is not None and prefix_sep is not None:
data = {f"{prefix}{prefix_sep}{col}": enc for col, enc in data.items()}
if dtype:
data = {k: v.astype(dtype) for k, v in data.items()}
return data
def apply_boolean_mask(self, mask):
mask = as_column(mask, dtype="bool")
data = libcudf.stream_compaction.apply_boolean_mask([self], mask)
if not data:
return column_empty_like(self, newsize=0)
else:
result = data[0]
return result
def dropna(self):
dropped_col = libcudf.stream_compaction.drop_nulls([self])
if not dropped_col:
return column_empty_like(self, newsize=0)
else:
dropped_col = dropped_col[0]
dropped_col.mask = None
return dropped_col
def column_empty_like_same_mask(column, dtype):
"""Create a new empty Column with the same length and the same mask.
Parameters
----------
dtype : np.dtype like
The dtype of the data buffer.
"""
result = column_empty_like(column, dtype)
if column.nullable:
result = result.set_mask(column.mask)
return result
def time_col_replace_nulls(input_col):
null = column.column_empty_like(input_col, masked=True, newsize=1)
out_col = cudf._lib.replace.replace(
input_col,
column.as_column(
Buffer(np.array([np.datetime64("NaT")], dtype=input_col.dtype)),
dtype=input_col.dtype,
),
null,
)
return out_col
def time_col_replace_nulls(input_col):
null = column.column_empty_like(input_col, masked=True, newsize=1)
out_col = cudf._lib.replace.replace(
input_col,
column.as_column(
Buffer(
np.array([input_col.default_na_value()],
dtype=input_col.dtype).view("|u1")),
dtype=input_col.dtype,
),
null,
)
return out_col
def time_col_replace_nulls(input_col):
from cudf.core.column import column_empty_like, as_column
import cudf._libxx.replace as replace
null = column_empty_like(input_col, masked=True, newsize=1)
out_col = replace.replace(
input_col,
as_column(
Buffer(np.array([np.datetime64("NaT")], dtype=input_col.dtype)),
dtype=input_col.dtype,
),
null,
)
return out_col
def _make_copy_with_na_as_null(self):
"""Return a copy with NaN values replaced with nulls."""
null = column_empty_like(self, masked=True, newsize=1)
out_col = cudf._lib.replace.replace(
self,
as_column(
Buffer(
np.array([self.default_na_value()],
dtype=self.dtype).view("|u1")),
dtype=self.dtype,
),
null,
)
return out_col
def column_empty_like(column, dtype=None, masked=False, newsize=None):
"""Allocate a new column like the given *column*
"""
if dtype is None:
dtype = column.dtype
row_count = len(column) if newsize is None else newsize
if (hasattr(column, "dtype") and is_categorical_dtype(column.dtype)
and dtype == column.dtype):
codes = column_empty_like(column.codes, masked=masked, newsize=newsize)
return build_column(data=None,
dtype=dtype,
mask=codes.mask,
children=(codes, ))
return column_empty(row_count, dtype, masked)
def take(self, indices, ignore_index=False):
"""Return Column by taking values from the corresponding *indices*.
"""
from cudf.core.column import column_empty_like
# Handle zero size
if indices.size == 0:
return column_empty_like(self, newsize=0)
try:
result = libcudf.copying.gather(self, indices)
except RuntimeError as e:
if "out of bounds" in str(e):
raise IndexError(
f"index out of bounds for column of size {len(self)}")
raise
return result
def as_column(arbitrary, nan_as_null=True, dtype=None, length=None):
"""Create a Column from an arbitrary object
Parameters
----------
arbitrary : object
Object to construct the Column from. See *Notes*.
nan_as_null : bool,optional
If True (default), treat NaN values in arbitrary as null.
dtype : optional
Optionally typecast the construted Column to the given
dtype.
length : int, optional
If `arbitrary` is a scalar, broadcast into a Column of
the given length.
Returns
-------
A Column of the appropriate type and size.
Notes
-----
Currently support inputs are:
* ``Column``
* ``Series``
* ``Index``
* Scalars (can be broadcasted to a specified `length`)
* Objects exposing ``__cuda_array_interface__`` (e.g., numba device arrays)
* Objects exposing ``__array_interface__``(e.g., numpy arrays)
* pyarrow array
* pandas.Categorical objects
"""
from cudf.core.column import numerical, categorical, datetime, string
from cudf.core.series import Series
from cudf.core.index import Index
if isinstance(arbitrary, ColumnBase):
if dtype is not None:
return arbitrary.astype(dtype)
else:
return arbitrary
elif isinstance(arbitrary, Series):
data = arbitrary._column
if dtype is not None:
data = data.astype(dtype)
elif isinstance(arbitrary, Index):
data = arbitrary._values
if dtype is not None:
data = data.astype(dtype)
elif isinstance(arbitrary, nvstrings.nvstrings):
byte_count = arbitrary.byte_count()
if byte_count > libcudfxx.MAX_STRING_COLUMN_BYTES:
raise MemoryError(
"Cannot construct string columns "
"containing > {} bytes. "
"Consider using dask_cudf to partition "
"your data.".format(libcudfxx.MAX_STRING_COLUMN_BYTES_STR)
)
sbuf = Buffer.empty(arbitrary.byte_count())
obuf = Buffer.empty(
(arbitrary.size() + 1) * np.dtype("int32").itemsize
)
nbuf = None
if arbitrary.null_count() > 0:
mask_size = calc_chunk_size(arbitrary.size(), mask_bitsize)
nbuf = Buffer.empty(mask_size)
arbitrary.set_null_bitmask(nbuf.ptr, bdevmem=True)
arbitrary.to_offsets(sbuf.ptr, obuf.ptr, None, bdevmem=True)
children = (
build_column(obuf, dtype="int32"),
build_column(sbuf, dtype="int8"),
)
data = build_column(
data=None, dtype="object", mask=nbuf, children=children
)
data._nvstrings = arbitrary
elif isinstance(arbitrary, Buffer):
if dtype is None:
raise TypeError(f"dtype cannot be None if 'arbitrary' is a Buffer")
data = build_column(arbitrary, dtype=dtype)
elif cuda.devicearray.is_cuda_ndarray(arbitrary):
data = as_column(Buffer(arbitrary), dtype=arbitrary.dtype)
if (
data.dtype in [np.float16, np.float32, np.float64]
and arbitrary.size > 0
):
if nan_as_null:
mask = libcudf.unaryops.nans_to_nulls(data)
data = data.set_mask(mask)
elif data.dtype.kind == "M":
null = column_empty_like(data, masked=True, newsize=1)
col = libcudf.replace.replace(
as_column(Buffer(arbitrary), dtype=arbitrary.dtype),
as_column(
Buffer(np.array([np.datetime64("NaT")], dtype=data.dtype)),
dtype=arbitrary.dtype,
),
null,
)
data = datetime.DatetimeColumn(
data=Buffer(arbitrary), dtype=data.dtype, mask=col.mask
)
elif hasattr(arbitrary, "__cuda_array_interface__"):
desc = arbitrary.__cuda_array_interface__
data = _data_from_cuda_array_interface_desc(arbitrary)
mask = _mask_from_cuda_array_interface_desc(arbitrary)
dtype = np.dtype(desc["typestr"])
col = build_column(data, dtype=dtype, mask=mask)
return col
elif isinstance(arbitrary, np.ndarray):
# CUDF assumes values are always contiguous
if not arbitrary.flags["C_CONTIGUOUS"]:
arbitrary = np.ascontiguousarray(arbitrary)
if dtype is not None:
arbitrary = arbitrary.astype(dtype)
if arbitrary.dtype.kind == "M":
data = datetime.DatetimeColumn.from_numpy(arbitrary)
elif arbitrary.dtype.kind in ("O", "U"):
data = as_column(pa.Array.from_pandas(arbitrary))
else:
data = as_column(rmm.to_device(arbitrary), nan_as_null=nan_as_null)
elif isinstance(arbitrary, pa.Array):
if isinstance(arbitrary, pa.StringArray):
pa_size, pa_offset, nbuf, obuf, sbuf = buffers_from_pyarrow(
arbitrary
)
children = (
build_column(data=obuf, dtype="int32"),
build_column(data=sbuf, dtype="int8"),
)
data = string.StringColumn(
mask=nbuf, children=children, size=pa_size, offset=pa_offset
)
elif isinstance(arbitrary, pa.NullArray):
new_dtype = pd.api.types.pandas_dtype(dtype)
if (type(dtype) == str and dtype == "empty") or dtype is None:
new_dtype = pd.api.types.pandas_dtype(
arbitrary.type.to_pandas_dtype()
)
if is_categorical_dtype(new_dtype):
arbitrary = arbitrary.dictionary_encode()
else:
if nan_as_null:
arbitrary = arbitrary.cast(np_to_pa_dtype(new_dtype))
else:
# casting a null array doesn't make nans valid
# so we create one with valid nans from scratch:
if new_dtype == np.dtype("object"):
arbitrary = utils.scalar_broadcast_to(
None, (len(arbitrary),), dtype=new_dtype
)
else:
arbitrary = utils.scalar_broadcast_to(
np.nan, (len(arbitrary),), dtype=new_dtype
)
data = as_column(arbitrary, nan_as_null=nan_as_null)
elif isinstance(arbitrary, pa.DictionaryArray):
codes = as_column(arbitrary.indices)
if isinstance(arbitrary.dictionary, pa.NullArray):
categories = as_column([], dtype="object")
else:
categories = as_column(arbitrary.dictionary)
dtype = CategoricalDtype(
categories=categories, ordered=arbitrary.type.ordered
)
data = categorical.CategoricalColumn(
dtype=dtype,
mask=codes.base_mask,
children=(codes,),
size=codes.size,
offset=codes.offset,
)
elif isinstance(arbitrary, pa.TimestampArray):
dtype = np.dtype("M8[{}]".format(arbitrary.type.unit))
pa_size, pa_offset, pamask, padata, _ = buffers_from_pyarrow(
arbitrary, dtype=dtype
)
data = datetime.DatetimeColumn(
data=padata,
mask=pamask,
dtype=dtype,
size=pa_size,
offset=pa_offset,
)
elif isinstance(arbitrary, pa.Date64Array):
raise NotImplementedError
pa_size, pa_offset, pamask, padata, _ = buffers_from_pyarrow(
arbitrary, dtype="M8[ms]"
)
data = datetime.DatetimeColumn(
data=padata,
mask=pamask,
dtype=np.dtype("M8[ms]"),
size=pa_size,
offset=pa_offset,
)
elif isinstance(arbitrary, pa.Date32Array):
# No equivalent np dtype and not yet supported
warnings.warn(
"Date32 values are not yet supported so this will "
"be typecast to a Date64 value",
UserWarning,
)
data = as_column(arbitrary.cast(pa.int32())).astype("M8[ms]")
elif isinstance(arbitrary, pa.BooleanArray):
# Arrow uses 1 bit per value while we use int8
dtype = np.dtype(np.bool)
# Needed because of bug in PyArrow
# https://issues.apache.org/jira/browse/ARROW-4766
if len(arbitrary) > 0:
arbitrary = arbitrary.cast(pa.int8())
else:
arbitrary = pa.array([], type=pa.int8())
pa_size, pa_offset, pamask, padata, _ = buffers_from_pyarrow(
arbitrary, dtype=dtype
)
data = numerical.NumericalColumn(
data=padata,
mask=pamask,
dtype=dtype,
size=pa_size,
offset=pa_offset,
)
else:
pa_size, pa_offset, pamask, padata, _ = buffers_from_pyarrow(
arbitrary
)
data = numerical.NumericalColumn(
data=padata,
dtype=np.dtype(arbitrary.type.to_pandas_dtype()),
mask=pamask,
size=pa_size,
offset=pa_offset,
)
elif isinstance(arbitrary, pa.ChunkedArray):
gpu_cols = [
as_column(chunk, dtype=dtype) for chunk in arbitrary.chunks
]
if dtype and dtype != "empty":
new_dtype = dtype
else:
pa_type = arbitrary.type
if pa.types.is_dictionary(pa_type):
new_dtype = "category"
else:
new_dtype = np.dtype(pa_type.to_pandas_dtype())
data = ColumnBase._concat(gpu_cols, dtype=new_dtype)
elif isinstance(arbitrary, (pd.Series, pd.Categorical)):
if is_categorical_dtype(arbitrary):
data = as_column(pa.array(arbitrary, from_pandas=True))
elif arbitrary.dtype == np.bool:
# Bug in PyArrow or HDF that requires us to do this
data = as_column(pa.array(np.asarray(arbitrary), from_pandas=True))
else:
data = as_column(pa.array(arbitrary, from_pandas=nan_as_null))
elif isinstance(arbitrary, pd.Timestamp):
# This will always treat NaTs as nulls since it's not technically a
# discrete value like NaN
data = as_column(pa.array(pd.Series([arbitrary]), from_pandas=True))
elif np.isscalar(arbitrary) and not isinstance(arbitrary, memoryview):
length = length or 1
data = as_column(
utils.scalar_broadcast_to(arbitrary, length, dtype=dtype)
)
if not nan_as_null:
data = data.fillna(np.nan)
elif isinstance(arbitrary, memoryview):
data = as_column(
np.asarray(arbitrary), dtype=dtype, nan_as_null=nan_as_null
)
else:
try:
data = as_column(
memoryview(arbitrary), dtype=dtype, nan_as_null=nan_as_null
)
except TypeError:
pa_type = None
np_type = None
try:
if dtype is not None:
dtype = pd.api.types.pandas_dtype(dtype)
if is_categorical_dtype(dtype):
raise TypeError
else:
np_type = np.dtype(dtype).type
if np_type == np.bool_:
pa_type = pa.bool_()
else:
pa_type = np_to_pa_dtype(np.dtype(dtype))
data = as_column(
pa.array(arbitrary, type=pa_type, from_pandas=nan_as_null),
dtype=dtype,
nan_as_null=nan_as_null,
)
except (pa.ArrowInvalid, pa.ArrowTypeError, TypeError):
if is_categorical_dtype(dtype):
sr = pd.Series(arbitrary, dtype="category")
data = as_column(sr, nan_as_null=nan_as_null)
elif np_type == np.str_:
sr = pd.Series(arbitrary, dtype="str")
data = as_column(sr, nan_as_null=nan_as_null)
else:
data = as_column(
np.asarray(arbitrary, dtype=np.dtype(dtype)),
nan_as_null=nan_as_null,
)
return data
def _concat(cls, objs, dtype=None):
from cudf.core.series import Series
from cudf.core.column import (
StringColumn,
CategoricalColumn,
NumericalColumn,
)
if len(objs) == 0:
dtype = pd.api.types.pandas_dtype(dtype)
if is_categorical_dtype(dtype):
dtype = CategoricalDtype()
return column_empty(0, dtype=dtype, masked=True)
# If all columns are `NumericalColumn` with different dtypes,
# we cast them to a common dtype.
# Notice, we can always cast pure null columns
not_null_cols = list(filter(lambda o: len(o) != o.null_count, objs))
if len(not_null_cols) > 0 and (
len(
[
o
for o in not_null_cols
if not isinstance(o, NumericalColumn)
or np.issubdtype(o.dtype, np.datetime64)
]
)
== 0
):
col_dtypes = [o.dtype for o in not_null_cols]
# Use NumPy to find a common dtype
common_dtype = np.find_common_type(col_dtypes, [])
# Cast all columns to the common dtype
for i in range(len(objs)):
objs[i] = objs[i].astype(common_dtype)
# Find the first non-null column:
head = objs[0]
for i, obj in enumerate(objs):
if len(obj) != obj.null_count:
head = obj
break
for i, obj in enumerate(objs):
# Check that all columns are the same type:
if not pd.api.types.is_dtype_equal(objs[i].dtype, head.dtype):
# if all null, cast to appropriate dtype
if len(obj) == obj.null_count:
from cudf.core.column import column_empty_like
objs[i] = column_empty_like(
head, dtype=head.dtype, masked=True, newsize=len(obj)
)
# Handle categories for categoricals
if all(isinstance(o, CategoricalColumn) for o in objs):
cats = (
Series(ColumnBase._concat([o.categories for o in objs]))
.drop_duplicates()
._column
)
objs = [
o.cat()._set_categories(cats, is_unique=True) for o in objs
]
head = objs[0]
for obj in objs:
if not (obj.dtype == head.dtype):
raise ValueError("All series must be of same type")
newsize = sum(map(len, objs))
if newsize > libcudfxx.MAX_COLUMN_SIZE:
raise MemoryError(
"Result of concat cannot have "
"size > {}".format(libcudfxx.MAX_COLUMN_SIZE_STR)
)
# Handle strings separately
if all(isinstance(o, StringColumn) for o in objs):
result_nbytes = sum(o._nbytes for o in objs)
if result_nbytes > libcudfxx.MAX_STRING_COLUMN_BYTES:
raise MemoryError(
"Result of concat cannot have > {} bytes".format(
libcudfxx.MAX_STRING_COLUMN_BYTES_STR
)
)
objs = [o.nvstrings for o in objs]
return as_column(nvstrings.from_strings(*objs))
# Filter out inputs that have 0 length
objs = [o for o in objs if len(o) > 0]
nulls = any(col.nullable for col in objs)
if is_categorical_dtype(head):
data_dtype = head.codes.dtype
data = None
children = (column_empty(newsize, dtype=head.codes.dtype),)
else:
data_dtype = head.dtype
data = Buffer.empty(size=newsize * data_dtype.itemsize)
children = ()
# Allocate output mask only if there's nulls in the input objects
mask = None
if nulls:
mask = Buffer(utils.make_mask(newsize))
col = build_column(
data=data, dtype=head.dtype, mask=mask, children=children
)
# Performance the actual concatenation
if newsize > 0:
col = libcudf.concat._column_concat(objs, col)
return col
