def dtype_can_compare_equal_to_other(dtype):
# return True if values of this dtype can compare
# as equal to equal values of a different dtype
return not (
is_string_dtype(dtype)
or is_list_dtype(dtype)
or is_struct_dtype(dtype)
or is_decimal_dtype(dtype)
or is_interval_dtype(dtype)
)
def _convert_str_col(col, errors, _downcast=None):
"""
Converts a string column to numeric column
Converts to integer column if all strings are integer-like (isinteger.all)
Otherwise, converts to float column if all strings are float-like (
isfloat.all)
If error == 'coerce', fill non-numerics strings with null
Looks ahead to ``downcast`` parameter, if the float may be casted to
integer, then only process in float32 pipeline.
Parameters
----------
col : The string column to convert, must be string dtype
errors : {'raise', 'ignore', 'coerce'}, same as ``to_numeric``
_downcast : Same as ``to_numeric``, see description for use
Returns
-------
Converted numeric column
"""
if not is_string_dtype(col):
raise TypeError("col must be string dtype.")
is_integer = libstrings.is_integer(col)
if is_integer.all():
return col.as_numerical_column(dtype=cudf.dtype("i8"))
col = _proc_inf_empty_strings(col)
is_float = libstrings.is_float(col)
if is_float.all():
if _downcast in {"unsigned", "signed", "integer"}:
warnings.warn(
UserWarning("Downcasting from float to int will be "
"limited by float32 precision."))
return col.as_numerical_column(dtype=cudf.dtype("f"))
else:
return col.as_numerical_column(dtype=cudf.dtype("d"))
else:
if errors == "coerce":
col = libcudf.string_casting.stod(col)
non_numerics = is_float.unary_operator("not")
col[non_numerics] = None
return col
else:
raise ValueError("Unable to convert some strings to numerics.")
def to_cudf_compatible_scalar(val, dtype=None):
"""
Converts the value `val` to a numpy/Pandas scalar,
optionally casting to `dtype`.
If `val` is None, returns None.
"""
if cudf._lib.scalar._is_null_host_scalar(val) or isinstance(
val, cudf.Scalar):
return val
if not _is_scalar_or_zero_d_array(val):
raise ValueError(f"Cannot convert value of type {type(val).__name__} "
"to cudf scalar")
if isinstance(val, Decimal):
return val
if isinstance(val, (np.ndarray, cp.ndarray)) and val.ndim == 0:
val = val.item()
if ((dtype is None) and isinstance(val, str)) or is_string_dtype(dtype):
dtype = "str"
if isinstance(val, dt.datetime):
val = np.datetime64(val)
elif isinstance(val, dt.timedelta):
val = np.timedelta64(val)
elif isinstance(val, pd.Timestamp):
val = val.to_datetime64()
elif isinstance(val, pd.Timedelta):
val = val.to_timedelta64()
val = pandas_dtype(type(val)).type(val)
if dtype is not None:
val = val.astype(dtype)
if val.dtype.type is np.datetime64:
time_unit, _ = np.datetime_data(val.dtype)
if time_unit in ("D", "W", "M", "Y"):
val = val.astype("datetime64[s]")
elif val.dtype.type is np.timedelta64:
time_unit, _ = np.datetime_data(val.dtype)
if time_unit in ("D", "W", "M", "Y"):
val = val.astype("timedelta64[ns]")
return val
def _get_non_empty_data(s):
if isinstance(s._column, cudf.core.column.CategoricalColumn):
categories = (s._column.categories
if len(s._column.categories) else [UNKNOWN_CATEGORIES])
codes = cudf.core.column.full(size=2, fill_value=0, dtype="int32")
ordered = s._column.ordered
data = cudf.core.column.build_categorical_column(categories=categories,
codes=codes,
ordered=ordered)
elif is_string_dtype(s.dtype):
data = pa.array(["cat", "dog"])
else:
if pd.api.types.is_numeric_dtype(s.dtype):
data = cudf.core.column.as_column(
cp.arange(start=0, stop=2, dtype=s.dtype))
else:
data = cudf.core.column.as_column(
cp.arange(start=0, stop=2, dtype="int64")).astype(s.dtype)
return data
def test_is_string_dtype(obj, expect):
assert types.is_string_dtype(obj) == expect
def test_pandas_agreement(obj):
assert types.is_categorical_dtype(obj) == ptypes.is_categorical_dtype(obj)
assert types.is_numeric_dtype(obj) == ptypes.is_numeric_dtype(obj)
assert types.is_integer_dtype(obj) == ptypes.is_integer_dtype(obj)
assert types.is_integer(obj) == ptypes.is_integer(obj)
assert types.is_string_dtype(obj) == ptypes.is_string_dtype(obj)
def to_numeric(arg, errors="raise", downcast=None):
"""
Convert argument into numerical types.
Parameters
----------
arg : column-convertible
The object to convert to numeric types
errors : {'raise', 'ignore', 'coerce'}, defaults 'raise'
Policy to handle errors during parsing.
* 'raise' will notify user all errors encountered.
* 'ignore' will skip error and returns ``arg``.
* 'coerce' will leave invalid values as nulls.
downcast : {'integer', 'signed', 'unsigned', 'float'}, defaults None
If set, will try to down-convert the datatype of the
parsed results to smallest possible type. For each `downcast`
type, this method will determine the smallest possible
dtype from the following sets:
* {'integer', 'signed'}: all integer types greater or equal to
`np.int8`
* {'unsigned'}: all unsigned types greater or equal to `np.uint8`
* {'float'}: all floating types greater or equal to `np.float32`
Note that downcast behavior is decoupled from parsing. Errors
encountered during downcast is raised regardless of ``errors``
parameter.
Returns
-------
Series or ndarray
Depending on the input, if series is passed in, series is returned,
otherwise ndarray
Notes
-------
An important difference from pandas is that this function does not accept
mixed numeric/non-numeric type sequences. For example ``[1, 'a']``.
A ``TypeError`` will be raised when such input is received, regardless of
``errors`` parameter.
Examples
--------
>>> s = cudf.Series(['1', '2.0', '3e3'])
>>> cudf.to_numeric(s)
0 1.0
1 2.0
2 3000.0
dtype: float64
>>> cudf.to_numeric(s, downcast='float')
0 1.0
1 2.0
2 3000.0
dtype: float32
>>> cudf.to_numeric(s, downcast='signed')
0 1
1 2
2 3000
dtype: int16
>>> s = cudf.Series(['apple', '1.0', '3e3'])
>>> cudf.to_numeric(s, errors='ignore')
0 apple
1 1.0
2 3e3
dtype: object
>>> cudf.to_numeric(s, errors='coerce')
0 <NA>
1 1.0
2 3000.0
dtype: float64
"""
if errors not in {"raise", "ignore", "coerce"}:
raise ValueError("invalid error value specified")
if downcast not in {None, "integer", "signed", "unsigned", "float"}:
raise ValueError("invalid downcasting method provided")
if not can_convert_to_column(arg) or (hasattr(arg, "ndim")
and arg.ndim > 1):
raise ValueError("arg must be column convertible")
col = as_column(arg)
dtype = col.dtype
if is_datetime_dtype(dtype) or is_timedelta_dtype(dtype):
col = col.as_numerical_column(cudf.dtype("int64"))
elif is_categorical_dtype(dtype):
cat_dtype = col.dtype.type
if _is_non_decimal_numeric_dtype(cat_dtype):
col = col.as_numerical_column(cat_dtype)
else:
try:
col = _convert_str_col(col._get_decategorized_column(), errors,
downcast)
except ValueError as e:
if errors == "ignore":
return arg
else:
raise e
elif is_string_dtype(dtype):
try:
col = _convert_str_col(col, errors, downcast)
except ValueError as e:
if errors == "ignore":
return arg
else:
raise e
elif is_list_dtype(dtype) or is_struct_dtype(dtype):
raise ValueError("Input does not support nested datatypes")
elif _is_non_decimal_numeric_dtype(dtype):
pass
else:
raise ValueError("Unrecognized datatype")
# str->float conversion may require lower precision
if col.dtype == cudf.dtype("f"):
col = col.as_numerical_column("d")
if downcast:
downcast_type_map = {
"integer": list(np.typecodes["Integer"]),
"signed": list(np.typecodes["Integer"]),
"unsigned": list(np.typecodes["UnsignedInteger"]),
}
float_types = list(np.typecodes["Float"])
idx = float_types.index(cudf.dtype(np.float32).char)
downcast_type_map["float"] = float_types[idx:]
type_set = downcast_type_map[downcast]
for t in type_set:
downcast_dtype = cudf.dtype(t)
if downcast_dtype.itemsize <= col.dtype.itemsize:
if col.can_cast_safely(downcast_dtype):
col = libcudf.unary.cast(col, downcast_dtype)
break
if isinstance(arg, (cudf.Series, pd.Series)):
return cudf.Series(col)
else:
if col.has_nulls():
# To match pandas, always return a floating type filled with nan.
col = col.astype(float).fillna(np.nan)
return col.values
def _is_string_dtype(obj):
if not HAS_GPU:
return pd.api.types.is_string_dtype(obj)
else:
return is_string_dtype(obj)