def scalar_broadcast_to(scalar, size, dtype=None): from cudf.utils.cudautils import fill_value from cudf.utils.dtypes import to_cudf_compatible_scalar, is_string_dtype from cudf.core.column import column_empty if isinstance(size, (tuple, list)): size = size[0] if scalar is None: if dtype is None: dtype = "object" return column_empty(size, dtype=dtype, masked=True) if isinstance(scalar, pd.Categorical): return scalar_broadcast_to(scalar.categories[0], size).astype(dtype) if isinstance(scalar, str) and (is_string_dtype(dtype) or dtype is None): dtype = "object" else: scalar = to_cudf_compatible_scalar(scalar, dtype=dtype) dtype = scalar.dtype if np.dtype(dtype) == np.dtype("object"): import nvstrings from cudf.core.column import as_column from cudf.utils.cudautils import zeros gather_map = zeros(size, dtype="int32") scalar_str_col = as_column(nvstrings.to_device([scalar])) return scalar_str_col[gather_map] else: da = rmm.device_array((size, ), dtype=dtype) if da.size != 0: fill_value(da, scalar) return da
def test_full_df(num_rows, tmpdir, distro): json_sample["num_rows"] = num_rows cats = list(json_sample["cats"].keys()) cols = datagen._get_cols_from_schema(json_sample, distros=distro) df_gen = datagen.DatasetGen(datagen.UniformDistro(), gpu_frac=0.00001) df_files = df_gen.full_df_create(num_rows, cols, entries=True, output=tmpdir) test_size = 0 full_df = cudf.DataFrame() for fi in df_files: df = cudf.read_parquet(fi) test_size = test_size + df.shape[0] full_df = cudf.concat([full_df, df]) assert test_size == num_rows conts_rep = cols["conts"] cats_rep = cols["cats"] labels_rep = cols["labels"] assert df.shape[1] == len(conts_rep) + len(cats_rep) + len(labels_rep) for idx, cat in enumerate(cats[1:]): dist = cats_rep[idx + 1].distro or df_gen.dist if not is_string_dtype(full_df[cat]._column): sts, ps = dist.verify(full_df[cat].to_pandas()) assert all(s > 0.9 for s in sts) assert full_df[cat].nunique() == cats_rep[idx + 1].cardinality assert full_df[cat].str.len().min() == cats_rep[idx + 1].min_entry_size assert full_df[cat].str.len().max() == cats_rep[idx + 1].max_entry_size check_ser = cudf.Series(full_df[cats[0]]._column.elements.values_host) assert check_ser.nunique() == cats_rep[0].cardinality assert check_ser.str.len().min() == cats_rep[0].min_entry_size assert check_ser.str.len().max() == cats_rep[0].max_entry_size
def scalar_broadcast_to(scalar, size, dtype=None): from cudf.utils.dtypes import to_cudf_compatible_scalar, is_string_dtype from cudf.core.column import column_empty if isinstance(size, (tuple, list)): size = size[0] if scalar is None: if dtype is None: dtype = "object" return column_empty(size, dtype=dtype, masked=True) if isinstance(scalar, pd.Categorical): return scalar_broadcast_to(scalar.categories[0], size).astype(dtype) if isinstance(scalar, str) and (is_string_dtype(dtype) or dtype is None): dtype = "object" else: scalar = to_cudf_compatible_scalar(scalar, dtype=dtype) dtype = scalar.dtype if np.dtype(dtype) == np.dtype("object"): from cudf.core.column import as_column gather_map = cupy.zeros(size, dtype="int32") scalar_str_col = as_column([scalar], dtype="str") return scalar_str_col[gather_map] else: out_col = column_empty(size, dtype=dtype) if out_col.size != 0: out_col.data_array_view[:] = scalar return out_col
def _fill(self, fill_value, begin=0, end=-1, inplace=False): if end <= begin or begin >= self.size: return self if inplace else self.copy() if is_categorical_dtype(self.dtype): return self._fill_categorical(fill_value, begin, end, inplace) fill_scalar = Scalar(fill_value, self.dtype) if not inplace: return libcudfxx.filling.fill(self, begin, end, fill_scalar) if is_string_dtype(self.dtype): return self._mimic_inplace( libcudfxx.filling.fill(self, begin, end, fill_scalar), inplace=True, ) if fill_value is None and not self.nullable: mask = create_null_mask(self.size, state=MaskState.ALL_VALID) self.set_base_mask(mask) libcudfxx.filling.fill_in_place(self, begin, end, fill_scalar) return self
def _convert_str_col(col, errors, _downcast=None): """ Converts a string column to numeric column Converts to integer column if all strings are integer-like (isinteger.all) Otherwise, converts to float column if all strings are float-like ( isfloat.all) If error == 'coerce', fill non-numerics strings with null Looks ahead to ``downcast`` parameter, if the float may be casted to integer, then only process in float32 pipeline. Parameters ---------- col : The string column to convert, must be string dtype errors : {'raise', 'ignore', 'coerce'}, same as ``to_numeric`` _downcast : Same as ``to_numeric``, see description for use Returns ------- Converted numeric column """ if not is_string_dtype(col): raise TypeError("col must be string dtype.") is_integer = col.str().isinteger() if is_integer.all(): return col.as_numerical_column(dtype=np.dtype("i8")) col = _proc_inf_empty_strings(col) is_float = col.str().isfloat() if is_float.all(): if _downcast in {"unsigned", "signed", "integer"}: warnings.warn( UserWarning("Downcasting from float to int will be " "limited by float32 precision.")) return col.as_numerical_column(dtype=np.dtype("f")) else: return col.as_numerical_column(dtype=np.dtype("d")) else: if errors == "coerce": col = libcudf.string_casting.stod(col) non_numerics = is_float.unary_operator("not") col[non_numerics] = None return col else: raise ValueError("Unable to convert some strings to numerics.")
def _get_non_empty_data(s): if isinstance(s._column, cudf.core.column.CategoricalColumn): categories = (s._column.categories if len(s._column.categories) else [UNKNOWN_CATEGORIES]) codes = column.full(size=2, fill_value=0, dtype="int32") ordered = s._column.ordered data = column.build_categorical_column(categories=categories, codes=codes, ordered=ordered) elif is_string_dtype(s.dtype): data = pa.array(["cat", "dog"]) else: if pd.api.types.is_numeric_dtype(s.dtype): data = column.as_column(cp.arange(start=0, stop=2, dtype=s.dtype)) else: data = column.as_column(cp.arange(start=0, stop=2, dtype="int64")).astype(s.dtype) return data
def _nonempty_series(s, idx=None): if idx is None: idx = _nonempty_index(s.index) dtype = s.dtype if is_categorical_dtype(dtype): categories = ( s._column.categories if len(s._column.categories) else ["a"] ) codes = [0, 0] ordered = s._column.ordered data = column.build_categorical_column( categories=categories, codes=codes, ordered=ordered ) elif is_string_dtype(dtype): data = ["cat", "dog"] else: data = np.arange(start=0, stop=2, dtype=dtype) return cudf.Series(data, name=s.name, index=idx)
def _mimic_inplace(self, result, inplace=False): """ If `inplace=True`, used to mimic an inplace operation by replacing data in ``self`` with data in ``result``. Otherwise, returns ``result`` unchanged. """ if inplace: self.data = result.data self.mask = result.mask self.dtype = result.dtype self.size = result.size self.offset = result.offset if hasattr(result, "children"): self.children = result.children if is_string_dtype(self): # force recomputation of nvstrings/nvcategory self._nvstrings = None self._nvcategory = None self.__class__ = result.__class__ else: return result
def to_numeric(arg, errors="raise", downcast=None): """ Convert argument into numerical types. Parameters ---------- arg : column-convertible The object to convert to numeric types errors : {'raise', 'ignore', 'coerce'}, defaults 'raise' Policy to handle errors during parsing. * 'raise' will notify user all errors encountered. * 'ignore' will skip error and returns ``arg``. * 'coerce' will leave invalid values as nulls. downcast : {'integer', 'signed', 'unsigned', 'float'}, defaults None If set, will try to down-convert the datatype of the parsed results to smallest possible type. For each `downcast` type, this method will determine the smallest possible dtype from the following sets: * {'integer', 'signed'}: all integer types greater or equal to `np.int8` * {'unsigned'}: all unsigned types greater or equal to `np.uint8` * {'float'}: all floating types greater or equal to `np.float32` Note that downcast behavior is decoupled from parsing. Errors encountered during downcast is raised regardless of ``errors`` parameter. Returns ------- Series or ndarray Depending on the input, if series is passed in, series is returned, otherwise ndarray Notes ------- An important difference from pandas is that this function does not accept mixed numeric/non-numeric type sequences. For example ``[1, 'a']``. A ``TypeError`` will be raised when such input is received, regardless of ``errors`` parameter. Examples -------- >>> s = cudf.Series(['1', '2.0', '3e3']) >>> cudf.to_numeric(s) 0 1.0 1 2.0 2 3000.0 dtype: float64 >>> cudf.to_numeric(s, downcast='float') 0 1.0 1 2.0 2 3000.0 dtype: float32 >>> cudf.to_numeric(s, downcast='signed') 0 1 1 2 2 3000 dtype: int16 >>> s = cudf.Series(['apple', '1.0', '3e3']) >>> cudf.to_numeric(s, errors='ignore') 0 apple 1 1.0 2 3e3 dtype: object >>> cudf.to_numeric(s, errors='coerce') 0 <NA> 1 1.0 2 3000.0 dtype: float64 """ if errors not in {"raise", "ignore", "coerce"}: raise ValueError("invalid error value specified") if downcast not in {None, "integer", "signed", "unsigned", "float"}: raise ValueError("invalid downcasting method provided") if not can_convert_to_column(arg) or (hasattr(arg, "ndim") and arg.ndim > 1): raise ValueError("arg must be column convertible") col = as_column(arg) dtype = col.dtype if is_datetime_dtype(dtype) or is_timedelta_dtype(dtype): col = col.as_numerical_column(np.dtype("int64")) elif is_categorical_dtype(dtype): cat_dtype = col.dtype.type if is_numerical_dtype(cat_dtype): col = col.as_numerical_column(cat_dtype) else: try: col = _convert_str_col(col._get_decategorized_column(), errors, downcast) except ValueError as e: if errors == "ignore": return arg else: raise e elif is_string_dtype(dtype): try: col = _convert_str_col(col, errors, downcast) except ValueError as e: if errors == "ignore": return arg else: raise e elif is_list_dtype(dtype) or is_struct_dtype(dtype): raise ValueError("Input does not support nested datatypes") elif is_numerical_dtype(dtype): pass else: raise ValueError("Unrecognized datatype") # str->float conversion may require lower precision if col.dtype == np.dtype("f"): col = col.as_numerical_column("d") if downcast: downcast_type_map = { "integer": list(np.typecodes["Integer"]), "signed": list(np.typecodes["Integer"]), "unsigned": list(np.typecodes["UnsignedInteger"]), } float_types = list(np.typecodes["Float"]) idx = float_types.index(np.dtype(np.float32).char) downcast_type_map["float"] = float_types[idx:] type_set = downcast_type_map[downcast] for t in type_set: downcast_dtype = np.dtype(t) if downcast_dtype.itemsize <= col.dtype.itemsize: if col.can_cast_safely(downcast_dtype): col = libcudf.unary.cast(col, downcast_dtype) break if isinstance(arg, (cudf.Series, pd.Series)): return cudf.Series(col) else: col = col.fillna(col.default_na_value()) return col.values