def _can_downcast_to_series(self, df, arg): """ This method encapsulates the logic used to determine whether or not the result of a loc/iloc operation should be "downcasted" from a DataFrame to a Series """ from cudf.core.column import as_column if isinstance(df, cudf.Series): return False nrows, ncols = df.shape if nrows == 1: if type(arg[0]) is slice: if not is_scalar(arg[1]): return False elif (is_list_like(arg[0]) or is_column_like(arg[0])) and ( is_list_like(arg[1]) or is_column_like(arg[0]) or type(arg[1]) is slice ): return False else: if pd.api.types.is_bool_dtype( as_column(arg[0]).dtype ) and not isinstance(arg[1], slice): return True dtypes = df.dtypes.values.tolist() all_numeric = all( [pd.api.types.is_numeric_dtype(t) for t in dtypes] ) if all_numeric: return True if ncols == 1: if type(arg[1]) is slice: return False if isinstance(arg[1], tuple): # Multiindex indexing with a slice if any(isinstance(v, slice) for v in arg): return False if not (is_list_like(arg[1]) or is_column_like(arg[1])): return True return False
def length_check(obj, name): err_msg = ("Length of '{name}' ({len_obj}) did not match the " "length of the columns being encoded ({len_required}).") if is_list_like(obj): if len(obj) != len(columns): err_msg = err_msg.format(name=name, len_obj=len(obj), len_required=len(columns)) raise ValueError(err_msg)
def _filter_stripes(filters, filepath_or_buffer, stripes=None, skip_rows=None, num_rows=None): # Multiple sources are passed as a list. If a single source is passed, # wrap it in a list for unified processing downstream. if not is_list_like(filepath_or_buffer): filepath_or_buffer = [filepath_or_buffer] # Prepare filters filters = ioutils._prepare_filters(filters) # Get columns relevant to filtering columns_in_predicate = [ col for conjunction in filters for (col, op, val) in conjunction ] # Read and parse file-level and stripe-level statistics file_statistics, stripes_statistics = read_orc_statistics( filepath_or_buffer, columns_in_predicate) file_stripe_map = [] for file_stat in file_statistics: # Filter using file-level statistics if not ioutils._apply_filters(filters, file_stat): continue # Filter using stripe-level statistics selected_stripes = [] num_rows_scanned = 0 for i, stripe_statistics in enumerate(stripes_statistics): num_rows_before_stripe = num_rows_scanned num_rows_scanned += next(iter( stripe_statistics.values()))["number_of_values"] if stripes is not None and i not in stripes: continue if skip_rows is not None and num_rows_scanned <= skip_rows: continue else: skip_rows = 0 if (skip_rows is not None and num_rows is not None and num_rows_before_stripe >= skip_rows + num_rows): continue if ioutils._apply_filters(filters, stripe_statistics): selected_stripes.append(i) file_stripe_map.append(selected_stripes) return file_stripe_map
def isin(self, values, level=None): """Return a boolean array where the index values are in values. Compute boolean array of whether each index value is found in the passed set of values. The length of the returned boolean array matches the length of the index. Parameters ---------- values : set, list-like, Index or Multi-Index Sought values. level : str or int, optional Name or position of the index level to use (if the index is a MultiIndex). Returns ------- is_contained : cupy array CuPy array of boolean values. Notes ------- When `level` is None, `values` can only be MultiIndex, or a set/list-like tuples. When `level` is provided, `values` can be Index or MultiIndex, or a set/list-like tuples. """ from cudf.utils.dtypes import is_list_like if level is None: if isinstance(values, cudf.MultiIndex): values_idx = values elif ( ( isinstance( values, ( cudf.Series, cudf.Index, cudf.DataFrame, column.ColumnBase, ), ) ) or (not is_list_like(values)) or ( is_list_like(values) and len(values) > 0 and not isinstance(values[0], tuple) ) ): raise TypeError( "values need to be a Multi-Index or set/list-like tuple " "squences when `level=None`." ) else: values_idx = cudf.MultiIndex.from_tuples( values, names=self.names ) res = [] for name in self.names: level_idx = self.get_level_values(name) value_idx = values_idx.get_level_values(name) existence = level_idx.isin(value_idx) res.append(existence) result = res[0] for i in res[1:]: result = result & i else: level_series = self.get_level_values(level) result = level_series.isin(values) return result
def read_json( path_or_buf, engine="auto", dtype=True, lines=False, compression="infer", byte_range=None, *args, **kwargs, ): """{docstring}""" if engine == "cudf" and not lines: raise ValueError("cudf engine only supports JSON Lines format") if engine == "auto": engine = "cudf" if lines else "pandas" if engine == "cudf": # Multiple sources are passed as a list. If a single source is passed, # wrap it in a list for unified processing downstream. if not is_list_like(path_or_buf): path_or_buf = [path_or_buf] filepaths_or_buffers = [] for source in path_or_buf: if ioutils.is_directory(source, **kwargs): fs = ioutils._ensure_filesystem(passed_filesystem=None, path=source) source = ioutils.stringify_pathlike(source) source = fs.sep.join([source, "*.json"]) tmp_source, compression = ioutils.get_filepath_or_buffer( path_or_data=source, compression=compression, iotypes=(BytesIO, StringIO), **kwargs, ) if isinstance(tmp_source, list): filepaths_or_buffers.extend(tmp_source) else: filepaths_or_buffers.append(tmp_source) return cudf.DataFrame._from_data(*libjson.read_json( filepaths_or_buffers, dtype, lines, compression, byte_range)) else: warnings.warn("Using CPU via Pandas to read JSON dataset, this may " "be GPU accelerated in the future") if not ioutils.ensure_single_filepath_or_buffer( path_or_data=path_or_buf, **kwargs, ): raise NotImplementedError( "`read_json` does not yet support reading " "multiple files via pandas") path_or_buf, compression = ioutils.get_filepath_or_buffer( path_or_data=path_or_buf, compression=compression, iotypes=(BytesIO, StringIO), **kwargs, ) if kwargs.get("orient") == "table": pd_value = pd.read_json( path_or_buf, lines=lines, compression=compression, *args, **kwargs, ) else: pd_value = pd.read_json( path_or_buf, lines=lines, dtype=dtype, compression=compression, *args, **kwargs, ) df = cudf.from_pandas(pd_value) return df
def read_parquet( filepath_or_buffer, engine="cudf", columns=None, filters=None, row_groups=None, skip_rows=None, num_rows=None, strings_to_categorical=False, use_pandas_metadata=True, *args, **kwargs, ): """{docstring}""" # Multiple sources are passed as a list. If a single source is passed, # wrap it in a list for unified processing downstream. if not is_list_like(filepath_or_buffer): filepath_or_buffer = [filepath_or_buffer] # a list of row groups per source should be passed. make the list of # lists that is expected for multiple sources if row_groups is not None: if not is_list_like(row_groups): row_groups = [[row_groups]] elif not is_list_like(row_groups[0]): row_groups = [row_groups] filepaths_or_buffers = [] for source in filepath_or_buffer: tmp_source, compression = ioutils.get_filepath_or_buffer( path_or_data=source, compression=None, **kwargs) if compression is not None: raise ValueError( "URL content-encoding decompression is not supported") filepaths_or_buffers.append(tmp_source) if filters is not None: # Convert filters to ds.Expression filters = pq._filters_to_expression(filters) # Initialize ds.FilesystemDataset dataset = ds.dataset(filepaths_or_buffers, format="parquet", partitioning="hive") # Load IDs of filtered row groups for each file in dataset filtered_rg_ids = defaultdict(list) for fragment in dataset.get_fragments(filter=filters): for rg_fragment in fragment.get_row_group_fragments(filters): for rg_id in rg_fragment.row_groups: filtered_rg_ids[rg_fragment.path].append(rg_id) # TODO: Use this with pyarrow 1.0.0 # # Load IDs of filtered row groups for each file in dataset # filtered_row_group_ids = {} # for fragment in dataset.get_fragments(filters): # for row_group_fragment in fragment.split_by_row_group(filters): # for row_group_info in row_group_fragment.row_groups: # path = row_group_fragment.path # if path not in filtered_row_group_ids: # filtered_row_group_ids[path] = [row_group_info.id] # else: # filtered_row_group_ids[path].append(row_group_info.id) # Initialize row_groups to be selected if row_groups is None: row_groups = [None for _ in dataset.files] # Store IDs of selected row groups for each file for i, file in enumerate(dataset.files): if row_groups[i] is None: row_groups[i] = filtered_rg_ids[file] else: row_groups[i] = filter(lambda id: id in row_groups[i], filtered_rg_ids[file]) if engine == "cudf": return libparquet.read_parquet( filepaths_or_buffers, columns=columns, row_groups=row_groups, skip_rows=skip_rows, num_rows=num_rows, strings_to_categorical=strings_to_categorical, use_pandas_metadata=use_pandas_metadata, ) else: warnings.warn("Using CPU via PyArrow to read Parquet dataset.") return cudf.DataFrame.from_arrow( pq.ParquetDataset(filepaths_or_buffers).read_pandas( columns=columns, *args, **kwargs))
def cat(self, others=None, sep=None, na_rep=None): """ Concatenate strings in the Series/Index with given separator. If *others* is specified, this function concatenates the Series/Index and elements of others element-wise. If others is not passed, then all values in the Series/Index are concatenated into a single string with a given sep. Parameters ---------- others : Series or List of str Strings to be appended. The number of strings must match size() of this instance. This must be either a Series of string dtype or a Python list of strings. sep : str If specified, this separator will be appended to each string before appending the others. na_rep : str This character will take the place of any null strings (not empty strings) in either list. - If `na_rep` is None, and `others` is None, missing values in the Series/Index are omitted from the result. - If `na_rep` is None, and `others` is not None, a row containing a missing value in any of the columns (before concatenation) will have a missing value in the result. Returns ------- concat : str or Series/Index of str dtype If `others` is None, `str` is returned, otherwise a `Series/Index` (same type as caller) of str dtype is returned. """ from cudf.core import Series, Index if isinstance(others, Series): assert others.dtype == np.dtype("object") others = others._column.nvstrings elif isinstance(others, Index): assert others.dtype == np.dtype("object") others = others.as_column().nvstrings elif isinstance(others, StringMethods): """ If others is a StringMethods then raise an exception """ msg = "series.str is an accessor, not an array-like of strings." raise ValueError(msg) elif is_list_like(others) and others: """ If others is a list-like object (in our case lists & tuples) just another Series/Index, great go ahead with concatenation. """ """ Picking first element and checking if it really adheres to list like conditions, if not we switch to next case Note: We have made a call not to iterate over the entire list as it could be more expensive if it was of very large size. Thus only doing a sanity check on just the first element of list. """ first = others[0] if is_list_like(first) or isinstance( first, (Series, Index, pd.Series, pd.Index) ): """ Internal elements in others list should also be list-like and not a regular string/byte """ first = None for frame in others: if not isinstance(frame, Series): """ Make sure all inputs to .cat function call are of type nvstrings so creating a Series object. """ frame = Series(frame, dtype="str") if first is None: """ extracting nvstrings pointer since `frame` is of type Series/Index and first isn't yet initialized. """ first = frame._column.nvstrings else: assert frame.dtype == np.dtype("object") frame = frame._column.nvstrings first = first.cat(frame, sep=sep, na_rep=na_rep) others = first elif not is_list_like(first): """ Picking first element and checking if it really adheres to non-list like conditions. Note: We have made a call not to iterate over the entire list as it could be more expensive if it was of very large size. Thus only doing a sanity check on just the first element of list. """ others = Series(others) others = others._column.nvstrings elif isinstance(others, (pd.Series, pd.Index)): others = Series(others) others = others._column.nvstrings data = self._parent.nvstrings.cat( others=others, sep=sep, na_rep=na_rep ) out = Series(data, index=self._index, name=self._name) if len(out) == 1 and others is None: out = out[0] return out
def read_orc( filepath_or_buffer, engine="cudf", columns=None, filters=None, stripes=None, skiprows=None, num_rows=None, use_index=True, decimal_cols_as_float=None, timestamp_type=None, **kwargs, ): """{docstring}""" from cudf import DataFrame # Multiple sources are passed as a list. If a single source is passed, # wrap it in a list for unified processing downstream. if not is_list_like(filepath_or_buffer): filepath_or_buffer = [filepath_or_buffer] # Each source must have a correlating stripe list. If a single stripe list # is provided rather than a list of list of stripes then extrapolate that # stripe list across all input sources if stripes is not None: if any(not isinstance(stripe, list) for stripe in stripes): stripes = [stripes] # Must ensure a stripe for each source is specified, unless None if not len(stripes) == len(filepath_or_buffer): raise ValueError( "A list of stripes must be provided for each input source") filepaths_or_buffers = [] for source in filepath_or_buffer: if ioutils.is_directory(source, **kwargs): fs = ioutils._ensure_filesystem(passed_filesystem=None, path=source) source = stringify_path(source) source = fs.sep.join([source, "*.orc"]) tmp_source, compression = ioutils.get_filepath_or_buffer( path_or_data=source, compression=None, **kwargs, ) if compression is not None: raise ValueError( "URL content-encoding decompression is not supported") if isinstance(tmp_source, list): filepaths_or_buffers.extend(tmp_source) else: filepaths_or_buffers.append(tmp_source) if filters is not None: selected_stripes = _filter_stripes(filters, filepaths_or_buffers, stripes, skiprows, num_rows) # Return empty if everything was filtered if len(selected_stripes) == 0: return _make_empty_df(filepaths_or_buffers[0], columns) else: stripes = selected_stripes if engine == "cudf": return DataFrame._from_data(*liborc.read_orc( filepaths_or_buffers, columns, stripes, skiprows, num_rows, use_index, decimal_cols_as_float, timestamp_type, )) else: def read_orc_stripe(orc_file, stripe, columns): pa_table = orc_file.read_stripe(stripe, columns) if isinstance(pa_table, pa.RecordBatch): pa_table = pa.Table.from_batches([pa_table]) return pa_table warnings.warn("Using CPU via PyArrow to read ORC dataset.") if len(filepath_or_buffer) > 1: raise NotImplementedError( "Using CPU via PyArrow only supports a single a " "single input source") orc_file = orc.ORCFile(filepath_or_buffer[0]) if stripes is not None and len(stripes) > 0: for stripe_source_file in stripes: pa_tables = [ read_orc_stripe(orc_file, i, columns) for i in stripe_source_file ] pa_table = pa.concat_tables(pa_tables) else: pa_table = orc_file.read(columns=columns) df = cudf.DataFrame.from_arrow(pa_table) return df
def cut( x, bins, right: bool = True, labels=None, retbins: bool = False, precision: int = 3, include_lowest: bool = False, duplicates: str = "raise", ordered: bool = True, ): """ Bin values into discrete intervals. Use cut when you need to segment and sort data values into bins. This function is also useful for going from a continuous variable to a categorical variable. Parameters ---------- x : array-like The input array to be binned. Must be 1-dimensional. bins : int, sequence of scalars, or IntervalIndex The criteria to bin by. * int : Defines the number of equal-width bins in the range of x. The range of x is extended by .1% on each side to include the minimum and maximum values of x. right : bool, default True Indicates whether bins includes the rightmost edge or not. labels : array or False, default None Specifies the labels for the returned bins. Must be the same length as the resulting bins. If False, returns only integer indicators of thebins. If True,raises an error. When ordered=False, labels must be provided. retbins : bool, default False Whether to return the bins or not. precision : int, default 3 The precision at which to store and display the bins labels. include_lowest : bool, default False Whether the first interval should be left-inclusive or not. duplicates : {default 'raise', 'drop'}, optional If bin edges are not unique, raise ValueError or drop non-uniques. ordered : bool, default True Whether the labels are ordered or not. Applies to returned types Categorical and Series (with Categorical dtype). If True, the resulting categorical will be ordered. If False, the resulting categorical will be unordered (labels must be provided). Returns ------- out : CategoricalIndex An array-like object representing the respective bin for each value of x. The type depends on the value of labels. bins : numpy.ndarray or IntervalIndex. The computed or specified bins. Only returned when retbins=True. For scalar or sequence bins, this is an ndarray with the computed bins. If set duplicates=drop, bins will drop non-unique bin. For an IntervalIndex bins, this is equal to bins. Examples -------- Discretize into three equal-sized bins. >>> cudf.cut(np.array([1, 7, 5, 4, 6, 3]), 3) CategoricalIndex([(0.994, 3.0], (5.0, 7.0], (3.0, 5.0], (3.0, 5.0], ... (5.0, 7.0],(0.994, 3.0]], categories=[(0.994, 3.0], ... (3.0, 5.0], (5.0, 7.0]], ordered=True, dtype='category') >>> cudf.cut(np.array([1, 7, 5, 4, 6, 3]), 3, retbins=True) (CategoricalIndex([(0.994, 3.0], (5.0, 7.0], (3.0, 5.0], (3.0, 5.0], ... (5.0, 7.0],(0.994, 3.0]],categories=[(0.994, 3.0], ... (3.0, 5.0], (5.0, 7.0]],ordered=True, dtype='category'), array([0.994, 3. , 5. , 7. ])) >>> cudf.cut(np.array([1, 7, 5, 4, 6, 3]), ... 3, labels=["bad", "medium", "good"]) CategoricalIndex(['bad', 'good', 'medium', 'medium', 'good', 'bad'], ... categories=['bad', 'medium', 'good'],ordered=True, ... dtype='category') >>> cudf.cut(np.array([1, 7, 5, 4, 6, 3]), 3, ... labels=["B", "A", "B"], ordered=False) CategoricalIndex(['B', 'B', 'A', 'A', 'B', 'B'], categories=['A', 'B'], ... ordered=False, dtype='category') >>> cudf.cut([0, 1, 1, 2], bins=4, labels=False) array([0, 1, 1, 3], dtype=int32) Passing a Series as an input returns a Series with categorical dtype: >>> s = cudf.Series(np.array([2, 4, 6, 8, 10]), ... index=['a', 'b', 'c', 'd', 'e']) >>> cudf.cut(s, 3) """ left_inclusive = False right_inclusive = True # saving the original input x for use in case its a series orig_x = x old_bins = bins if not ordered and labels is None: raise ValueError("'labels' must be provided if 'ordered = False'") if duplicates not in ["raise", "drop"]: raise ValueError( "invalid value for 'duplicates' parameter, valid options are: " "raise, drop") if labels is not False: if not (labels is None or is_list_like(labels)): raise ValueError( "Bin labels must either be False, None or passed in as a " "list-like argument") elif ordered and labels is not None: if len(set(labels)) != len(labels): raise ValueError("labels must be unique if ordered=True;" "pass ordered=False for duplicate labels") # bins can either be an int, sequence of scalars or an intervalIndex if isinstance(bins, Sequence): if len(set(bins)) is not len(bins): if duplicates == "raise": raise ValueError( f"Bin edges must be unique: {repr(bins)}.\n" f"You can drop duplicate edges by setting the 'duplicates'" "kwarg") elif duplicates == "drop": # get unique values but maintain list dtype bins = list(dict.fromkeys(bins)) # if bins is an intervalIndex we ignore the value of right elif isinstance(bins, (pd.IntervalIndex, cudf.IntervalIndex)): right = bins.closed == "right" # create bins if given an int or single scalar if not isinstance(bins, pd.IntervalIndex): if not isinstance(bins, (Sequence)): if isinstance(x, (pd.Series, cudf.Series, np.ndarray, cupy.ndarray)): mn = x.min() mx = x.max() else: mn = min(x) mx = max(x) bins = np.linspace(mn, mx, bins + 1, endpoint=True) adj = (mx - mn) * 0.001 if right: bins[0] -= adj else: bins[-1] += adj # if right and include lowest we adjust the first # bin edge to make sure it is included if right and include_lowest: bins[0] = bins[0] - 10**(-precision) # if right is false the last bin edge is not included if not right: right_edge = bins[-1] x = cupy.asarray(x) x[x == right_edge] = right_edge + 1 # adjust bin edges decimal precision int_label_bins = np.around(bins, precision) # the inputs is a column of the values in the array x input_arr = as_column(x) # checking for the correct inclusivity values if right: closed = "right" else: closed = "left" left_inclusive = True if isinstance(bins, pd.IntervalIndex): interval_labels = bins elif labels is None: if duplicates == "drop" and len(bins) == 1 and len(old_bins) != 1: if right and include_lowest: old_bins[0] = old_bins[0] - 10**(-precision) interval_labels = interval_range(old_bins[0], old_bins[1], periods=1, closed=closed) else: interval_labels = IntervalIndex.from_breaks(old_bins, closed=closed) else: # get labels for categories interval_labels = IntervalIndex.from_breaks(int_label_bins, closed=closed) elif labels is not False: if not (is_list_like(labels)): raise ValueError( "Bin labels must either be False, None or passed in as a " "list-like argument") if ordered and len(set(labels)) != len(labels): raise ValueError( "labels must be unique if ordered=True; pass ordered=False for" "duplicate labels") else: if len(labels) != len(bins) - 1: raise ValueError( "Bin labels must be one fewer than the number of bin edges" ) if not ordered and len(set(labels)) != len(labels): interval_labels = cudf.CategoricalIndex(labels, categories=None, ordered=False) else: interval_labels = (labels if len(set(labels)) == len(labels) else None) if isinstance(bins, pd.IntervalIndex): # get the left and right edges of the bins as columns # we cannot typecast an IntervalIndex, so we need to # make the edges the same type as the input array left_edges = as_column(bins.left).astype(input_arr.dtype) right_edges = as_column(bins.right).astype(input_arr.dtype) else: # get the left and right edges of the bins as columns left_edges = as_column(bins[:-1:], dtype="float64") right_edges = as_column(bins[+1::], dtype="float64") # the input arr must be changed to the same type as the edges input_arr = input_arr.astype(left_edges.dtype) # get the indexes for the appropriate number index_labels = cudf._lib.labeling.label_bins(input_arr, left_edges, left_inclusive, right_edges, right_inclusive) if labels is False: # if labels is false we return the index labels, we return them # as a series if we have a series input if isinstance(orig_x, (pd.Series, cudf.Series)): # need to run more tests but looks like in this case pandas # always returns a float64 dtype indx_arr_series = cudf.Series(index_labels, dtype="float64") # if retbins we return the bins as well if retbins: return indx_arr_series, bins else: return indx_arr_series elif retbins: return index_labels.values, bins else: return index_labels.values if labels is not None: if labels is not ordered and len(set(labels)) != len(labels): # when we have duplicate labels and ordered is False, we # should allow duplicate categories. The categories are # returned in order new_data = [interval_labels[i][0] for i in index_labels.values] return cudf.CategoricalIndex(new_data, categories=sorted(set(labels)), ordered=False) col = build_categorical_column( categories=interval_labels, codes=index_labels, mask=index_labels.base_mask, offset=index_labels.offset, size=index_labels.size, ordered=ordered, ) # we return a categorical index, as we don't have a Categorical method categorical_index = cudf.core.index.as_index(col) if isinstance(orig_x, (pd.Series, cudf.Series)): # if we have a series input we return a series output res_series = cudf.Series(categorical_index, index=orig_x.index) if retbins: return res_series, bins else: return res_series elif retbins: # if retbins is true we return the bins as well return categorical_index, bins else: return categorical_index