def _group_inner_levels(self, columns, rowidcol, segs, markers):
"""Group the second and onwards level.
Parameters
----------
columns : sequence[str]
Group keys. The order is important.
rowid_column : str
The name of the special column with the original rowid.
It's internally used to determine the shuffling order.
df : DataFrame
The dataframe being grouped.
segs : Series
First level group begin offsets.
Returns
-------
(sorted_keys, reordering_indices, segments)
- sorted_keys : list[Series]
List of sorted key columns.
Column order is same as arg *columns*.
- reordering_indices : device array
The indices to gather on to shuffle the dataframe
into the grouped seqence.
- segments : Series
Group begin offsets.
"""
dsegs = segs.astype(dtype=np.int32).data.mem
sorted_keys = []
plan_cache = {}
for col in columns:
# Shuffle the key column according to the previous groups
srkeys = self._df[col].take(rowidcol.to_gpu_array(),
ignore_index=True)
# Segmented sort on the key
shuf = Column(Buffer(cudautils.arange(len(srkeys))))
cache_key = (len(srkeys), srkeys.dtype, shuf.dtype)
plan = plan_cache.get(cache_key)
plan = apply_segsort(srkeys._column, shuf, dsegs, plan=plan)
plan_cache[cache_key] = plan
sorted_keys.append(srkeys) # keep sorted key cols
# Determine segments
dsegs, markers = cudautils.find_segments(srkeys.to_gpu_array(),
dsegs,
markers=markers)
# Shuffle
rowidcol = rowidcol.take(shuf.to_gpu_array(), ignore_index=True)
reordering_indices = rowidcol.to_gpu_array()
return sorted_keys, reordering_indices, Series(dsegs)
def column_empty_like(column, dtype, masked):
"""Allocate a new column like the given *column*
"""
data = rmm.device_array(shape=len(column), dtype=dtype)
params = dict(data=Buffer(data))
if masked:
mask = utils.make_mask(data.size)
params.update(dict(mask=Buffer(mask), null_count=data.size))
return Column(**params)
def from_cffi_view(cffi_view):
"""Create a Column object from a cffi struct gdf_column*.
"""
data_mem, mask_mem = _gdf.cffi_view_to_column_mem(cffi_view)
data_buf = Buffer(data_mem)
if mask_mem is not None:
mask = Buffer(mask_mem)
else:
mask = None
return Column(data=data_buf, mask=mask)
def _concat(cls, objs, axis=0, index=True):
# Concatenate index if not provided
if index is True:
index = Index._concat([o.index for o in objs])
names = {obj.name for obj in objs}
if len(names) == 1:
[name] = names
else:
name = None
col = Column._concat([o._column for o in objs])
return cls(data=col, index=index, name=name)
def column_empty_like_same_mask(column, dtype):
"""Create a new empty Column with the same length and the same mask.
Parameters
----------
dtype : np.dtype like
The dtype of the data buffer.
"""
data = rmm.device_array(shape=len(column), dtype=dtype)
params = dict(data=Buffer(data))
if column.has_null_mask:
params.update(mask=column.nullmask)
return Column(**params)
def remove_base(dct):
# removes base attributes in the phyiscal layer.
basekeys = Column._replace_defaults(self).keys()
for k in basekeys:
del dct[k]
def as_column(arbitrary, nan_as_null=True, dtype=None):
"""Create a Column from an arbitrary object
Currently support inputs are:
* ``Column``
* ``Buffer``
* ``Series``
* ``Index``
* numba device array
* cuda array interface
* numpy array
* pyarrow array
* pandas.Categorical
Returns
-------
result : subclass of TypedColumnBase
- CategoricalColumn for pandas.Categorical input.
- DatetimeColumn for datetime input
- NumericalColumn for all other inputs.
"""
from cudf.dataframe import numerical, categorical, datetime, string
from cudf.dataframe.series import Series
from cudf.dataframe.index import Index
if isinstance(arbitrary, Column):
categories = None
if hasattr(arbitrary, "categories"):
categories = arbitrary.categories
data = build_column(arbitrary.data,
arbitrary.dtype,
mask=arbitrary.mask,
categories=categories)
elif isinstance(arbitrary, Series):
data = arbitrary._column
elif isinstance(arbitrary, Index):
data = arbitrary._values
elif isinstance(arbitrary, Buffer):
data = numerical.NumericalColumn(data=arbitrary, dtype=arbitrary.dtype)
elif isinstance(arbitrary, nvstrings.nvstrings):
data = string.StringColumn(data=arbitrary)
elif cuda.devicearray.is_cuda_ndarray(arbitrary):
data = as_column(Buffer(arbitrary))
if (data.dtype in [np.float16, np.float32, np.float64]
and arbitrary.size > 0):
if nan_as_null:
mask = cudautils.mask_from_devary(arbitrary)
data = data.set_mask(mask)
elif cuda.is_cuda_array(arbitrary):
# Use cuda array interface to do create a numba device array by
# reference
new_dev_array = cuda.as_cuda_array(arbitrary)
# Allocate new output array using rmm and copy the numba device array
# to an rmm owned device array
out_dev_array = rmm.device_array_like(new_dev_array)
out_dev_array.copy_to_device(new_dev_array)
data = as_column(out_dev_array)
elif isinstance(arbitrary, np.ndarray):
# CUDF assumes values are always contiguous
if not arbitrary.flags['C_CONTIGUOUS']:
arbitrary = np.ascontiguousarray(arbitrary)
if arbitrary.dtype.kind == 'M':
data = datetime.DatetimeColumn.from_numpy(arbitrary)
elif arbitrary.dtype.kind in ('O', 'U'):
data = as_column(pa.Array.from_pandas(arbitrary))
else:
data = as_column(rmm.to_device(arbitrary), nan_as_null=nan_as_null)
elif isinstance(arbitrary, pa.Array):
if isinstance(arbitrary, pa.StringArray):
count = len(arbitrary)
null_count = arbitrary.null_count
buffers = arbitrary.buffers()
# Buffer of actual strings values
if buffers[2] is not None:
sbuf = np.frombuffer(buffers[2], dtype='int8')
else:
sbuf = np.empty(0, dtype='int8')
# Buffer of offsets values
obuf = np.frombuffer(buffers[1], dtype='int32')
# Buffer of null bitmask
nbuf = None
if null_count > 0:
nbuf = np.frombuffer(buffers[0], dtype='int8')
data = as_column(
nvstrings.from_offsets(sbuf,
obuf,
count,
nbuf=nbuf,
ncount=null_count))
elif isinstance(arbitrary, pa.NullArray):
new_dtype = dtype
if (type(dtype) == str and dtype == 'empty') or dtype is None:
new_dtype = np.dtype(arbitrary.type.to_pandas_dtype())
if pd.api.types.is_categorical_dtype(new_dtype):
arbitrary = arbitrary.dictionary_encode()
else:
if nan_as_null:
arbitrary = arbitrary.cast(_gdf.np_to_pa_dtype(new_dtype))
else:
# casting a null array doesn't make nans valid
# so we create one with valid nans from scratch:
if new_dtype == np.dtype("object"):
arbitrary = utils.scalar_broadcast_to(
None, (len(arbitrary), ), dtype=new_dtype)
else:
arbitrary = utils.scalar_broadcast_to(
np.nan, (len(arbitrary), ), dtype=new_dtype)
data = as_column(arbitrary, nan_as_null=nan_as_null)
elif isinstance(arbitrary, pa.DictionaryArray):
pamask, padata = buffers_from_pyarrow(arbitrary)
data = categorical.CategoricalColumn(
data=padata,
mask=pamask,
null_count=arbitrary.null_count,
categories=arbitrary.dictionary.to_pylist(),
ordered=arbitrary.type.ordered,
)
elif isinstance(arbitrary, pa.TimestampArray):
arbitrary = arbitrary.cast(pa.timestamp('ms'))
pamask, padata = buffers_from_pyarrow(arbitrary, dtype='M8[ms]')
data = datetime.DatetimeColumn(data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=np.dtype('M8[ms]'))
elif isinstance(arbitrary, pa.Date64Array):
pamask, padata = buffers_from_pyarrow(arbitrary, dtype='M8[ms]')
data = datetime.DatetimeColumn(data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=np.dtype('M8[ms]'))
elif isinstance(arbitrary, pa.Date32Array):
# No equivalent np dtype and not yet supported
warnings.warn(
"Date32 values are not yet supported so this will "
"be typecast to a Date64 value", UserWarning)
arbitrary = arbitrary.cast(pa.date64())
data = as_column(arbitrary)
elif isinstance(arbitrary, pa.BooleanArray):
# Arrow uses 1 bit per value while we use int8
dtype = np.dtype(np.bool)
# Needed because of bug in PyArrow
# https://issues.apache.org/jira/browse/ARROW-4766
if len(arbitrary) > 0:
arbitrary = arbitrary.cast(pa.int8())
else:
arbitrary = pa.array([], type=pa.int8())
pamask, padata = buffers_from_pyarrow(arbitrary, dtype=dtype)
data = numerical.NumericalColumn(data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=dtype)
else:
pamask, padata = buffers_from_pyarrow(arbitrary)
data = numerical.NumericalColumn(
data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=np.dtype(arbitrary.type.to_pandas_dtype()))
elif isinstance(arbitrary, pa.ChunkedArray):
gpu_cols = [
as_column(chunk, dtype=dtype) for chunk in arbitrary.chunks
]
if dtype and dtype != 'empty':
new_dtype = dtype
else:
pa_type = arbitrary.type
if pa.types.is_dictionary(pa_type):
new_dtype = 'category'
else:
new_dtype = np.dtype(pa_type.to_pandas_dtype())
data = Column._concat(gpu_cols, dtype=new_dtype)
elif isinstance(arbitrary, (pd.Series, pd.Categorical)):
if pd.api.types.is_categorical_dtype(arbitrary):
data = as_column(pa.array(arbitrary, from_pandas=True))
elif arbitrary.dtype == np.bool:
# Bug in PyArrow or HDF that requires us to do this
data = as_column(pa.array(np.array(arbitrary), from_pandas=True))
else:
data = as_column(pa.array(arbitrary, from_pandas=nan_as_null))
elif isinstance(arbitrary, pd.Timestamp):
# This will always treat NaTs as nulls since it's not technically a
# discrete value like NaN
data = as_column(pa.array(pd.Series([arbitrary]), from_pandas=True))
elif np.isscalar(arbitrary) and not isinstance(arbitrary, memoryview):
if hasattr(arbitrary, 'dtype'):
data_type = _gdf.np_to_pa_dtype(arbitrary.dtype)
if data_type in (pa.date64(), pa.date32()):
# PyArrow can't construct date64 or date32 arrays from np
# datetime types
arbitrary = arbitrary.astype('int64')
data = as_column(pa.array([arbitrary], type=data_type))
else:
data = as_column(pa.array([arbitrary]), nan_as_null=nan_as_null)
elif isinstance(arbitrary, memoryview):
data = as_column(np.array(arbitrary),
dtype=dtype,
nan_as_null=nan_as_null)
else:
try:
data = as_column(memoryview(arbitrary))
except TypeError:
try:
pa_type = None
if dtype is not None:
if pd.api.types.is_categorical_dtype(dtype):
raise TypeError
else:
np_type = np.dtype(dtype).type
if np_type == np.bool_:
pa_type = pa.bool_()
else:
pa_type = _gdf.np_to_pa_dtype(np.dtype(dtype).type)
data = as_column(pa.array(arbitrary,
type=pa_type,
from_pandas=nan_as_null),
nan_as_null=nan_as_null)
except (pa.ArrowInvalid, pa.ArrowTypeError, TypeError):
np_type = None
if pd.api.types.is_categorical_dtype(dtype):
data = as_column(pd.Series(arbitrary, dtype='category'),
nan_as_null=nan_as_null)
else:
if dtype is None:
np_type = None
else:
np_type = np.dtype(dtype)
data = as_column(np.array(arbitrary, dtype=np_type),
nan_as_null=nan_as_null)
return data
def _concat(cls, objs):
data = Column._concat([o.as_column() for o in objs])
return as_index(data)
def read_csv_strings(filepath_or_buffer,
lineterminator='\n',
quotechar='"',
quoting=0,
doublequote=True,
header='infer',
sep=',',
delimiter=None,
delim_whitespace=False,
skipinitialspace=False,
names=None,
dtype=None,
skipfooter=0,
skiprows=0,
dayfirst=False,
compression='infer',
thousands=None,
decimal='.',
true_values=None,
false_values=None,
nrows=None,
byte_range=None,
skip_blank_lines=True,
comment=None,
na_values=None,
keep_default_na=True,
na_filter=True,
prefix=None,
index_col=None):
"""
**Experimental**: This function exists only as a beta way to use
`nvstrings <https://nvstrings.readthedocs.io/en/latest/>`_. with cudf.
Future versions of cuDF will provide cleaner integration.
Uses mostly same arguments as read_csv.
Note: Doesn't currently support auto-column detection, header, usecols
and mangle_dupe_cols args.
Returns
-------
columns : ordered list of cudf.dataframe.Series and nvstrings objects
numeric or date dtyped columns will be Series.
'str' dtyped columns will be
`nvstrings <https://nvstrings.readthedocs.io/en/latest/>`_.
Examples
--------
Create a test csv file
>>> import cudf
>>> filename = 'foo.csv'
>>> lines = [
... "num1,datetime,text",
... "123,2018-11-13T12:00:00,abc",
... "456,2018-11-14T12:35:01,def",
... "789,2018-11-15T18:02:59,ghi"
... ]
>>> with open(filename, 'w') as fp:
... fp.write('\\n'.join(lines)+'\\n')
Read the file with cudf
>>> names = ['num1', 'datetime', 'text']
>>> dtypes = ['int', 'date', 'str']
>>> columns = cudf.io.csv.read_csv_strings(filename, delimiter=',',
... names=names, dtype=dtypes,
... skiprows=1)
Display results
>>> print(columns[0])
0 123
1 456
2 789
>>> print(columns[2])
['abc', 'def', 'ghi']
See Also
--------
.read_csv
"""
import nvstrings
from cudf.dataframe.series import Series
# Alias sep -> delimiter.
if delimiter is None:
delimiter = sep
if dtype is not None:
if isinstance(dtype, collections.abc.Mapping):
dtype_dict = True
elif isinstance(dtype, collections.abc.Iterable):
dtype_dict = False
else:
msg = '''dtype must be 'list like' or 'dict' '''
raise TypeError(msg)
if names is not None and len(dtype) != len(names):
msg = '''All column dtypes must be specified.'''
raise TypeError(msg)
csv_reader = ffi.new('csv_read_arg*')
# Populate csv_reader struct
if is_file_like(filepath_or_buffer):
buffer = filepath_or_buffer.read()
# check if StringIO is used
if hasattr(buffer, 'encode'):
buffer_as_bytes = buffer.encode()
else:
buffer_as_bytes = buffer
buffer_data_holder = ffi.new("char[]", buffer_as_bytes)
csv_reader.input_data_form = libgdf.HOST_BUFFER
csv_reader.filepath_or_buffer = buffer_data_holder
csv_reader.buffer_size = len(buffer_as_bytes)
else:
if (not os.path.isfile(filepath_or_buffer)):
raise (FileNotFoundError)
if (not os.path.exists(filepath_or_buffer)):
raise (FileNotFoundError)
file_path = _wrap_string(filepath_or_buffer)
csv_reader.input_data_form = libgdf.FILE_PATH
csv_reader.filepath_or_buffer = file_path
if header == 'infer':
header = -1
header_infer = header
arr_names = []
arr_dtypes = []
if names is None:
if header is -1:
header_infer = 0
if header is None:
header_infer = -1
csv_reader.names = ffi.NULL
csv_reader.num_cols = 0
else:
if header is None:
header_infer = -1
csv_reader.num_cols = len(names)
for col_name in names:
arr_names.append(_wrap_string(col_name))
if dtype is not None:
if dtype_dict:
arr_dtypes.append(_wrap_string(str(dtype[col_name])))
names_ptr = ffi.new('char*[]', arr_names)
csv_reader.names = names_ptr
if dtype is None:
csv_reader.dtype = ffi.NULL
else:
if not dtype_dict:
for col_dtype in dtype:
arr_dtypes.append(_wrap_string(str(col_dtype)))
dtype_ptr = ffi.new('char*[]', arr_dtypes)
csv_reader.dtype = dtype_ptr
if decimal == delimiter:
raise ValueError("decimal cannot be the same as delimiter")
if thousands == delimiter:
raise ValueError("thousands cannot be the same as delimiter")
if nrows is not None and skipfooter != 0:
raise ValueError("cannot use both nrows and skipfooter parameters")
if byte_range is not None:
if skipfooter != 0 or skiprows != 0 or nrows is not None:
raise ValueError("""cannot manually limit rows to be read when
using the byte range parameter""")
# Start with default values recognized as boolean
arr_true_values = [_wrap_string(str('True')), _wrap_string(str('TRUE'))]
arr_false_values = [_wrap_string(str('False')), _wrap_string(str('FALSE'))]
for value in true_values or []:
arr_true_values.append(_wrap_string(str(value)))
arr_true_values_ptr = ffi.new('char*[]', arr_true_values)
csv_reader.true_values = arr_true_values_ptr
csv_reader.num_true_values = len(arr_true_values)
for value in false_values or []:
arr_false_values.append(_wrap_string(str(value)))
false_values_ptr = ffi.new('char*[]', arr_false_values)
csv_reader.false_values = false_values_ptr
csv_reader.num_false_values = len(arr_false_values)
arr_na_values = []
for value in na_values or []:
arr_na_values.append(_wrap_string(str(value)))
arr_na_values_ptr = ffi.new('char*[]', arr_na_values)
csv_reader.na_values = arr_na_values_ptr
csv_reader.num_na_values = len(arr_na_values)
compression_bytes = _wrap_string(compression)
prefix_bytes = _wrap_string(prefix)
csv_reader.delimiter = delimiter.encode()
csv_reader.lineterminator = lineterminator.encode()
csv_reader.quotechar = quotechar.encode()
csv_reader.quoting = _quoting_enum[quoting]
csv_reader.doublequote = doublequote
csv_reader.delim_whitespace = delim_whitespace
csv_reader.skipinitialspace = skipinitialspace
csv_reader.dayfirst = dayfirst
csv_reader.header = header_infer
csv_reader.skiprows = skiprows
csv_reader.skipfooter = skipfooter
csv_reader.compression = compression_bytes
csv_reader.decimal = decimal.encode()
csv_reader.thousands = thousands.encode() if thousands else b'\0'
csv_reader.nrows = nrows if nrows is not None else -1
if byte_range is not None:
csv_reader.byte_range_offset = byte_range[0]
csv_reader.byte_range_size = byte_range[1]
else:
csv_reader.byte_range_offset = 0
csv_reader.byte_range_size = 0
csv_reader.skip_blank_lines = skip_blank_lines
csv_reader.comment = comment.encode() if comment else b'\0'
csv_reader.keep_default_na = keep_default_na
csv_reader.na_filter = na_filter
csv_reader.prefix = prefix_bytes
# Call read_csv
libgdf.read_csv(csv_reader)
out = csv_reader.data
if out == ffi.NULL:
raise ValueError("Failed to parse CSV")
# Extract parsed columns
outcols = []
for i in range(csv_reader.num_cols_out):
if out[i].dtype == libgdf.GDF_STRING:
ptr = int(ffi.cast("uintptr_t", out[i].data))
outcols.append(nvstrings.bind_cpointer(ptr))
else:
newcol = Column.from_cffi_view(out[i])
if (newcol.dtype == np.dtype('datetime64[ms]')):
col = newcol.view(DatetimeColumn, dtype='datetime64[ms]')
else:
col = newcol.view(NumericalColumn, dtype=newcol.dtype)
outcols.append(Series(col))
return outcols
def read_csv(filepath_or_buffer,
lineterminator='\n',
quotechar='"',
quoting=0,
doublequote=True,
header='infer',
mangle_dupe_cols=True,
usecols=None,
sep=',',
delimiter=None,
delim_whitespace=False,
skipinitialspace=False,
names=None,
dtype=None,
skipfooter=0,
skiprows=0,
dayfirst=False,
compression='infer',
thousands=None,
decimal='.',
true_values=None,
false_values=None,
nrows=None,
byte_range=None,
skip_blank_lines=True,
comment=None,
na_values=None,
keep_default_na=True,
na_filter=True,
prefix=None,
index_col=None):
"""
Load and parse a CSV file into a DataFrame
Parameters
----------
filepath_or_buffer : str
Path of file to be read or a file-like object containing the file.
sep : char, default ','
Delimiter to be used.
delimiter : char, default None
Alternative argument name for sep.
delim_whitespace : bool, default False
Determines whether to use whitespace as delimiter.
lineterminator : char, default '\\n'
Character to indicate end of line.
skipinitialspace : bool, default False
Skip spaces after delimiter.
names : list of str, default None
List of column names to be used.
dtype : list of str or dict of {col: dtype}, default None
List of data types in the same order of the column names
or a dictionary with column_name:dtype (pandas style).
quotechar : char, default '"'
Character to indicate start and end of quote item.
quoting : str or int, default 0
Controls quoting behavior. Set to one of
0 (csv.QUOTE_MINIMAL), 1 (csv.QUOTE_ALL),
2 (csv.QUOTE_NONNUMERIC) or 3 (csv.QUOTE_NONE).
Quoting is enabled with all values except 3.
doublequote : bool, default True
When quoting is enabled, indicates whether to interpret two
consecutive quotechar inside fields as single quotechar
header : int, default 'infer'
Row number to use as the column names. Default behavior is to infer
the column names: if no names are passed, header=0;
if column names are passed explicitly, header=None.
usecols : list of int or str, default None
Returns subset of the columns given in the list. All elements must be
either integer indices (column number) or strings that correspond to
column names
mangle_dupe_cols : boolean, default True
Duplicate columns will be specified as 'X','X.1',...'X.N'.
skiprows : int, default 0
Number of rows to be skipped from the start of file.
skipfooter : int, default 0
Number of rows to be skipped at the bottom of file.
compression : {'infer', 'gzip', 'zip', None}, default 'infer'
For on-the-fly decompression of on-disk data. If ‘infer’, then detect
compression from the following extensions: ‘.gz’,‘.zip’ (otherwise no
decompression). If using ‘zip’, the ZIP file must contain only one
data file to be read in, otherwise the first non-zero-sized file will
be used. Set to None for no decompression.
decimal : char, default '.'
Character used as a decimal point.
thousands : char, default None
Character used as a thousands delimiter.
true_values : list, default None
Values to consider as boolean True
false_values : list, default None
Values to consider as boolean False
nrows : int, default None
If specified, maximum number of rows to read
byte_range : list or tuple, default None
Byte range within the input file to be read. The first number is the
offset in bytes, the second number is the range size in bytes. Set the
size to zero to read all data after the offset location. Reads the row
that starts before or at the end of the range, even if it ends after
the end of the range.
skip_blank_lines : bool, default True
If True, discard and do not parse empty lines
If False, interpret empty lines as NaN values
comment : char, default None
Character used as a comments indicator. If found at the beginning of a
line, the line will be ignored altogether.
na_values : list, default None
Values to consider as invalid
keep_default_na : bool, default True
Whether or not to include the default NA values when parsing the data.
na_filter : bool, default True
Detect missing values (empty strings and the values in na_values).
Passing False can improve performance.
prefix : str, default None
Prefix to add to column numbers when parsing without a header row
index_col : int or string, default None
Column to use as the row labels
Returns
-------
GPU ``DataFrame`` object.
Examples
--------
Create a test csv file
>>> import cudf
>>> filename = 'foo.csv'
>>> lines = [
... "num1,datetime,text",
... "123,2018-11-13T12:00:00,abc",
... "456,2018-11-14T12:35:01,def",
... "789,2018-11-15T18:02:59,ghi"
... ]
>>> with open(filename, 'w') as fp:
... fp.write('\\n'.join(lines)+'\\n')
Read the file with ``cudf.read_csv``
>>> cudf.read_csv(filename)
num1 datetime text
0 123 2018-11-13T12:00:00.000 5451
1 456 2018-11-14T12:35:01.000 5784
2 789 2018-11-15T18:02:59.000 6117
See Also
--------
.read_csv_strings
"""
if delim_whitespace:
if delimiter is not None:
raise ValueError("cannot set both delimiter and delim_whitespace")
if sep != ',':
raise ValueError("cannot set both sep and delim_whitespace")
# Alias sep -> delimiter.
if delimiter is None:
delimiter = sep
if dtype is not None:
if isinstance(dtype, collections.abc.Mapping):
dtype_dict = True
elif isinstance(dtype, collections.abc.Iterable):
dtype_dict = False
else:
msg = '''dtype must be 'list like' or 'dict' '''
raise TypeError(msg)
if names is not None and len(dtype) != len(names):
msg = '''All column dtypes must be specified.'''
raise TypeError(msg)
nvtx_range_push("CUDF_READ_CSV", "purple")
csv_reader = ffi.new('csv_read_arg*')
# Populate csv_reader struct
if is_file_like(filepath_or_buffer):
if compression == 'infer':
compression = None
buffer = filepath_or_buffer.read()
# check if StringIO is used
if hasattr(buffer, 'encode'):
buffer_as_bytes = buffer.encode()
else:
buffer_as_bytes = buffer
buffer_data_holder = ffi.new("char[]", buffer_as_bytes)
csv_reader.input_data_form = libgdf.HOST_BUFFER
csv_reader.filepath_or_buffer = buffer_data_holder
csv_reader.buffer_size = len(buffer_as_bytes)
else:
if (not os.path.isfile(filepath_or_buffer)):
raise (FileNotFoundError)
if (not os.path.exists(filepath_or_buffer)):
raise (FileNotFoundError)
file_path = _wrap_string(filepath_or_buffer)
csv_reader.input_data_form = libgdf.FILE_PATH
csv_reader.filepath_or_buffer = file_path
if header == 'infer':
header = -1
header_infer = header
arr_names = []
arr_dtypes = []
if names is None:
if header is -1:
header_infer = 0
if header is None:
header_infer = -1
csv_reader.names = ffi.NULL
csv_reader.num_cols = 0
else:
if header is None:
header_infer = -1
csv_reader.num_cols = len(names)
for col_name in names:
arr_names.append(_wrap_string(col_name))
if dtype is not None:
if dtype_dict:
arr_dtypes.append(_wrap_string(str(dtype[col_name])))
names_ptr = ffi.new('char*[]', arr_names)
csv_reader.names = names_ptr
if dtype is None:
csv_reader.dtype = ffi.NULL
else:
if not dtype_dict:
for col_dtype in dtype:
arr_dtypes.append(_wrap_string(str(col_dtype)))
dtype_ptr = ffi.new('char*[]', arr_dtypes)
csv_reader.dtype = dtype_ptr
csv_reader.use_cols_int = ffi.NULL
csv_reader.use_cols_int_len = 0
csv_reader.use_cols_char = ffi.NULL
csv_reader.use_cols_char_len = 0
if usecols is not None:
arr_col_names = []
if (all(isinstance(x, int) for x in usecols)):
usecols_ptr = ffi.new('int[]', usecols)
csv_reader.use_cols_int = usecols_ptr
csv_reader.use_cols_int_len = len(usecols)
else:
for col_name in usecols:
arr_col_names.append(_wrap_string(col_name))
col_names_ptr = ffi.new('char*[]', arr_col_names)
csv_reader.use_cols_char = col_names_ptr
csv_reader.use_cols_char_len = len(usecols)
if decimal == delimiter:
raise ValueError("decimal cannot be the same as delimiter")
if thousands == delimiter:
raise ValueError("thousands cannot be the same as delimiter")
if nrows is not None and skipfooter != 0:
raise ValueError("cannot use both nrows and skipfooter parameters")
if byte_range is not None:
if skipfooter != 0 or skiprows != 0 or nrows is not None:
raise ValueError("""cannot manually limit rows to be read when
using the byte range parameter""")
arr_true_values = []
for value in true_values or []:
arr_true_values.append(_wrap_string(str(value)))
arr_true_values_ptr = ffi.new('char*[]', arr_true_values)
csv_reader.true_values = arr_true_values_ptr
csv_reader.num_true_values = len(arr_true_values)
arr_false_values = []
for value in false_values or []:
arr_false_values.append(_wrap_string(str(value)))
false_values_ptr = ffi.new('char*[]', arr_false_values)
csv_reader.false_values = false_values_ptr
csv_reader.num_false_values = len(arr_false_values)
arr_na_values = []
for value in na_values or []:
arr_na_values.append(_wrap_string(str(value)))
arr_na_values_ptr = ffi.new('char*[]', arr_na_values)
csv_reader.na_values = arr_na_values_ptr
csv_reader.num_na_values = len(arr_na_values)
compression_bytes = _wrap_string(compression)
prefix_bytes = _wrap_string(prefix)
csv_reader.delimiter = delimiter.encode()
csv_reader.lineterminator = lineterminator.encode()
csv_reader.quotechar = quotechar.encode()
csv_reader.quoting = _quoting_enum[quoting]
csv_reader.doublequote = doublequote
csv_reader.delim_whitespace = delim_whitespace
csv_reader.skipinitialspace = skipinitialspace
csv_reader.dayfirst = dayfirst
csv_reader.header = header_infer
csv_reader.skiprows = skiprows
csv_reader.skipfooter = skipfooter
csv_reader.mangle_dupe_cols = mangle_dupe_cols
csv_reader.windowslinetermination = False
csv_reader.compression = compression_bytes
csv_reader.decimal = decimal.encode()
csv_reader.thousands = thousands.encode() if thousands else b'\0'
csv_reader.nrows = nrows if nrows is not None else -1
if byte_range is not None:
csv_reader.byte_range_offset = byte_range[0]
csv_reader.byte_range_size = byte_range[1]
else:
csv_reader.byte_range_offset = 0
csv_reader.byte_range_size = 0
csv_reader.skip_blank_lines = skip_blank_lines
csv_reader.comment = comment.encode() if comment else b'\0'
csv_reader.keep_default_na = keep_default_na
csv_reader.na_filter = na_filter
csv_reader.prefix = prefix_bytes
# Call read_csv
libgdf.read_csv(csv_reader)
out = csv_reader.data
if out == ffi.NULL:
raise ValueError("Failed to parse CSV")
# Extract parsed columns
outcols = []
new_names = []
for i in range(csv_reader.num_cols_out):
newcol = Column.from_cffi_view(out[i])
new_names.append(ffi.string(out[i].col_name).decode())
if (newcol.dtype == np.dtype('datetime64[ms]')):
outcols.append(newcol.view(DatetimeColumn, dtype='datetime64[ms]'))
else:
outcols.append(newcol.view(NumericalColumn, dtype=newcol.dtype))
# Build dataframe
df = DataFrame()
# if names is not None and header_infer is -1:
for k, v in zip(new_names, outcols):
df[k] = v
# Set index if the index_col parameter is passed
if index_col is not None and index_col is not False:
if isinstance(index_col, (int)):
df = df.set_index(df.columns[index_col])
else:
df = df.set_index(index_col)
nvtx_range_pop()
return df
def as_column(arbitrary, nan_as_null=True, dtype=None, name=None):
"""Create a Column from an arbitrary object
Currently support inputs are:
* ``Column``
* ``Buffer``
* ``Series``
* ``Index``
* numba device array
* cuda array interface
* numpy array
* pyarrow array
* pandas.Categorical
* Object exposing ``__cuda_array_interface__``
Returns
-------
result : subclass of TypedColumnBase
- CategoricalColumn for pandas.Categorical input.
- DatetimeColumn for datetime input.
- StringColumn for string input.
- NumericalColumn for all other inputs.
"""
from cudf.dataframe import numerical, categorical, datetime, string
from cudf.dataframe.series import Series
from cudf.dataframe.index import Index
from cudf.bindings.cudf_cpp import np_to_pa_dtype
if name is None and hasattr(arbitrary, "name"):
name = arbitrary.name
if isinstance(arbitrary, Column):
categories = None
if hasattr(arbitrary, "categories"):
categories = arbitrary.categories
data = build_column(
arbitrary.data,
arbitrary.dtype,
mask=arbitrary.mask,
categories=categories,
)
elif isinstance(arbitrary, Series):
data = arbitrary._column
if dtype is not None:
data = data.astype(dtype)
elif isinstance(arbitrary, Index):
data = arbitrary._values
if dtype is not None:
data = data.astype(dtype)
elif isinstance(arbitrary, Buffer):
data = numerical.NumericalColumn(data=arbitrary, dtype=arbitrary.dtype)
elif isinstance(arbitrary, nvstrings.nvstrings):
data = string.StringColumn(data=arbitrary)
elif cuda.devicearray.is_cuda_ndarray(arbitrary):
data = as_column(Buffer(arbitrary))
if (data.dtype in [np.float16, np.float32, np.float64]
and arbitrary.size > 0):
if nan_as_null:
mask = cudf.bindings.utils.mask_from_devary(data)
data = data.set_mask(mask)
elif hasattr(arbitrary, "__cuda_array_interface__"):
from cudf.bindings.cudf_cpp import count_nonzero_mask
desc = arbitrary.__cuda_array_interface__
data = _data_from_cuda_array_interface_desc(desc)
mask = _mask_from_cuda_array_interface_desc(desc)
if mask is not None:
nelem = len(data.mem)
nnz = count_nonzero_mask(mask.mem, size=nelem)
null_count = nelem - nnz
else:
null_count = 0
return build_column(data,
dtype=data.dtype,
mask=mask,
name=name,
null_count=null_count)
elif isinstance(arbitrary, np.ndarray):
# CUDF assumes values are always contiguous
if not arbitrary.flags["C_CONTIGUOUS"]:
arbitrary = np.ascontiguousarray(arbitrary)
if dtype is not None:
arbitrary = arbitrary.astype(dtype)
if arbitrary.dtype.kind == "M":
data = datetime.DatetimeColumn.from_numpy(arbitrary)
elif arbitrary.dtype.kind in ("O", "U"):
data = as_column(pa.Array.from_pandas(arbitrary))
else:
data = as_column(rmm.to_device(arbitrary), nan_as_null=nan_as_null)
elif isinstance(arbitrary, pa.Array):
if isinstance(arbitrary, pa.StringArray):
count = len(arbitrary)
null_count = arbitrary.null_count
buffers = arbitrary.buffers()
# Buffer of actual strings values
if buffers[2] is not None:
sbuf = np.frombuffer(buffers[2], dtype="int8")
else:
sbuf = np.empty(0, dtype="int8")
# Buffer of offsets values
obuf = np.frombuffer(buffers[1], dtype="int32")
# Buffer of null bitmask
nbuf = None
if null_count > 0:
nbuf = np.frombuffer(buffers[0], dtype="int8")
data = as_column(
nvstrings.from_offsets(sbuf,
obuf,
count,
nbuf=nbuf,
ncount=null_count))
elif isinstance(arbitrary, pa.NullArray):
new_dtype = pd.api.types.pandas_dtype(dtype)
if (type(dtype) == str and dtype == "empty") or dtype is None:
new_dtype = pd.api.types.pandas_dtype(
arbitrary.type.to_pandas_dtype())
if is_categorical_dtype(new_dtype):
arbitrary = arbitrary.dictionary_encode()
else:
if nan_as_null:
arbitrary = arbitrary.cast(np_to_pa_dtype(new_dtype))
else:
# casting a null array doesn't make nans valid
# so we create one with valid nans from scratch:
if new_dtype == np.dtype("object"):
arbitrary = utils.scalar_broadcast_to(
None, (len(arbitrary), ), dtype=new_dtype)
else:
arbitrary = utils.scalar_broadcast_to(
np.nan, (len(arbitrary), ), dtype=new_dtype)
data = as_column(arbitrary, nan_as_null=nan_as_null)
elif isinstance(arbitrary, pa.DictionaryArray):
pamask, padata = buffers_from_pyarrow(arbitrary)
data = categorical.CategoricalColumn(
data=padata,
mask=pamask,
null_count=arbitrary.null_count,
categories=arbitrary.dictionary,
ordered=arbitrary.type.ordered,
)
elif isinstance(arbitrary, pa.TimestampArray):
dtype = np.dtype("M8[{}]".format(arbitrary.type.unit))
pamask, padata = buffers_from_pyarrow(arbitrary, dtype=dtype)
data = datetime.DatetimeColumn(
data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=dtype,
)
elif isinstance(arbitrary, pa.Date64Array):
pamask, padata = buffers_from_pyarrow(arbitrary, dtype="M8[ms]")
data = datetime.DatetimeColumn(
data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=np.dtype("M8[ms]"),
)
elif isinstance(arbitrary, pa.Date32Array):
# No equivalent np dtype and not yet supported
warnings.warn(
"Date32 values are not yet supported so this will "
"be typecast to a Date64 value",
UserWarning,
)
data = as_column(arbitrary.cast(pa.int32())).astype("M8[ms]")
elif isinstance(arbitrary, pa.BooleanArray):
# Arrow uses 1 bit per value while we use int8
dtype = np.dtype(np.bool)
# Needed because of bug in PyArrow
# https://issues.apache.org/jira/browse/ARROW-4766
if len(arbitrary) > 0:
arbitrary = arbitrary.cast(pa.int8())
else:
arbitrary = pa.array([], type=pa.int8())
pamask, padata = buffers_from_pyarrow(arbitrary, dtype=dtype)
data = numerical.NumericalColumn(
data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=dtype,
)
else:
pamask, padata = buffers_from_pyarrow(arbitrary)
data = numerical.NumericalColumn(
data=padata,
mask=pamask,
null_count=arbitrary.null_count,
dtype=np.dtype(arbitrary.type.to_pandas_dtype()),
)
elif isinstance(arbitrary, pa.ChunkedArray):
gpu_cols = [
as_column(chunk, dtype=dtype) for chunk in arbitrary.chunks
]
if dtype and dtype != "empty":
new_dtype = dtype
else:
pa_type = arbitrary.type
if pa.types.is_dictionary(pa_type):
new_dtype = "category"
else:
new_dtype = np.dtype(pa_type.to_pandas_dtype())
data = Column._concat(gpu_cols, dtype=new_dtype)
elif isinstance(arbitrary, (pd.Series, pd.Categorical)):
if is_categorical_dtype(arbitrary):
data = as_column(pa.array(arbitrary, from_pandas=True))
elif arbitrary.dtype == np.bool:
# Bug in PyArrow or HDF that requires us to do this
data = as_column(pa.array(np.array(arbitrary), from_pandas=True))
else:
data = as_column(pa.array(arbitrary, from_pandas=nan_as_null))
elif isinstance(arbitrary, pd.Timestamp):
# This will always treat NaTs as nulls since it's not technically a
# discrete value like NaN
data = as_column(pa.array(pd.Series([arbitrary]), from_pandas=True))
elif np.isscalar(arbitrary) and not isinstance(arbitrary, memoryview):
if hasattr(arbitrary, "dtype"):
data_type = np_to_pa_dtype(arbitrary.dtype)
# PyArrow can't construct date64 or date32 arrays from np
# datetime types
if pa.types.is_date64(data_type) or pa.types.is_date32(data_type):
arbitrary = arbitrary.astype("int64")
data = as_column(pa.array([arbitrary], type=data_type))
else:
data = as_column(pa.array([arbitrary]), nan_as_null=nan_as_null)
elif isinstance(arbitrary, memoryview):
data = as_column(np.array(arbitrary),
dtype=dtype,
nan_as_null=nan_as_null)
else:
try:
data = as_column(memoryview(arbitrary),
dtype=dtype,
nan_as_null=nan_as_null)
except TypeError:
pa_type = None
np_type = None
try:
if dtype is not None:
dtype = pd.api.types.pandas_dtype(dtype)
if is_categorical_dtype(dtype):
raise TypeError
else:
np_type = np.dtype(dtype).type
if np_type == np.bool_:
pa_type = pa.bool_()
else:
pa_type = np_to_pa_dtype(np.dtype(dtype))
data = as_column(
pa.array(arbitrary, type=pa_type, from_pandas=nan_as_null),
dtype=dtype,
nan_as_null=nan_as_null,
)
except (pa.ArrowInvalid, pa.ArrowTypeError, TypeError):
if is_categorical_dtype(dtype):
data = as_column(
pd.Series(arbitrary, dtype="category"),
nan_as_null=nan_as_null,
)
else:
data = as_column(
np.array(arbitrary, dtype=np_type),
nan_as_null=nan_as_null,
)
if hasattr(data, "name") and (name is not None):
data.name = name
return data