def _read_text_full(
parser_func: Callable,
path_root: str,
path: str,
boto3_session: Union[boto3.Session, Dict[str, Optional[str]]],
pandas_args: Dict[str, Any],
s3_additional_kwargs: Optional[Dict[str, str]] = None,
dataset: bool = False,
) -> pd.DataFrame:
fs: s3fs.S3FileSystem = _utils.get_fs(
session=boto3_session, s3_additional_kwargs=s3_additional_kwargs)
if pandas_args.get("compression", "infer") == "infer":
pandas_args["compression"] = infer_compression(path,
compression="infer")
mode: str = "r" if pandas_args.get("compression") is None else "rb"
encoding: Optional[str] = pandas_args.get("encoding", None)
newline: Optional[str] = pandas_args.get("lineterminator", None)
with fs.open(path=path, mode=mode, encoding=encoding,
newline=newline) as f:
df: pd.DataFrame = parser_func(f, **pandas_args)
if dataset is True:
partitions: Dict[str, Any] = _utils.extract_partitions_from_path(
path_root=path_root, path=path)
for column_name, value in partitions.items():
df[column_name] = value
return df
def _get_write_details(path: str, pandas_kwargs: Dict[str, Any]) -> Tuple[str, Optional[str], Optional[str]]:
if pandas_kwargs.get("compression", "infer") == "infer":
pandas_kwargs["compression"] = infer_compression(path, compression="infer")
mode: str = "w" if pandas_kwargs.get("compression") is None else "wb"
encoding: Optional[str] = pandas_kwargs.get("encoding", None)
newline: Optional[str] = pandas_kwargs.get("lineterminator", "")
return mode, encoding, newline
def _read_text_chunksize(
parser_func: Callable,
path_root: str,
paths: List[str],
boto3_session: boto3.Session,
chunksize: int,
pandas_args: Dict[str, Any],
s3_additional_kwargs: Optional[Dict[str, str]] = None,
dataset: bool = False,
) -> Iterator[pd.DataFrame]:
fs: s3fs.S3FileSystem = _utils.get_fs(
session=boto3_session, s3_additional_kwargs=s3_additional_kwargs)
for path in paths:
_logger.debug("path: %s", path)
partitions: Dict[str, Any] = {}
if dataset is True:
partitions = _utils.extract_partitions_from_path(
path_root=path_root, path=path)
if pandas_args.get("compression", "infer") == "infer":
pandas_args["compression"] = infer_compression(path,
compression="infer")
mode: str = "r" if pandas_args.get("compression") is None else "rb"
with fs.open(path, mode) as f:
reader: pandas.io.parsers.TextFileReader = parser_func(
f, chunksize=chunksize, **pandas_args)
for df in reader:
if dataset is True:
for column_name, value in partitions.items():
df[column_name] = value
yield df
def read_json(
path_or_buf=None,
orient=None,
typ="frame",
dtype=None,
convert_axes=None,
convert_dates=True,
keep_default_dates=True,
numpy=False,
precise_float=False,
date_unit=None,
encoding=None,
lines=False,
chunksize=None,
compression="infer",
):
"""
Convert a JSON string to pandas object.
Parameters
----------
path_or_buf : a valid JSON str, path object or file-like object
Any valid string path is acceptable. The string could be a URL. Valid
URL schemes include http, ftp, s3, and file. For file URLs, a host is
expected. A local file could be:
``file://localhost/path/to/table.json``.
If you want to pass in a path object, pandas accepts any
``os.PathLike``.
By file-like object, we refer to objects with a ``read()`` method,
such as a file handler (e.g. via builtin ``open`` function)
or ``StringIO``.
orient : str
Indication of expected JSON string format.
Compatible JSON strings can be produced by ``to_json()`` with a
corresponding orient value.
The set of possible orients is:
- ``'split'`` : dict like
``{index -> [index], columns -> [columns], data -> [values]}``
- ``'records'`` : list like
``[{column -> value}, ... , {column -> value}]``
- ``'index'`` : dict like ``{index -> {column -> value}}``
- ``'columns'`` : dict like ``{column -> {index -> value}}``
- ``'values'`` : just the values array
The allowed and default values depend on the value
of the `typ` parameter.
* when ``typ == 'series'``,
- allowed orients are ``{'split','records','index'}``
- default is ``'index'``
- The Series index must be unique for orient ``'index'``.
* when ``typ == 'frame'``,
- allowed orients are ``{'split','records','index',
'columns','values', 'table'}``
- default is ``'columns'``
- The DataFrame index must be unique for orients ``'index'`` and
``'columns'``.
- The DataFrame columns must be unique for orients ``'index'``,
``'columns'``, and ``'records'``.
.. versionadded:: 0.23.0
'table' as an allowed value for the ``orient`` argument
typ : {'frame', 'series'}, default 'frame'
The type of object to recover.
dtype : bool or dict, default None
If True, infer dtypes; if a dict of column to dtype, then use those;
if False, then don't infer dtypes at all, applies only to the data.
For all ``orient`` values except ``'table'``, default is True.
.. versionchanged:: 0.25.0
Not applicable for ``orient='table'``.
convert_axes : bool, default None
Try to convert the axes to the proper dtypes.
For all ``orient`` values except ``'table'``, default is True.
.. versionchanged:: 0.25.0
Not applicable for ``orient='table'``.
convert_dates : bool or list of str, default True
If True then default datelike columns may be converted (depending on
keep_default_dates).
If False, no dates will be converted.
If a list of column names, then those columns will be converted and
default datelike columns may also be converted (depending on
keep_default_dates).
keep_default_dates : bool, default True
If parsing dates (convert_dates is not False), then try to parse the
default datelike columns.
A column label is datelike if
* it ends with ``'_at'``,
* it ends with ``'_time'``,
* it begins with ``'timestamp'``,
* it is ``'modified'``, or
* it is ``'date'``.
numpy : bool, default False
Direct decoding to numpy arrays. Supports numeric data only, but
non-numeric column and index labels are supported. Note also that the
JSON ordering MUST be the same for each term if numpy=True.
.. deprecated:: 1.0.0
precise_float : bool, default False
Set to enable usage of higher precision (strtod) function when
decoding string to double values. Default (False) is to use fast but
less precise builtin functionality.
date_unit : str, default None
The timestamp unit to detect if converting dates. The default behaviour
is to try and detect the correct precision, but if this is not desired
then pass one of 's', 'ms', 'us' or 'ns' to force parsing only seconds,
milliseconds, microseconds or nanoseconds respectively.
encoding : str, default is 'utf-8'
The encoding to use to decode py3 bytes.
lines : bool, default False
Read the file as a json object per line.
chunksize : int, optional
Return JsonReader object for iteration.
See the `line-delimited json docs
<https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#line-delimited-json>`_
for more information on ``chunksize``.
This can only be passed if `lines=True`.
If this is None, the file will be read into memory all at once.
compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None}, default 'infer'
For on-the-fly decompression of on-disk data. If 'infer', then use
gzip, bz2, zip or xz if path_or_buf is a string ending in
'.gz', '.bz2', '.zip', or 'xz', respectively, and no decompression
otherwise. If using 'zip', the ZIP file must contain only one data
file to be read in. Set to None for no decompression.
Returns
-------
Series or DataFrame
The type returned depends on the value of `typ`.
See Also
--------
DataFrame.to_json : Convert a DataFrame to a JSON string.
Series.to_json : Convert a Series to a JSON string.
Notes
-----
Specific to ``orient='table'``, if a :class:`DataFrame` with a literal
:class:`Index` name of `index` gets written with :func:`to_json`, the
subsequent read operation will incorrectly set the :class:`Index` name to
``None``. This is because `index` is also used by :func:`DataFrame.to_json`
to denote a missing :class:`Index` name, and the subsequent
:func:`read_json` operation cannot distinguish between the two. The same
limitation is encountered with a :class:`MultiIndex` and any names
beginning with ``'level_'``.
Examples
--------
>>> df = pd.DataFrame([['a', 'b'], ['c', 'd']],
... index=['row 1', 'row 2'],
... columns=['col 1', 'col 2'])
Encoding/decoding a Dataframe using ``'split'`` formatted JSON:
>>> df.to_json(orient='split')
'{"columns":["col 1","col 2"],
"index":["row 1","row 2"],
"data":[["a","b"],["c","d"]]}'
>>> pd.read_json(_, orient='split')
col 1 col 2
row 1 a b
row 2 c d
Encoding/decoding a Dataframe using ``'index'`` formatted JSON:
>>> df.to_json(orient='index')
'{"row 1":{"col 1":"a","col 2":"b"},"row 2":{"col 1":"c","col 2":"d"}}'
>>> pd.read_json(_, orient='index')
col 1 col 2
row 1 a b
row 2 c d
Encoding/decoding a Dataframe using ``'records'`` formatted JSON.
Note that index labels are not preserved with this encoding.
>>> df.to_json(orient='records')
'[{"col 1":"a","col 2":"b"},{"col 1":"c","col 2":"d"}]'
>>> pd.read_json(_, orient='records')
col 1 col 2
0 a b
1 c d
Encoding with Table Schema
>>> df.to_json(orient='table')
'{"schema": {"fields": [{"name": "index", "type": "string"},
{"name": "col 1", "type": "string"},
{"name": "col 2", "type": "string"}],
"primaryKey": "index",
"pandas_version": "0.20.0"},
"data": [{"index": "row 1", "col 1": "a", "col 2": "b"},
{"index": "row 2", "col 1": "c", "col 2": "d"}]}'
"""
if orient == "table" and dtype:
raise ValueError("cannot pass both dtype and orient='table'")
if orient == "table" and convert_axes:
raise ValueError("cannot pass both convert_axes and orient='table'")
if dtype is None and orient != "table":
dtype = True
if convert_axes is None and orient != "table":
convert_axes = True
if encoding is None:
encoding = "utf-8"
compression = infer_compression(path_or_buf, compression)
filepath_or_buffer, _, compression, should_close = get_filepath_or_buffer(
path_or_buf, encoding=encoding, compression=compression)
json_reader = JsonReader(
filepath_or_buffer,
orient=orient,
typ=typ,
dtype=dtype,
convert_axes=convert_axes,
convert_dates=convert_dates,
keep_default_dates=keep_default_dates,
numpy=numpy,
precise_float=precise_float,
date_unit=date_unit,
encoding=encoding,
lines=lines,
chunksize=chunksize,
compression=compression,
)
if chunksize:
return json_reader
result = json_reader.read()
if should_close:
filepath_or_buffer.close()
return result
def test_infer_compression_from_path(self, extension, expected, path_type):
path = path_type("foo/bar.csv" + extension)
compression = icom.infer_compression(path, compression="infer")
assert compression == expected
def __init__(
self,
obj,
path_or_buf: Optional[FilePathOrBuffer[str]] = None,
sep: str = ",",
na_rep: str = "",
float_format: Optional[str] = None,
cols=None,
header: Union[bool, Sequence[Hashable]] = True,
index: bool = True,
index_label: Optional[Union[bool, Hashable,
Sequence[Hashable]]] = None,
mode: str = "w",
encoding: Optional[str] = None,
compression: Union[str, Mapping[str, str], None] = "infer",
quoting: Optional[int] = None,
line_terminator="\n",
chunksize: Optional[int] = None,
quotechar='"',
date_format: Optional[str] = None,
doublequote: bool = True,
escapechar: Optional[str] = None,
decimal=".",
):
self.obj = obj
if path_or_buf is None:
path_or_buf = StringIO()
# Extract compression mode as given, if dict
compression, self.compression_args = get_compression_method(
compression)
self.path_or_buf, _, _, self.should_close = get_filepath_or_buffer(
path_or_buf, encoding=encoding, compression=compression, mode=mode)
self.sep = sep
self.na_rep = na_rep
self.float_format = float_format
self.decimal = decimal
self.header = header
self.index = index
self.index_label = index_label
self.mode = mode
if encoding is None:
encoding = "utf-8"
self.encoding = encoding
self.compression = infer_compression(self.path_or_buf, compression)
if quoting is None:
quoting = csvlib.QUOTE_MINIMAL
self.quoting = quoting
if quoting == csvlib.QUOTE_NONE:
# prevents crash in _csv
quotechar = None
self.quotechar = quotechar
self.doublequote = doublequote
self.escapechar = escapechar
self.line_terminator = line_terminator or os.linesep
self.date_format = date_format
self.has_mi_columns = isinstance(obj.columns, ABCMultiIndex)
# validate mi options
if self.has_mi_columns:
if cols is not None:
raise TypeError(
"cannot specify cols with a MultiIndex on the columns")
if cols is not None:
if isinstance(cols, ABCIndexClass):
cols = cols.to_native_types(
na_rep=na_rep,
float_format=float_format,
date_format=date_format,
quoting=self.quoting,
)
else:
cols = list(cols)
self.obj = self.obj.loc[:, cols]
# update columns to include possible multiplicity of dupes
# and make sure sure cols is just a list of labels
cols = self.obj.columns
if isinstance(cols, ABCIndexClass):
cols = cols.to_native_types(
na_rep=na_rep,
float_format=float_format,
date_format=date_format,
quoting=self.quoting,
)
else:
cols = list(cols)
# save it
self.cols = cols
# preallocate data 2d list
self.blocks = self.obj._data.blocks
ncols = sum(b.shape[0] for b in self.blocks)
self.data = [None] * ncols
if chunksize is None:
chunksize = (100000 // (len(self.cols) or 1)) or 1
self.chunksize = int(chunksize)
self.data_index = obj.index
if (isinstance(self.data_index, (ABCDatetimeIndex, ABCPeriodIndex))
and date_format is not None):
from pandas import Index
self.data_index = Index([
x.strftime(date_format) if notna(x) else ""
for x in self.data_index
])
self.nlevels = getattr(self.data_index, "nlevels", 1)
if not index:
self.nlevels = 0
def _iterparse_nodes(self, iterparse: Callable) -> list[dict[str, str | None]]:
"""
Iterparse xml nodes.
This method will read in local disk, decompressed XML files for elements
and underlying descendants using iterparse, a method to iterate through
an XML tree without holding entire XML tree in memory.
Raises
------
TypeError
* If `iterparse` is not a dict or its dict value is not list-like.
ParserError
* If `path_or_buffer` is not a physical, decompressed file on disk.
* If no data is returned from selected items in `iterparse`.
Notes
-----
Namespace URIs will be removed from return node values. Also,
elements with missing children or attributes in submitted list
will have optional keys filled with None values.
"""
dicts: list[dict[str, str | None]] = []
row: dict[str, str | None] | None = None
if not isinstance(self.iterparse, dict):
raise TypeError(
f"{type(self.iterparse).__name__} is not a valid type for iterparse"
)
row_node = next(iter(self.iterparse.keys())) if self.iterparse else ""
if not is_list_like(self.iterparse[row_node]):
raise TypeError(
f"{type(self.iterparse[row_node])} is not a valid type "
"for value in iterparse"
)
if (
not isinstance(self.path_or_buffer, str)
or is_url(self.path_or_buffer)
or is_fsspec_url(self.path_or_buffer)
or self.path_or_buffer.startswith(("<?xml", "<"))
or infer_compression(self.path_or_buffer, "infer") is not None
):
raise ParserError(
"iterparse is designed for large XML files that are fully extracted on "
"local disk and not as compressed files or online sources."
)
for event, elem in iterparse(self.path_or_buffer, events=("start", "end")):
curr_elem = elem.tag.split("}")[1] if "}" in elem.tag else elem.tag
if event == "start":
if curr_elem == row_node:
row = {}
if row is not None:
if self.names:
for col, nm in zip(self.iterparse[row_node], self.names):
if curr_elem == col:
elem_val = elem.text.strip() if elem.text else None
if row.get(nm) != elem_val and nm not in row:
row[nm] = elem_val
if col in elem.attrib:
if elem.attrib[col] not in row.values() and nm not in row:
row[nm] = elem.attrib[col]
else:
for col in self.iterparse[row_node]:
if curr_elem == col:
row[col] = elem.text.strip() if elem.text else None
if col in elem.attrib:
row[col] = elem.attrib[col]
if event == "end":
if curr_elem == row_node and row is not None:
dicts.append(row)
row = None
elem.clear()
if hasattr(elem, "getprevious"):
while (
elem.getprevious() is not None and elem.getparent() is not None
):
del elem.getparent()[0]
if dicts == []:
raise ParserError("No result from selected items in iterparse.")
keys = list(dict.fromkeys([k for d in dicts for k in d.keys()]))
dicts = [{k: d[k] if k in d.keys() else None for k in keys} for d in dicts]
if self.names:
dicts = [{nm: v for nm, v in zip(self.names, d.values())} for d in dicts]
return dicts
def read_into_dataframe(file: IO,
filename: str = "",
nrows: int = 100,
max_characters: int = 50) -> pd.DataFrame:
"""Reads a file into a DataFrame.
Infers the file encoding and whether a header column exists
Args:
file (IO): file buffer.
filename (str): filename. Used to infer compression.
nrows (int, optional): number of rows to peek. Default: 100.
max_characters (int, optional): max characters a column name can have to be distinguished from a real text value
Returns:
A pandas.DataFrame.
"""
detector = UniversalDetector()
for line, text in enumerate(file):
detector.feed(text)
if detector.done or line > nrows:
break
detector.close()
encoding = detector.result.get("encoding")
compression = infer_compression(filename, "infer")
file.seek(0, SEEK_SET)
contents = file.read()
with BytesIO(contents) as file:
df0 = pd.read_csv(
file,
encoding=encoding,
compression=compression,
sep=None,
engine="python",
header="infer",
nrows=nrows,
)
df0_cols = list(df0.columns)
#Check if all columns are strins and short strings(text values tend to be long)
column_names_checker = all([type(item) == str for item in df0_cols])
if column_names_checker:
column_names_checker = all(
[len(item) < max_characters for item in df0_cols])
#Check if any column can be turned to float
conversion_checker = True
for item in df0_cols:
try:
item = float(item)
conversion_checker = False
break
except:
pass
#Prefix and header
final_checker = True if (column_names_checker
and conversion_checker) else False
header = "infer" if final_checker else None
prefix = None if header else "col"
with BytesIO(contents) as file:
df = pd.read_csv(
file,
encoding=encoding,
compression=compression,
sep=None,
engine="python",
header=header,
prefix=prefix,
)
return df
def read_csv(
filepath_or_buffer,
sep=",",
delimiter=None,
header="infer",
names=None,
index_col=None,
usecols=None,
prefix=None,
mangle_dupe_cols=True,
dtype=None,
true_values=None,
false_values=None,
skiprows=None,
skipfooter=0,
nrows=None,
na_values=None,
skip_blank_lines=True,
parse_dates=False,
compression="infer",
quotechar='"',
quoting=0,
doublequote=True,
verify_header=False,
**kwargs,
# TODO: Put back these options once we figure out how to support them
# with the Arrows CSV reader.
# skipinitialspace=False, # GPU only
# keep_default_na=True, # GPU only
# na_filter=True, # GPU only
# dayfirst=False, # GPU only
# thousands=None, # GPU only
# decimal=".", # GPU only
# lineterminator=None, # GPU only
# comment=None, # GPU only
# delim_whitespace=False, # GPU only
):
# Checks on filepath_or_buffer
paths = util.to_list_if_scalar(filepath_or_buffer)
if any(not isinstance(path, str) for path in paths):
raise err._unsupported_error(
"'filepath_or_buffer' must be a string or a list of strings")
if len(paths) == 0:
raise ValueError("'filepath_or_buffer' must be a non-empty list")
for path in paths:
if not os.path.exists(path):
raise ValueError(f"{path} does not exist")
if not isinstance(compression, str):
raise err._unsupported_error("compression", compression)
compressions = [
_parse_compression(infer_compression(path, compression))
for path in paths
]
# Checks on sep and delimiter
if sep is None and delimiter is None:
raise ValueError("at least one of 'sep' or 'delimiter' must be given")
sep = delimiter if delimiter is not None else sep
if len(sep) > 1:
raise ValueError("'sep' must be a 1-character string")
# Checks on sep and delimiter
if header == "infer":
header = 0 if names is None else None
if header not in (
0,
None,
):
raise err._unsupported_error("header", header)
# Checks on skiprows, kipfooter, and nrows
skiprows = 0 if skiprows is None else skiprows
if not is_integer(skiprows):
raise ValueError("'skiprows' must be an integer")
if not is_integer(skipfooter):
raise ValueError("'skipfooter' must be an integer")
if not (nrows is None or is_integer(nrows)):
raise ValueError("'nrows' must be None or an integer")
# If either column names or dtype is missing, infer them by parsing
# the first few of lines using Pandas
# FIXME: We should use cuDF for this
if names is None or dtype is None:
engine = ("python" if skipfooter > 0 else "c", )
column_names, dtypes = _extract_header_using_pandas(
paths[0],
sep,
header,
names,
dtype,
true_values,
false_values,
skiprows,
na_values,
skip_blank_lines,
parse_dates,
compression,
quotechar,
quoting,
doublequote,
engine,
peek_rows=3,
)
if verify_header:
for path in paths[1:]:
result = _extract_header_using_pandas(
path,
sep,
header,
names,
dtype,
true_values,
false_values,
skiprows,
na_values,
skip_blank_lines,
parse_dates,
compression,
quotechar,
quoting,
doublequote,
engine,
peek_rows=3,
)
if not column_names.equals(result[0]):
raise ValueError(
f"{paths[0]} and {path} have different headers")
else:
column_names = pandas.Index(names)
if is_dict_like(dtype):
dtypes = []
for name in names:
if name not in dtype:
raise ValueError(f"'dtype' has no entry for '{name}'")
dtypes.append(_ensure_dtype(dtype[name]))
elif is_list_like(dtype):
raise err._unsupported_error(
"'dtype' must be a string, a dtype, or a dictionary")
else:
dtype = _ensure_dtype(dtype)
dtypes = [dtype] * len(names)
if column_names.has_duplicates:
raise ValueError("Header must not have any duplicates")
# Checks on unsupported options
if prefix is not None:
raise err._unsupported_error("prefix", prefix)
if mangle_dupe_cols not in (True, ):
raise err._unsupported_error("mangle_dupe_cols", mangle_dupe_cols)
# If there was a header in the file, we should skip that line as well
if header == 0:
skiprows += 1
# Checks on parse_dates
_ERR_MSG_PARSE_DATES = (
"'parse_dates' must be a list of integers or strings for now")
if is_dict_like(parse_dates):
raise err._unsupported_error(_ERR_MSG_PARSE_DATES)
parse_dates = parse_dates if parse_dates is not False else []
if not is_list_like(parse_dates):
raise err._unsupported_error(_ERR_MSG_PARSE_DATES)
date_cols = _get_indexer(column_names, parse_dates, "parse_dates")
# Override dtypes for the datetime columns
for idx in date_cols:
dtypes[idx] = ty.ts_ns
# If a column is given a datetime dtype but not added to the parse_dates,
# we should record it
for idx, dtype in enumerate(dtypes):
if idx not in parse_dates:
parse_dates.append(idx)
# Checks on quoting
if quoting != 0:
raise err._unsupported_error("quoting", quoting)
if len(quotechar) > 1:
raise ValueError("'quotechar' must be a 1-character string")
# Checks on index_col
index_col = None if index_col is False else index_col
if index_col is not None:
if is_integer(index_col) or isinstance(index_col, str):
index_col = [index_col]
if not is_list_like(index_col):
raise err._unsupported_error("index_col", index_col)
index_col = _get_indexer(column_names, index_col, "index_col")
# Checks on true_values, false_values, and na_values
_check_string_list(true_values, "true_values")
_check_string_list(false_values, "false_values")
_check_string_list(na_values, "na_values")
# Checks on nrows
if skipfooter != 0 and nrows is not None:
raise ValueError("'skipfooter' not supported with 'nrows'")
df = DataFrame(
frame=io.read_csv(
paths,
sep=sep,
usecols=usecols,
dtypes=dtypes,
true_values=true_values,
false_values=false_values,
skiprows=skiprows,
skipfooter=skipfooter,
nrows=nrows,
na_values=na_values,
skip_blank_lines=skip_blank_lines,
date_cols=date_cols,
compressions=compressions,
quotechar=quotechar,
quoting=quoting,
doublequote=doublequote,
),
columns=column_names,
)
if index_col is not None:
df = df.set_index(column_names[index_col])
# Make sure we reset the names for unnamed indices
names = df._raw_index.names
names = [
None if name.startswith("Unnamed") else name for name in names
]
df._raw_index.names = names
return df
def read_into_dataframe(file, filename=None, nrows=100, max_characters=50):
"""
Reads a file into a DataFrame.
Infers the file encoding and whether a header column exists
The file can be in any format (.csv, .txt, .zip, .gif,...).
If it's not a .csv file, it will throw an exception (pandas.errors.EmptyDataError).
One-column .csv gives exception there in try...except.
Parameters
----------
file : IO
File buffer.
filename : str
Filename. Used to infer compression. Default to None.
nrows : int
Number of rows to peek. Default to 100.
max_characters : int
Max characters a column name can have to be distinguished from a real text value. Default to 50.
Returns
-------
pd.DataFrame
The dataframe content.
Raises
------
pandas.errors.EmptyDataError
Notes
-----
If no filename is given, a hex uuid will be used as the file name.
"""
detector = UniversalDetector()
for line, text in enumerate(file):
detector.feed(text)
if detector.done or line > nrows:
break
detector.close()
encoding = detector.result.get("encoding")
if filename is None:
filename = uuid4().hex
compression = infer_compression(filename, "infer")
file.seek(0, SEEK_SET)
pdread = TextIOWrapper(file, encoding=encoding)
try:
# check if the file has header.
sniffer = csv.Sniffer()
pdread.seek(0, SEEK_SET)
pdreadline = pdread.readline()
pdreadline += pdread.readline()
has_header = sniffer.has_header(pdreadline)
sep = None
except csv.Error:
sep = ","
has_header = True
# Prefix and header
header = "infer" if has_header else None
prefix = None if header else "col"
pdread.seek(0, SEEK_SET)
df = pd.read_csv(
pdread,
encoding=encoding,
compression=compression,
sep=sep,
engine="python",
header=header,
nrows=nrows,
prefix=prefix,
)
return df
