def file_exists(cls, file_path: str) -> bool:
"""
Check if the `file_path` is valid.
Parameters
----------
file_path : str
String representing a path.
Returns
-------
bool
True if the path is valid.
"""
if isinstance(file_path, str):
match = S3_ADDRESS_REGEX.search(file_path)
if match is not None:
if file_path[0] == "S":
file_path = "{}{}".format("s", file_path[1:])
S3FS = import_optional_dependency(
"s3fs", "Module s3fs is required to read S3FS files."
)
from botocore.exceptions import NoCredentialsError
s3fs = S3FS.S3FileSystem(anon=False)
exists = False
try:
exists = len(s3fs.glob(file_path)) > 0 or exists
except NoCredentialsError:
pass
s3fs = S3FS.S3FileSystem(anon=True)
return exists or len(s3fs.glob(file_path)) > 0
return len(glob.glob(file_path)) > 0
def file_exists(cls, file_path):
"""
Check if `file_path` exists.
Parameters
----------
file_path : str
String that represents the path to the file (paths to S3 buckets
are also acceptable).
Returns
-------
bool
Whether file exists or not.
"""
if isinstance(file_path, str):
match = S3_ADDRESS_REGEX.search(file_path)
if match is not None:
if file_path[0] == "S":
file_path = "{}{}".format("s", file_path[1:])
S3FS = import_optional_dependency(
"s3fs", "Module s3fs is required to read S3FS files.")
from botocore.exceptions import NoCredentialsError
s3fs = S3FS.S3FileSystem(anon=False)
exists = False
try:
exists = s3fs.exists(file_path) or exists
except NoCredentialsError:
pass
s3fs = S3FS.S3FileSystem(anon=True)
return exists or s3fs.exists(file_path)
return os.path.exists(file_path)
def _read(cls, path, columns=None, **kwargs):
"""
Read data from the file path, returning a query compiler.
Parameters
----------
path : str or file-like object
The filepath of the feather file.
columns : array-like, optional
Columns to read from file. If not provided, all columns are read.
**kwargs : dict
`read_feather` function kwargs.
Returns
-------
BaseQueryCompiler
Query compiler with imported data for further processing.
Notes
-----
`PyArrow` engine and local files only are supported for now,
multi threading is set to False by default.
PyArrow feather is used. Please refer to the documentation here
https://arrow.apache.org/docs/python/api.html#feather-format
"""
path = cls.get_path(path)
if columns is None:
import_optional_dependency(
"pyarrow", "pyarrow is required to read feather files.")
from pyarrow.feather import read_feather
with OpenFile(
path,
**(kwargs.get("storage_options", None) or {}),
) as file:
df = read_feather(file)
# pyarrow.feather.read_feather doesn't support columns as pandas.Index
columns = list(df.columns)
return cls.build_query_compiler(path, columns, use_threads=False)
def get_path(cls, file_path: str) -> list:
"""
Return the path of the file(s).
Parameters
----------
file_path : str
String representing a path.
Returns
-------
list
List of strings of absolute file paths.
"""
if S3_ADDRESS_REGEX.search(file_path):
# S3FS does not allow captial S in s3 addresses.
if file_path[0] == "S":
file_path = "{}{}".format("s", file_path[1:])
S3FS = import_optional_dependency(
"s3fs", "Module s3fs is required to read S3FS files.")
from botocore.exceptions import NoCredentialsError
def get_file_path(fs_handle) -> List[str]:
file_paths = fs_handle.glob(file_path)
s3_addresses = [
"{}{}".format("s3://", path) for path in file_paths
]
return s3_addresses
s3fs = S3FS.S3FileSystem(anon=False)
try:
return get_file_path(s3fs)
except NoCredentialsError:
pass
s3fs = S3FS.S3FileSystem(anon=True)
return get_file_path(s3fs)
else:
relative_paths = glob.glob(file_path)
abs_paths = [os.path.abspath(path) for path in relative_paths]
return abs_paths
def _read(cls, path, engine, columns, **kwargs):
"""
Load a parquet object from the file path, returning a query compiler.
Parameters
----------
path : str, path object or file-like object
The filepath of the parquet file in local filesystem or hdfs.
engine : str
Parquet library to use (only 'PyArrow' is supported for now).
columns : list
If not None, only these columns will be read from the file.
**kwargs : dict
Keyword arguments.
Returns
-------
BaseQueryCompiler
A new Query Compiler.
Notes
-----
ParquetFile API is used. Please refer to the documentation here
https://arrow.apache.org/docs/python/parquet.html
"""
import_optional_dependency(
"pyarrow",
"pyarrow is required to read parquet files.",
)
from pyarrow.parquet import ParquetDataset
from modin.pandas.io import PQ_INDEX_REGEX
if isinstance(path, str) and os.path.isdir(path):
partitioned_columns = set()
# We do a tree walk of the path directory because partitioned
# parquet directories have a unique column at each directory level.
# Thus, we can use os.walk(), which does a dfs search, to walk
# through the different columns that the data is partitioned on
for (root, dir_names, files) in os.walk(path):
if dir_names:
partitioned_columns.add(dir_names[0].split("=")[0])
if files:
# Metadata files, git files, .DSStore
if files[0][0] == ".":
continue
break
partitioned_columns = list(partitioned_columns)
if len(partitioned_columns):
ErrorMessage.default_to_pandas("Mixed Partitioning Columns in Parquet")
return cls.single_worker_read(
path, engine=engine, columns=columns, **kwargs
)
if not columns:
import fsspec.core
from pandas.io.common import is_fsspec_url
fs, path_ = (
fsspec.core.url_to_fs(path, **(kwargs.get("storage_options") or {}))
if is_fsspec_url(path)
else (None, path)
)
dataset = ParquetDataset(path_, filesystem=fs, use_legacy_dataset=False)
column_names = dataset.schema.names
if dataset.schema.pandas_metadata is not None:
index_columns = dataset.schema.pandas_metadata.get("index_columns", [])
column_names = [c for c in column_names if c not in index_columns]
columns = [name for name in column_names if not PQ_INDEX_REGEX.match(name)]
return cls.build_query_compiler(path, columns, **kwargs)