def delete_dir(self, filesystem: FileSystem = None):
"""Delete dir from filesystem"""
if not self.is_dir(filesystem=filesystem):
raise FileNotFoundError(str(self))
if filesystem is not None:
filesystem.rm(str(self), recursive=True)
else:
if self.is_hdfs:
with HDFSContext() as hdfs:
hdfs.rm(str(self), recursive=True)
else:
shutil.rmtree(str(self))
def delete(self, filesystem: FileSystem = None):
"""Delete file from filesystem"""
if not self.is_file(filesystem=filesystem):
raise FileNotFoundError(str(self))
if filesystem is not None:
filesystem.delete(str(self))
else:
if self.is_hdfs:
with HDFSContext() as hdfs:
hdfs.delete(str(self))
else:
pathlib.Path(str(self)).unlink()
def mkdir(self, parents: bool = False, exist_ok: bool = False, filesystem: FileSystem = None):
"""Create directory"""
if self.is_dir(filesystem=filesystem):
if exist_ok:
return
else:
raise Exception(f"Directory {self} already exists.")
if filesystem is not None:
filesystem.mkdir(str(self))
else:
if self.is_hdfs:
with HDFSFileSystem() as hdfs:
hdfs.mkdir(str(self))
else:
pathlib.Path(str(self)).mkdir(parents=parents, exist_ok=exist_ok)
def is_file(self, filesystem: FileSystem = None) -> bool:
"""Return True if the path points to a regular file."""
if filesystem is not None:
return filesystem.isfile(str(self))
if self.is_hdfs:
with HDFSContext() as hdfs:
return hdfs.isfile(str(self))
return pathlib.Path(str(self)).is_file()
def exists(self, filesystem: FileSystem = None) -> bool:
"""Return True if the path points to an existing file or dir."""
if filesystem is not None:
return filesystem.exists(str(self))
if self.is_hdfs:
with HDFSContext() as hdfs:
return hdfs.exists(str(self))
return pathlib.Path(str(self)).exists()
def __init__(self, filesystem: FileSystem, path: str, mode: str = "rb"):
self.filesystem = filesystem
self.path = path
self.mode = mode
self._file = filesystem.open(self.path,
mode={
"r": "rb",
"w": "wb"
}.get(mode, mode))
def is_dir(self, filesystem: FileSystem = None) -> bool:
"""Return True if the path points to a regular directory."""
if filesystem is not None:
return filesystem.isdir(str(self))
else:
if self.is_hdfs:
with HDFSFileSystem() as hdfs:
return hdfs.isdir(str(self))
else:
return pathlib.Path(str(self)).is_dir()
def iterdir(self, filesystem: FileSystem = None) -> Generator["Path", None, None]:
"""Retrieve directory content."""
if filesystem is not None:
return (Path(path) for path in list(filesystem.ls(str(self))))
else:
if self.is_hdfs:
with HDFSFileSystem() as hdfs:
return (Path(path) for path in list(hdfs.ls(str(self))))
else:
return (Path(str(path)) for path in pathlib.Path(str(self)).iterdir())
def __init__(self,
filesystem: FileSystem,
path: str,
mode: str = "rb",
encoding: Optional[str] = "utf-8"):
self.filesystem = filesystem
self.path = path
self.mode = mode
self.encoding = None if "b" in mode else encoding
self._file = filesystem.open(self.path,
mode={
"r": "rb",
"w": "wb"
}.get(mode, mode))
def test_fspath(tempdir, use_legacy_dataset):
# ARROW-12472 support __fspath__ objects without using str()
path = tempdir / "test.parquet"
table = pa.table({"a": [1, 2, 3]})
_write_table(table, path)
fs_protocol_obj = util.FSProtocolClass(path)
result = _read_table(fs_protocol_obj,
use_legacy_dataset=use_legacy_dataset)
assert result.equals(table)
# combined with non-local filesystem raises
with pytest.raises(TypeError):
_read_table(fs_protocol_obj, filesystem=FileSystem())