def delete_dir(self, filesystem: FileSystem = None): """Delete dir from filesystem""" if not self.is_dir(filesystem=filesystem): raise FileNotFoundError(str(self)) if filesystem is not None: filesystem.rm(str(self), recursive=True) else: if self.is_hdfs: with HDFSContext() as hdfs: hdfs.rm(str(self), recursive=True) else: shutil.rmtree(str(self))
def delete(self, filesystem: FileSystem = None): """Delete file from filesystem""" if not self.is_file(filesystem=filesystem): raise FileNotFoundError(str(self)) if filesystem is not None: filesystem.delete(str(self)) else: if self.is_hdfs: with HDFSContext() as hdfs: hdfs.delete(str(self)) else: pathlib.Path(str(self)).unlink()
def mkdir(self, parents: bool = False, exist_ok: bool = False, filesystem: FileSystem = None): """Create directory""" if self.is_dir(filesystem=filesystem): if exist_ok: return else: raise Exception(f"Directory {self} already exists.") if filesystem is not None: filesystem.mkdir(str(self)) else: if self.is_hdfs: with HDFSFileSystem() as hdfs: hdfs.mkdir(str(self)) else: pathlib.Path(str(self)).mkdir(parents=parents, exist_ok=exist_ok)
def is_file(self, filesystem: FileSystem = None) -> bool: """Return True if the path points to a regular file.""" if filesystem is not None: return filesystem.isfile(str(self)) if self.is_hdfs: with HDFSContext() as hdfs: return hdfs.isfile(str(self)) return pathlib.Path(str(self)).is_file()
def exists(self, filesystem: FileSystem = None) -> bool: """Return True if the path points to an existing file or dir.""" if filesystem is not None: return filesystem.exists(str(self)) if self.is_hdfs: with HDFSContext() as hdfs: return hdfs.exists(str(self)) return pathlib.Path(str(self)).exists()
def __init__(self, filesystem: FileSystem, path: str, mode: str = "rb"): self.filesystem = filesystem self.path = path self.mode = mode self._file = filesystem.open(self.path, mode={ "r": "rb", "w": "wb" }.get(mode, mode))
def is_dir(self, filesystem: FileSystem = None) -> bool: """Return True if the path points to a regular directory.""" if filesystem is not None: return filesystem.isdir(str(self)) else: if self.is_hdfs: with HDFSFileSystem() as hdfs: return hdfs.isdir(str(self)) else: return pathlib.Path(str(self)).is_dir()
def iterdir(self, filesystem: FileSystem = None) -> Generator["Path", None, None]: """Retrieve directory content.""" if filesystem is not None: return (Path(path) for path in list(filesystem.ls(str(self)))) else: if self.is_hdfs: with HDFSFileSystem() as hdfs: return (Path(path) for path in list(hdfs.ls(str(self)))) else: return (Path(str(path)) for path in pathlib.Path(str(self)).iterdir())
def __init__(self, filesystem: FileSystem, path: str, mode: str = "rb", encoding: Optional[str] = "utf-8"): self.filesystem = filesystem self.path = path self.mode = mode self.encoding = None if "b" in mode else encoding self._file = filesystem.open(self.path, mode={ "r": "rb", "w": "wb" }.get(mode, mode))
def test_fspath(tempdir, use_legacy_dataset): # ARROW-12472 support __fspath__ objects without using str() path = tempdir / "test.parquet" table = pa.table({"a": [1, 2, 3]}) _write_table(table, path) fs_protocol_obj = util.FSProtocolClass(path) result = _read_table(fs_protocol_obj, use_legacy_dataset=use_legacy_dataset) assert result.equals(table) # combined with non-local filesystem raises with pytest.raises(TypeError): _read_table(fs_protocol_obj, filesystem=FileSystem())