def walk(node, forward=True, onerror=None):
"""
Traverse path for directory and file tree.
Read from the buffer first.If there is not enough data in the buffer,
data will be read from the file system.
Args:
node (str): Current path.
forward (bool): If True, it will return the sub-directories and files in the top-level
directory first and then iterate the files in the sub-directories. Default: True.
onerror (Optional[Callable]): If None, it indicates that errors during file traversal
will be ignored. Default: None.
Yields:
Tuple, (node, sub_dirs, files).
"""
logger.debug(
"The walk method enter, param: node=%s, "
"forward=%s, onerror=%s.", node, forward, type(onerror))
file_system = FileHandler.get_file_system(node)
node = to_str(node)
dirs = []
try:
dirs = file_system.list_dir(node)
except PathNotExistError as err:
if onerror:
onerror(err)
else:
logger.warning("Get dir list error, dir_path=%s error=%s.",
node, str(err))
return
sub_dirs, files = [], []
for item in dirs:
full_path = file_system.join(node, to_str(item))
if file_system.is_dir(full_path):
sub_dirs.append(item)
else:
files.append(item)
result = (node, sub_dirs, files)
if forward:
logger.debug("The walk method return, pre result=%s.", result)
yield result
for subdir in sub_dirs:
joined_subdir = file_system.join(node, to_str(subdir))
for sub_results in FileHandler.walk(joined_subdir, forward,
onerror):
yield sub_results
if not forward:
logger.debug("The walk method return, post result=%s.", result)
yield result
def read_access(file_path):
"""
Determine if it has read permission.
Args:
file_path (str): File path.
Returns:
bool, if it has read permission, return True.
"""
return os.access(to_str(file_path), os.R_OK)
def exists(self, path):
"""
Determine if it exists.
Args:
path (str): Directory path or file path.
Returns:
bool, if it exists, return True.
"""
return os.path.exists(to_str(path))
def is_file(self, path):
"""
Determine if it is a file.
Args:
path (str): Directory path or file path.
Returns:
bool, if it is a file path, return True.
"""
return os.path.isfile(to_str(path))
def list_dir(self, path):
"""
List directories by path.
Args:
path (str): Directory path or file path.
Returns:
list[str], directories.
"""
path = to_str(path)
if not self.is_dir(path):
raise exceptions.PathNotDirectoryError("Path is %s." % path)
return os.listdir(path)
def file_stat(self, file_path):
"""
Get file stat information.
Args:
file_path (str): File path.
Returns:
Nametuple, the (size, mtime) of file.
"""
try:
file_info = os.stat(to_str(file_path))
except OSError:
raise PathNotExistError("File %s is not exist." % file_path)
return StatInfo(size=file_info.st_size, mtime=file_info.st_mtime)
def get_file_system(path):
"""
Get file system object from path.
Args:
path (str): Directory path or file path.
Returns:
BaseFileSystem, a file system object.
"""
path = to_str(path)
prefix_index = path.find("://")
prefix = path[:prefix_index] if prefix_index >= 0 else ""
file_system = _FILE_SYSTEMS.get(prefix, None)
if file_system is None:
raise ValueError("No filesystem can be found for prefix %s" %
prefix)
return file_system
def __init__(self, file_path, mode='rb'):
"""
Init FileHandler.
Args:
file_path (str): File path.
mode (Literal['r', 'rb', 'br', 'w', 'wb', 'bw']): It must be
in ['r', 'rb', 'br', 'w', 'wb', 'bw'].
"""
logger.debug(
"The __init__ method enter, param: file_path=%s"
"mode=%s", file_path, mode)
if mode not in ('r', 'rb', 'br', 'w', 'wb', 'bw'):
raise ValueError("mode %s is not supported by FileHandler." % mode)
self._file_path = to_str(file_path)
self._file_system = self.get_file_system(self._file_path)
self._buff_chunk_size = _DEFAULT_BUFFER_SIZE
self._buff = None
self._buff_offset = 0
self._offset = 0
self._binary_mode = 'b' in mode