def listdir(self, dirname):
"""Returns a list of entries contained within a directory."""
if not self.isdir(dirname):
raise errors.NotFoundError(None, None, "Could not find directory")
entries = os.listdir(compat.as_str_any(dirname))
entries = [compat.as_str_any(item) for item in entries]
return entries
def walk(top, topdown=True, onerror=None):
"""Recursive directory tree generator for directories.
Args:
top: string, a Directory name
topdown: bool, Traverse pre order if True, post order if False.
onerror: optional handler for errors. Should be a function, it will be
called with the error as argument. Rethrowing the error aborts the walk.
Errors that happen while listing directories are ignored.
Yields:
Each yield is a 3-tuple: the pathname of a directory, followed by lists
of all its subdirectories and leaf files.
(dirname, [subdirname, subdirname, ...], [filename, filename, ...])
as strings
"""
top = compat.as_str_any(top)
fs = get_filesystem(top)
try:
listing = listdir(top)
except errors.NotFoundError as err:
if onerror:
onerror(err)
else:
return
files = []
subdirs = []
for item in listing:
full_path = fs.join(top, compat.as_str_any(item))
if isdir(full_path):
subdirs.append(item)
else:
files.append(item)
here = (top, subdirs, files)
if topdown:
yield here
for subdir in subdirs:
joined_subdir = fs.join(top, compat.as_str_any(subdir))
for subitem in walk(joined_subdir, topdown, onerror=onerror):
yield subitem
if not topdown:
yield here
def glob(self, filename):
"""Returns a list of files that match the given pattern(s)."""
if isinstance(filename, six.string_types):
return [
# Convert the filenames to string from bytes.
compat.as_str_any(matching_filename) for matching_filename in
py_glob.glob(compat.as_bytes(filename))
]
else:
return [
# Convert the filenames to string from bytes.
compat.as_str_any(matching_filename)
for single_filename in filename
for matching_filename in py_glob.glob(
compat.as_bytes(single_filename))
]
def bucket_and_path(self, url):
"""Split an S3-prefixed URL into bucket and path."""
url = compat.as_str_any(url)
if url.startswith("s3://"):
url = url[len("s3://") :]
idx = url.index("/")
bucket = url[:idx]
path = url[(idx + 1) :]
return bucket, path
def get_filesystem(filename):
"""Return the registered filesystem for the given file."""
filename = compat.as_str_any(filename)
prefix = ""
index = filename.find("://")
if index >= 0:
prefix = filename[:index]
fs = _REGISTERED_FILESYSTEMS.get(prefix, None)
if fs is None:
raise ValueError("No recognized filesystem for prefix %s" % prefix)
return fs
