def listdir(self, dirname): """Returns a list of entries contained within a directory.""" if not self.isdir(dirname): raise errors.NotFoundError(None, None, "Could not find directory") entries = os.listdir(compat.as_str_any(dirname)) entries = [compat.as_str_any(item) for item in entries] return entries
def walk(top, topdown=True, onerror=None): """Recursive directory tree generator for directories. Args: top: string, a Directory name topdown: bool, Traverse pre order if True, post order if False. onerror: optional handler for errors. Should be a function, it will be called with the error as argument. Rethrowing the error aborts the walk. Errors that happen while listing directories are ignored. Yields: Each yield is a 3-tuple: the pathname of a directory, followed by lists of all its subdirectories and leaf files. (dirname, [subdirname, subdirname, ...], [filename, filename, ...]) as strings """ top = compat.as_str_any(top) fs = get_filesystem(top) try: listing = listdir(top) except errors.NotFoundError as err: if onerror: onerror(err) else: return files = [] subdirs = [] for item in listing: full_path = fs.join(top, compat.as_str_any(item)) if isdir(full_path): subdirs.append(item) else: files.append(item) here = (top, subdirs, files) if topdown: yield here for subdir in subdirs: joined_subdir = fs.join(top, compat.as_str_any(subdir)) for subitem in walk(joined_subdir, topdown, onerror=onerror): yield subitem if not topdown: yield here
def glob(self, filename): """Returns a list of files that match the given pattern(s).""" if isinstance(filename, six.string_types): return [ # Convert the filenames to string from bytes. compat.as_str_any(matching_filename) for matching_filename in py_glob.glob(compat.as_bytes(filename)) ] else: return [ # Convert the filenames to string from bytes. compat.as_str_any(matching_filename) for single_filename in filename for matching_filename in py_glob.glob( compat.as_bytes(single_filename)) ]
def bucket_and_path(self, url): """Split an S3-prefixed URL into bucket and path.""" url = compat.as_str_any(url) if url.startswith("s3://"): url = url[len("s3://") :] idx = url.index("/") bucket = url[:idx] path = url[(idx + 1) :] return bucket, path
def get_filesystem(filename): """Return the registered filesystem for the given file.""" filename = compat.as_str_any(filename) prefix = "" index = filename.find("://") if index >= 0: prefix = filename[:index] fs = _REGISTERED_FILESYSTEMS.get(prefix, None) if fs is None: raise ValueError("No recognized filesystem for prefix %s" % prefix) return fs