def walk(self, path="/", wildcard=None, dir_wildcard=None,
search="breadth", ignore_errors=False):
if dir_wildcard is not None:
# If there is a dir_wildcard, fall back to the default impl
# that uses listdir(). Otherwise we run the risk of enumerating
# lots of directories that will just be thrown away.
for item in super(HideFS, self).walk(path, wildcard, dir_wildcard,
search, ignore_errors):
yield item
# Otherwise, the wrapped FS may provide a more efficient impl
# which we can use directly.
else:
if wildcard is not None and not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn: bool(wildcard_re.match(fn))
walk = self.wrapped_fs.walk(self._encode(path), search=search,
ignore_errors=ignore_errors)
for (dirpath, filepaths) in walk:
if self.is_hidden(dirpath):
continue
filepaths = [basename(self._decode(pathcombine(dirpath, p)))
for p in filepaths]
dirpath = abspath(self._decode(dirpath))
if wildcard is not None:
filepaths = [p for p in filepaths if wildcard(p)]
yield (dirpath, filepaths)
def walk(self, path="/", wildcard=None, dir_wildcard=None,
search="breadth", ignore_errors=False):
if dir_wildcard is not None:
# If there is a dir_wildcard, fall back to the default impl
# that uses listdir(). Otherwise we run the risk of enumerating
# lots of directories that will just be thrown away.
for item in super(HideFS, self).walk(path, wildcard, dir_wildcard,
search, ignore_errors):
yield item
# Otherwise, the wrapped FS may provide a more efficient impl
# which we can use directly.
else:
if wildcard is not None and not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn: bool(wildcard_re.match(fn))
walk = self.wrapped_fs.walk(self._encode(path), search=search,
ignore_errors=ignore_errors)
for (dirpath, filepaths) in walk:
if self.is_hidden(dirpath):
continue
filepaths = [basename(self._decode(pathcombine(dirpath, p)))
for p in filepaths]
dirpath = abspath(self._decode(dirpath))
if wildcard is not None:
filepaths = [p for p in filepaths if wildcard(p)]
yield (dirpath, filepaths)
def _ilistdir_helper(self,
path,
entries,
wildcard=None,
full=False,
absolute=False,
dirs_only=False,
files_only=False):
"""A helper method called by ilistdir method that applies filtering.
Given the path to a directory and a list of the names of entries within
that directory, this method applies the semantics of the ilistdir()
keyword arguments. An appropriately modified and filtered list of
directory entries is returned.
"""
path = normpath(path)
if dirs_only and files_only:
raise ValueError("dirs_only and files_only cannot both be True")
if wildcard is not None:
if not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
def wildcard(fn):
return bool(wildcard_re.match(fn))
entries = (p for p in entries if wildcard(p.name))
if dirs_only:
entries = (p for p in entries
if self.isdir(p.name, _statobj=p.statinfo))
elif files_only:
entries = (p for p in entries
if self.isfile(p.name, _statobj=p.statinfo))
if full:
entries = (pathcombine(path, p.name) for p in entries)
elif absolute:
path = self._p(path)
entries = ((pathcombine(path, p.name)) for p in entries)
else:
entries = (p.name for p in entries)
return entries
def copydir(self, src, dst, overwrite=False, parallel=True):
"""Copy a directory from source to destination.
By default the copy is done by recreating the source directory
structure at the destination, and then copy files in parallel from
source to destination.
:param src: Source directory path.
:type src: str
:param dst: Destination directory path.
:type dst: str
:param overwrite: If True then any existing files in the destination
directory will be overwritten.
:type overwrite: bool
:param parallel: If True (default), the copy will be done in parallel.
:type parallel: bool
"""
if not self.isdir(src):
if self.isfile(src):
raise ResourceInvalidError(
src, msg="Source is not a directory: %(path)s")
raise ResourceNotFoundError(src)
if self.exists(dst):
if overwrite:
if self.isdir(dst):
self.removedir(dst, force=True)
elif self.isfile(dst):
self.remove(dst)
else:
raise DestinationExistsError(dst)
if parallel:
process = CopyProcess()
def process_copy(src, dst, overwrite=False):
process.add_job(src, dst)
copyfile = process_copy
else:
copyfile = self.copy
self.makedir(dst, allow_recreate=True)
for src_dirpath, filenames in self.walk(src):
dst_dirpath = pathcombine(dst, frombase(src, src_dirpath))
self.makedir(dst_dirpath, allow_recreate=True, recursive=True)
for filename in filenames:
src_filename = pathjoin(src_dirpath, filename)
dst_filename = pathjoin(dst_dirpath, filename)
copyfile(src_filename, dst_filename, overwrite=overwrite)
if parallel:
process.prepare()
process.run()
return True
def copydir(self, src, dst, overwrite=False, parallel=True):
"""Copy a directory from source to destination.
By default the copy is done by recreating the source directory
structure at the destination, and then copy files in parallel from
source to destination.
:param src: Source directory path.
:type src: string
:param dst: Destination directory path.
:type dst: string
:param overwrite: If True then any existing files in the destination
directory will be overwritten.
:type overwrite: bool
:param parallel: If True (default), the copy will be done in parallel.
:type parallel: bool
"""
if not self.isdir(src):
if self.isfile(src):
raise ResourceInvalidError(
src, msg="Source is not a directory: %(path)s")
raise ResourceNotFoundError(src)
if self.exists(dst):
if overwrite:
if self.isdir(dst):
self.removedir(dst, force=True)
elif self.isfile(dst):
self.remove(dst)
else:
raise DestinationExistsError(dst)
if parallel:
process = CopyProcess()
def process_copy(src, dst, overwrite=False):
process.add_job(src, dst)
copyfile = process_copy
else:
copyfile = self.copy
self.makedir(dst, allow_recreate=True)
for src_dirpath, filenames in self.walk(src):
dst_dirpath = pathcombine(dst, frombase(src, src_dirpath))
self.makedir(dst_dirpath, allow_recreate=True, recursive=True)
for filename in filenames:
src_filename = pathjoin(src_dirpath, filename)
dst_filename = pathjoin(dst_dirpath, filename)
copyfile(src_filename, dst_filename, overwrite=overwrite)
if parallel:
process.prepare()
process.run()
return True
def _ilistdir_helper(self, path, entries, wildcard=None, full=False,
absolute=False, dirs_only=False, files_only=False):
"""A helper method called by ilistdir method that applies filtering.
Given the path to a directory and a list of the names of entries within
that directory, this method applies the semantics of the ilistdir()
keyword arguments. An appropriately modified and filtered list of
directory entries is returned.
"""
path = normpath(path)
if dirs_only and files_only:
raise ValueError("dirs_only and files_only cannot both be True")
if wildcard is not None:
if not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
def wildcard(fn):
return bool(wildcard_re.match(fn))
entries = (p for p in entries if wildcard(p.name))
if dirs_only:
entries = (
p for p in entries if self.isdir(p.name, _statobj=p.statinfo)
)
elif files_only:
entries = (
p for p in entries if self.isfile(p.name, _statobj=p.statinfo)
)
if full:
entries = (pathcombine(path, p.name) for p in entries)
elif absolute:
path = self._p(path)
entries = ((pathcombine(path, p.name)) for p in entries)
else:
entries = (p.name for p in entries)
return entries
def _get_module_info(self, fullname):
if not fullname.startswith('__moyapy__'):
raise ImportError(fullname)
path = self._get_path(fullname)
module_path, type = self._find_module_file(path)
if module_path is not None:
return module_path, type, False
module_path, type = self._find_module_file(pathcombine(path, '__init__'))
if module_path is not None:
return module_path, type, True
raise ImportError(fullname)
def _get_module_info(self, fullname):
if not fullname.startswith('__moyapy__'):
raise ImportError(fullname)
path = self._get_path(fullname)
module_path, type = self._find_module_file(path)
if module_path is not None:
return module_path, type, False
module_path, type = self._find_module_file(
pathcombine(path, '__init__'))
if module_path is not None:
return module_path, type, True
raise ImportError(fullname)
def walk(self, path="/", wildcard=None, dir_wildcard=None, search="breadth",
ignore_errors=False):
if dir_wildcard is not None:
for item in super(WrapFS, self).walk(path, wildcard, dir_wildcard, search,
ignore_errors):
yield item
else:
if wildcard is not None and not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn: bool(wildcard_re.match(fn))
for (dirpath, filepaths) in self.wrapped_fs.walk(path, search=search,
ignore_errors=ignore_errors):
filepaths = [basename(self._decode(pathcombine(dirpath, p)))
for p in filepaths]
if wildcard is not None:
filepaths = [p for p in filepaths if wildcard(p)]
yield (dirpath, filepaths)
def walk(self,
path="/",
wildcard=None,
dir_wildcard=None,
search="breadth",
ignore_errors=False):
if dir_wildcard is not None:
for item in super(WrapFS, self).walk(path, wildcard, dir_wildcard,
search, ignore_errors):
yield item
else:
if wildcard is not None and not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn: bool(wildcard_re.match(fn))
for (dirpath, filepaths) in self.wrapped_fs.walk(
path, search=search, ignore_errors=ignore_errors):
filepaths = [
basename(self._decode(pathcombine(dirpath, p)))
for p in filepaths
]
if wildcard is not None:
filepaths = [p for p in filepaths if wildcard(p)]
yield (dirpath, filepaths)
