def getpathurl(self, path, allow_none=False):
path = normpath(path)
credentials = "%s:%s" % (self.user, self.passwd)
if credentials == ":":
url = "ftp://%s%s" % (self.host.rstrip("/"), abspath(path))
else:
url = "ftp://%s@%s%s" % (credentials, self.host.rstrip("/"), abspath(path))
return url
def getpathurl(self, path, allow_none=False):
path = normpath(path)
credentials = '%s:%s' % (self.user, self.passwd)
if credentials == ':':
url = 'ftp://%s%s' % (self.host.rstrip('/'), abspath(path))
else:
url = 'ftp://%s@%s%s' % (credentials, self.host.rstrip('/'), abspath(path))
return url
def getpathurl(self, path, allow_none=False):
path = normpath(path)
credentials = '%s:%s' % (self.user, self.passwd)
if credentials == ':':
url = 'ftp://%s%s' % (self.host.rstrip('/'), abspath(path))
else:
url = 'ftp://%s@%s%s' % (credentials, self.host.rstrip('/'), abspath(path))
return url
def copydir(self, src, dst, *args, **kwargs):
"""
@param src: Path to the folder to be copied
@param dst: Path to the folder in which to copy the folder
@return: Path to the copied folder
"""
src = abspath(normpath(src))
dst = abspath(normpath(dst))
return self.client.file_copy(src, dst)
def movedir(self, src, dst, *args, **kwargs):
"""
@param src: Path to the folder to be moved
@param dst: Path to the folder in which the folder will be moved
@param chunk_size: if using chunk upload
@return: Path to the moved folder
"""
src = abspath(normpath(src))
dst = abspath(normpath(dst))
return self.client.file_move(src, dst)
def rename(self, src, dst, *args, **kwargs):
"""
@param src: Path to the file to be renamed
@param dst: Full path with the new name
@raise UnsupportedError: If trying to remove the root directory
@return: Path to the renamed file
"""
src = abspath(normpath(src))
dst = abspath(normpath(dst))
return self.client.file_move(src, dst)
def copydir(self, src, dst, *args, **kwargs):
"""Copy a directory to another location.
:param src: Path to the folder to be copied
:param dst: Path to the folder in which to copy the folder
:return: Path to the copied folder
"""
src = abspath(normpath(src))
dst = abspath(normpath(dst))
return self.client.file_copy(src, dst)
def copydir(self, src, dst, *args, **kwargs):
"""Copy a directory to another location.
:param src: Path to the folder to be copied
:param dst: Path to the folder in which to copy the folder
:return: Path to the copied folder
"""
src = abspath(normpath(src))
dst = abspath(normpath(dst))
return self.client.file_copy(src, dst)
def rename(self, src, dst, *args, **kwargs):
"""Rename a file of a given path.
:param src: Path to the file to be renamed
:param dst: Full path with the new name
:raise UnsupportedError: If trying to remove the root directory
:return: Path to the renamed file
"""
src = abspath(normpath(src))
dst = abspath(normpath(dst))
return self.client.file_move(src, dst)
def movedir(self, src, dst, *args, **kwargs):
"""Move a directory to another location.
:param src: Path to the folder to be moved
:param dst: Path to the folder in which the folder will be moved
:param chunk_size: if using chunk upload
:return: Path to the moved folder
"""
src = abspath(normpath(src))
dst = abspath(normpath(dst))
return self.client.file_move(src, dst)
def movedir(self, src, dst, *args, **kwargs):
"""Move a directory to another location.
:param src: Path to the folder to be moved
:param dst: Path to the folder in which the folder will be moved
:param chunk_size: if using chunk upload
:return: Path to the moved folder
"""
src = abspath(normpath(src))
dst = abspath(normpath(dst))
return self.client.file_move(src, dst)
def check_template(self, template_path):
"""Check if a template exists"""
template_path = abspath(normpath(template_path))
if template_path in self.templates:
return
if not self.template_fs.exists(template_path):
raise MissingTemplateError(template_path)
def listdir(self,
path="/",
wildcard=None,
full=False,
absolute=False,
dirs_only=False,
files_only=False):
"""List the the files and directories under a given path.
The directory contents are returned as a list of unicode paths
:param path: path to the folder to list
:type path: string
:param wildcard: Only returns paths that match this wildcard
:type wildcard: string containing a wildcard, or a callable
that accepts a path and returns a boolean
:param full: returns full paths (relative to the root)
:type full: bool
:param absolute: returns absolute paths
(paths beginning with /)
:type absolute: bool
:param dirs_only: if True, only return directories
:type dirs_only: bool
:param files_only: if True, only return files
:type files_only: bool
:return: a list of unicode paths
"""
path = abspath(normpath(path))
children = self.client.children(path)
return self._listdir_helper(path, children, wildcard, full, absolute,
dirs_only, files_only)
def open(self, path, mode="rb", **kwargs):
"""Open the named file in the given mode.
This method downloads the file contents into a local temporary
file so that it can be worked on efficiently. Any changes
made to the file are only sent back to cloud storage when
the file is flushed or closed.
:param path: Path to the file to be opened
:param mode: In which mode to open the file
:raise ResourceNotFoundError: If given path doesn't exist and
'w' is not in mode
:return: RemoteFileBuffer object
"""
path = abspath(normpath(path))
spooled_file = SpooledTemporaryFile(mode=mode, bufsize=MAX_BUFFER)
if "w" in mode:
# Truncate the file if requested
self.client.put_file(path, "", True)
else:
# Try to write to the spooled file, if path doesn't exist create it
# if 'w' is in mode
try:
spooled_file.write(self.client.get_file(path).read())
spooled_file.seek(0, 0)
except:
if "w" not in mode:
raise ResourceNotFoundError(path)
else:
self.createfile(path, True)
# This will take care of closing the socket when it's done.
return RemoteFileBuffer(self, path, mode, spooled_file)
def desc(self, path):
"""
@return: The title for the given path.
"""
path = abspath(normpath(path))
info = self.getinfo(path)
return info["title"]
def inner_2(fs, *args, **kwargs):
new_args = []
for ndx, arg in enumerate(args):
if ndx < num_paths:
arg = abspath(normpath(arg))
new_args.append(arg)
return func(fs, *new_args, **kwargs)
def ilistdirinfo(self, path="/", wildcard=None, full=False, absolute=False,
dirs_only=False, files_only=False):
self._log(DEBUG, "Listing directory (listdirinfo) %s" % path)
if dirs_only and files_only:
raise ValueError("dirs_only and files_only can not both be True")
for item in self.tahoeutil.list(self.dircap, path):
if dirs_only and item['type'] == 'filenode':
continue
elif files_only and item['type'] == 'dirnode':
continue
if wildcard is not None:
if isinstance(wildcard,basestring):
if not fnmatch.fnmatch(item['name'], wildcard):
continue
else:
if not wildcard(item['name']):
continue
if full:
item_path = relpath(pathjoin(path, item['name']))
elif absolute:
item_path = abspath(pathjoin(path, item['name']))
else:
item_path = item['name']
yield (item_path, item)
def removedir(self, path, recursive=False, force=False):
path = abspath(normpath(path))
if not self.exists(path):
raise ResourceNotFoundError(path)
if self.isfile(path):
raise ResourceInvalidError(path)
if not force:
for _checkpath in self.listdir(path):
raise DirectoryNotEmptyError(path)
try:
if force:
for rpath in self.listdir(path, full=True):
try:
if self.isfile(rpath):
self.remove(rpath)
elif self.isdir(rpath):
self.removedir(rpath, force=force)
except FSError:
pass
self.clear_dircache(dirname(path))
self.ftp.rmd(_encode(path))
except error_reply:
pass
if recursive:
try:
self.removedir(dirname(path), recursive=True)
except DirectoryNotEmptyError:
pass
self.clear_dircache(dirname(path), path)
def walkfiles(self, path="/", wildcard=None, dir_wildcard=None,
search="breadth", ignore_errors=False):
if dir_wildcard is not None:
# If there is a dir_wildcard, fall back to the default impl
# that uses listdir(). Otherwise we run the risk of enumerating
# lots of directories that will just be thrown away.
for item in super(HideFS, self).walkfiles(path, wildcard,
dir_wildcard, search,
ignore_errors):
yield item
# Otherwise, the wrapped FS may provide a more efficient impl
# which we can use directly.
else:
if wildcard is not None and not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn: bool(wildcard_re.match(fn))
walk = self.wrapped_fs.walkfiles(self._encode(path),
search=search,
ignore_errors=ignore_errors)
for filepath in walk:
filepath = abspath(self._decode(filepath))
if wildcard is not None:
if not wildcard(basename(filepath)):
continue
if self.is_hidden(filepath):
continue
yield filepath
def _readdir(self, path):
path = abspath(normpath(path))
if self.dircache.count:
cached_dirlist = self.dircache.get(path)
if cached_dirlist is not None:
return cached_dirlist
dirlist = {}
parser = FTPListDataParser()
def on_line(line):
if not isinstance(line, unicode):
line = line.decode('utf-8')
info = parser.parse_line(line)
if info:
info = info.__dict__
if info['name'] not in ('.', '..'):
dirlist[info['name']] = info
try:
self.ftp.dir(_encode(path), on_line)
except error_reply:
pass
self.dircache[path] = dirlist
return dirlist
def listdirinfo(self, path="/", wildcard=None, full=False, absolute=False,
dirs_only=False, files_only=False):
self.wrapped_fs._log(DEBUG, "Listing directory (listdirinfo) %s" % path)
_fixpath = self.wrapped_fs._fixpath
_path = _fixpath(path)
if dirs_only and files_only:
raise errors.ValueError("dirs_only and files_only can not both be True")
result = []
for item in self.wrapped_fs.tahoeutil.list(self.dircap, _path):
if dirs_only and item['type'] == 'filenode':
continue
elif files_only and item['type'] == 'dirnode':
continue
if wildcard is not None and \
not fnmatch.fnmatch(item['name'], wildcard):
continue
if full:
item_path = relpath(pathjoin(_path, item['name']))
elif absolute:
item_path = abspath(pathjoin(_path, item['name']))
else:
item_path = item['name']
cache_name = self.wrapped_fs._fixpath(u"%s/%s" % \
(path, item['name']))
self._cache_set(cache_name, 'getinfo', (), {}, (True, item))
result.append((item_path, item))
return result
def removedir(self, path, recursive=False, force=False):
path = abspath(normpath(path))
if not self.exists(path):
raise ResourceNotFoundError(path)
if self.isfile(path):
raise ResourceInvalidError(path)
if not force:
for _checkpath in self.listdir(path):
raise DirectoryNotEmptyError(path)
try:
if force:
for rpath in self.listdir(path, full=True):
try:
if self.isfile(rpath):
self.remove(rpath)
elif self.isdir(rpath):
self.removedir(rpath, force=force)
except FSError:
pass
self.clear_dircache(dirname(path))
self.ftp.rmd(_encode(path))
except error_reply:
pass
if recursive:
try:
self.removedir(dirname(path), recursive=True)
except DirectoryNotEmptyError:
pass
self.clear_dircache(dirname(path), path)
def _readdir(self, path):
path = abspath(normpath(path))
if self.dircache.count:
cached_dirlist = self.dircache.get(path)
if cached_dirlist is not None:
return cached_dirlist
dirlist = {}
parser = FTPListDataParser()
def on_line(line):
if not isinstance(line, unicode):
line = line.decode('utf-8')
info = parser.parse_line(line)
if info:
info = info.__dict__
if info['name'] not in ('.', '..'):
dirlist[info['name']] = info
try:
self.ftp.dir(_encode(path), on_line)
except error_reply:
pass
self.dircache[path] = dirlist
return dirlist
def _check_path(self, path):
path = normpath(path)
base, fname = pathsplit(abspath(path))
dirlist = self._readdir(base)
if fname and fname not in dirlist:
raise ResourceNotFoundError(path)
return dirlist, fname
def iter_dirs(handle):
if hasattr(handle, 'seek') and handle.seekable():
handle.seek(0)
with py7zr.SevenZipFile(handle) as z:
for entry in z.files:
if entry.is_directory:
yield abspath(entry.filename)
def listdir(self, path="/", wildcard=None, full=False, absolute=False,
dirs_only=False, files_only=False):
"""List the the files and directories under a given path.
The directory contents are returned as a list of unicode paths
:param path: path to the folder to list
:type path: string
:param wildcard: Only returns paths that match this wildcard
:type wildcard: string containing a wildcard, or a callable
that accepts a path and returns a boolean
:param full: returns full paths (relative to the root)
:type full: bool
:param absolute: returns absolute paths
(paths beginning with /)
:type absolute: bool
:param dirs_only: if True, only return directories
:type dirs_only: bool
:param files_only: if True, only return files
:type files_only: bool
:return: a list of unicode paths
"""
path = abspath(normpath(path))
children = self.client.children(path)
return self._listdir_helper(path, children, wildcard, full, absolute,
dirs_only, files_only)
def inner_2(fs, *args, **kwargs):
new_args = []
for ndx, arg in enumerate(args):
if ndx < num_paths:
arg = abspath(normpath(arg))
new_args.append(arg)
return func(fs, *new_args, **kwargs)
def open(self, path, mode="rb", **kwargs):
"""Open the named file in the given mode.
This method downloads the file contents into a local temporary
file so that it can be worked on efficiently. Any changes
made to the file are only sent back to cloud storage when
the file is flushed or closed.
:param path: Path to the file to be opened
:param mode: In which mode to open the file
:raise ResourceNotFoundError: If given path doesn't exist and
'w' is not in mode
:return: RemoteFileBuffer object
"""
path = abspath(normpath(path))
spooled_file = SpooledTemporaryFile(mode=mode, bufsize=MAX_BUFFER)
if "w" in mode:
# Truncate the file if requested
self.client.put_file(path, "", True)
else:
# Try to write to the spooled file, if path doesn't exist create it
# if 'w' is in mode
try:
spooled_file.write(self.client.get_file(path).read())
spooled_file.seek(0, 0)
except:
if "w" not in mode:
raise ResourceNotFoundError(path)
else:
self.createfile(path, True)
# This will take care of closing the socket when it's done.
return RemoteFileBuffer(self, path, mode, spooled_file)
def walkfiles(self, path="/", wildcard=None, dir_wildcard=None,
search="breadth", ignore_errors=False):
if dir_wildcard is not None:
# If there is a dir_wildcard, fall back to the default impl
# that uses listdir(). Otherwise we run the risk of enumerating
# lots of directories that will just be thrown away.
for item in super(HideFS, self).walkfiles(path, wildcard,
dir_wildcard, search,
ignore_errors):
yield item
# Otherwise, the wrapped FS may provide a more efficient impl
# which we can use directly.
else:
if wildcard is not None and not callable(wildcard):
wildcard_re = re.compile(fnmatch.translate(wildcard))
wildcard = lambda fn: bool(wildcard_re.match(fn))
walk = self.wrapped_fs.walkfiles(self._encode(path),
search=search,
ignore_errors=ignore_errors)
for filepath in walk:
filepath = abspath(self._decode(filepath))
if wildcard is not None:
if not wildcard(basename(filepath)):
continue
if self.is_hidden(filepath):
continue
yield filepath
def iter_files(handle):
if hasattr(handle, 'seek') and handle.seekable():
handle.seek(0)
with zipfile.ZipFile(handle) as z:
for name in filter(None, z.namelist()):
if not name.endswith('/'):
yield abspath(name)
def check_template(self, template_path):
"""Check if a template exists"""
template_path = abspath(normpath(template_path))
if template_path in self.templates:
return
if not self.template_fs.exists(template_path):
raise MissingTemplateError(template_path)
def _check_path(self, path):
path = normpath(path)
base, fname = pathsplit(abspath(path))
dirlist = self._readdir(base)
if fname and fname not in dirlist:
raise ResourceNotFoundError(path)
return dirlist, fname
def component_files(fs: FS, dirpath: str) -> Iterator[str]:
"""
Returns the relative filepaths for files that are contained within
the directory for a PDS4 component. Files in PDS4 subcomponents
will have "$" in their paths (in the subcomponent directory name).
"""
return (abspath(filepath) for filepath in SubFS(fs, dirpath).walk.files()
if "$" not in filepath)
def desc(self, path):
"""Get the title of a given path.
:return: The title for the given path.
"""
path = abspath(normpath(path))
info = self.getinfo(path)
return info["title"]
def resolve_template_path(self, path, app_name, base_path="/"):
"""Get a template path in the appropriate directory for an app"""
if path.startswith('/'):
return path
if isinstance(app_name, Application):
app = app_name
else:
app = self.find_app(text_type(app_name))
template_path = abspath(pathjoin(base_path, app.templates_directory, path))
return template_path
def _IdFromPath(path, pathIdMap):
# if path[0] == "/":
# path2 = path[1:]
# result = pathIdMap.get(path, pathIdMap.get(path2, None))
result = pathIdMap.get(abspath(path), None)
if result is None:
info(f"Looking up {path}")
from pprint import pformat
info(f"Not found in {pformat(list(pathIdMap.keys()))}")
return result
def getpathurl(self, path):
"""
@param path: path to the file for which to return the url path
@param allow_none: if true, this method can return None if
there is no URL form of the given path
@type allow_none: bool
@return: url that corresponds to the given path, if one exists
"""
path = abspath(normpath(path))
return self.client.media(path)
def getinfo(self, path):
""" Returned information is metadata from cloud service +
a few more fields with standard names for some parts
of the metadata.
@param path: path to the file/folder for which to return
informations
@return: dictionary with informations about the specific file
"""
path = abspath(normpath(path))
metadata = self.client.metadata(path)
return self._metadata_to_info(metadata, localtime=self.localtime)
def iter_dirs(handle):
if hasattr(handle, 'seek') and handle.seekable():
handle.seek(0)
kwargs = {'fileobj': handle, 'mode': 'r'}
else:
kwargs = {'name': handle, 'mode': 'r'}
with tarfile.TarFile(**kwargs) as t:
for member in t.getmembers():
if member.isdir():
yield abspath(member.name)
def iter_dirs(handle):
zipname = lambda n: abspath(n).rstrip('/')
seen = set()
root_filter = '/'.__contains__
if hasattr(handle, 'seek') and handle.seekable():
handle.seek(0)
with zipfile.ZipFile(handle) as z:
for name in z.namelist():
# directory defined in the zipfile
if name.endswith('/'):
seen.add(name)
yield zipname(name)
# implicit directory
else:
for path in filterfalse(root_filter, recursepath(name)):
if path != abspath(name) and not path in seen:
seen.add(path)
yield zipname(path)
def getpathurl(self, path):
"""Get the url of a given path.
:param path: path to the file for which to return the url path
:param allow_none: if true, this method can return None if
there is no URL form of the given path
:type allow_none: bool
:return: url that corresponds to the given path, if one exists
"""
path = abspath(normpath(path))
return self.client.media(path)
def remove(self, path):
"""
@param path: path to the file to be deleted
@return: None if removal was successful
"""
path = abspath(normpath(path))
if self.is_root(path = path):
raise UnsupportedError("Can't remove the root directory")
if self.isdir(path = path):
raise ResourceInvalidError("Specified path is a directory. " +
"Please use removedir.")
self.client.file_delete(path)
def get_template(self, template_path, parse=True):
template_path = abspath(normpath(template_path))
template = self.templates.get(template_path, None)
if template is not None:
return template
if self.cache.enabled:
try:
info = self.template_fs.getinfokeys(template_path,
'modified_time',
'st_mtime')
except ResourceNotFoundError:
raise MissingTemplateError(template_path)
modified_time = info.get('st_mtime', None) or datetime_to_epoch(
info.get('modified_time', 0))
cache_name = "%s$%s@%s" % (TEMPLATE_VERSION, template_path,
modified_time)
cached_template = self.cache.get(cache_name, None)
if cached_template is not None:
template = Template.load(cached_template)
self.templates[template_path] = template
return template
try:
source = self.template_fs.getcontents(template_path,
'rt',
encoding='utf-8')
except ResourceNotFoundError:
raise MissingTemplateError(template_path)
except Exception as e:
raise BadTemplateError(template_path)
try:
display_path = self.template_fs.getsyspath(template_path)
except NoSysPathError:
display_path = template_path
if self.archive is None:
lib = None
else:
lib = self.archive.get_template_lib(template_path)
template = Template(source,
display_path,
raw_path=template_path,
lib=lib)
template.parse(self)
if self.cache.enabled:
self.cache.set(cache_name, template.dump(self))
self.templates[template_path] = template
return template
def getinfo(self, path):
"""Get info from cloud service.
Returned information is metadata from cloud service +
a few more fields with standard names for some parts
of the metadata.
:param path: path to the file/folder for which to return
informations
:return: dictionary with informations about the specific file
"""
path = abspath(normpath(path))
metadata = self.client.metadata(path)
return self._metadata_to_info(metadata, localtime=self.localtime)
def remove(self, path):
"""Remove a file of a given path.
:param path: path to the file to be deleted
:return: None if removal was successful
"""
path = abspath(normpath(path))
if self.is_root(path=path):
raise UnsupportedError("Can't remove the root directory")
if self.isdir(path=path):
raise ResourceInvalidError(
"Specified path is a directory. Please use removedir.")
self.client.file_delete(path)
def removedir(self, path, *args, **kwargs):
"""Remove a directory of a given path.
:param path: path to the file to be deleted
:return: None if removal was successful
"""
path = abspath(normpath(path))
if self.is_root(path=path):
raise UnsupportedError("Can't remove the root directory")
if self.isfile(path=path):
raise ResourceInvalidError(
"Specified path is a directory. Please use removedir.")
self.client.file_delete(path)
def get_template(self, template_path, parse=True):
template_path = abspath(normpath(template_path))
template = self.templates.get(template_path, None)
if template is not None:
return template
if self.cache.enabled:
try:
info = self.template_fs.getdetails(template_path)
except ResourceNotFound:
raise MissingTemplateError(template_path)
cache_name = "%s$%s@%s" % (TEMPLATE_VERSION, template_path,
info.modified)
cached_template = self.cache.get(cache_name, None)
if cached_template is not None:
template = Template.load(cached_template)
self.templates[template_path] = template
return template
try:
source = self.template_fs.gettext(template_path)
except ResourceNotFound:
raise MissingTemplateError(template_path)
except Exception as error:
raise BadTemplateError(
template_path,
diagnosis="failed to read text ({})".format(text_type(error)),
)
try:
display_path = self.template_fs.getsyspath(template_path)
except NoSysPath:
display_path = template_path
if self.archive is None:
lib = None
else:
lib = self.archive.get_template_lib(template_path)
template = Template(source,
display_path,
raw_path=template_path,
lib=lib)
template.parse(self)
if self.cache.enabled:
self.cache.set(cache_name, template.dump(self))
self.templates[template_path] = template
return template
def __init__(self, archive, fs, path, library=None):
self.built = False
try:
self.cache = archive.get_cache('parser')
except Exception as e:
self.cache = self._default_cache
self.archive = archive
self.fs = fs
self.path = abspath(path)
self.library = library
syspath = self.fs.getsyspath(path, allow_none=True)
if syspath is None:
self.location = self.fs.desc(path)
else:
self.location = syspath
self._xml = None
def __init__(self, archive, fs, path, library=None):
self.built = False
try:
self.cache = archive.get_cache('parser')
except Exception as e:
self.cache = self._default_cache
self.archive = archive
self.fs = fs
self.path = abspath(path)
self.library = library
syspath = self.fs.getsyspath(path, allow_none=True)
if syspath is None:
self.location = self.fs.desc(path)
else:
self.location = syspath
self._xml = None
def makedir(self, path, recursive=False, allow_recreate=False):
"""Create a directory of a given path.
:param path: path to the folder to be created.
If only the new folder is specified
it will be created in the root directory
:param recursive: allows recursive creation of directories
:param allow_recreate: dropbox currently doesn't support
allow_recreate, so if a folder exists it will
:return: Id of the created directory
"""
if not self._checkRecursive(recursive, path):
raise UnsupportedError("Recursively create specified folder.")
path = abspath(normpath(path))
return self.client.file_create_folder(path)
def makedir(self, path, recursive=False, allow_recreate=False):
"""Create a directory of a given path.
:param path: path to the folder to be created.
If only the new folder is specified
it will be created in the root directory
:param recursive: allows recursive creation of directories
:param allow_recreate: dropbox currently doesn't support
allow_recreate, so if a folder exists it will
:return: Id of the created directory
"""
if not self._checkRecursive(recursive, path):
raise UnsupportedError("Recursively create specified folder.")
path = abspath(normpath(path))
return self.client.file_create_folder(path)
def iterkeys(self, root="/", m=None):
"""Iterate over all keys beginning with the given root path."""
if m is None:
m = self._map
for name in iteratepath(root):
try:
m = m[name]
except KeyError:
return
for (nm, subm) in m.iteritems():
if not nm:
yield abspath(root)
else:
k = combine(root, nm)
for subk in self.iterkeys(k, subm):
yield subk
def iteritems(self, root="/", m=None):
"""Iterate over all (key,value) pairs beginning with the given root."""
root = normpath(root)
if m is None:
m = self._map
for name in iteratepath(root):
try:
m = m[name]
except KeyError:
return
for (nm, subm) in m.iteritems():
if not nm:
yield (abspath(normpath(root)), subm)
else:
k = combine(root, nm)
for (subk, subv) in self.iteritems(k, subm):
yield (subk, subv)
def walkdirs(self, path="/", wildcard=None, search="breadth",
ignore_errors=False):
if wildcard is not None:
# If there is a wildcard, fall back to the default impl
# that uses listdir(). Otherwise we run the risk of enumerating
# lots of directories that will just be thrown away.
for item in super(HideFS, self).walkdirs(path, wildcard, search,
ignore_errors):
yield item
# Otherwise, the wrapped FS may provide a more efficient impl
# which we can use directly.
else:
walk = self.wrapped_fs.walkdirs(self._encode(path), search=search,
ignore_errors=ignore_errors)
for dirpath in walk:
if self.is_hidden(dirpath):
continue
yield abspath(self._decode(dirpath))
def setcontents(self, path, data, *args, **kwargs):
"""Sets new content to remote file
Method works only with existing files and sets
new content to them.
@param path: Path the file in which to write the new content
@param contents: File contents as a string, or any object with
read and seek methods
@param kwargs: additional parameters like:
encoding: the type of encoding to use if data is text
errors: encoding errors
@param chunk_size: Number of bytes to read in a chunk,
if the implementation has to resort to a read copy loop
@return: Path of the updated file
"""
path = abspath(normpath(path))
self.client.put_file(path, data, overwrite=True)
return path
def clear_dircache(self, *paths):
"""
Clear cached directory information.
:param path: Path of directory to clear cache for, or all directories if
None (the default)
"""
if not paths:
self.dircache.clear()
else:
dircache = self.dircache
paths = [normpath(abspath(path)) for path in paths]
for cached_path in dircache.keys():
for path in paths:
if isbase(cached_path, path):
dircache.pop(cached_path, None)
break
def get_template(self, template_path):
template_path = abspath(normpath(template_path))
template = self.templates.get(template_path, None)
if template is not None:
return template
if self.cache.enabled:
try:
info = self.template_fs.getinfokeys(template_path, 'modified_time', 'st_mtime')
except ResourceNotFoundError:
raise MissingTemplateError(template_path)
modified_time = info.get('st_mtime', None) or datetime_to_epoch(info.get('modified_time', 0))
cache_name = "%s$%s@%s" % (TEMPLATE_VERSION, template_path, modified_time)
cached_template = self.cache.get(cache_name, None)
if cached_template is not None:
template = Template.load(cached_template)
self.templates[template_path] = template
return template
try:
source = self.template_fs.getcontents(template_path, 'rt', encoding='utf-8')
except ResourceNotFoundError:
raise MissingTemplateError(template_path)
except:
raise BadTemplateError(template_path)
try:
display_path = self.template_fs.getsyspath(template_path)
except NoSysPathError:
display_path = template_path
if self.archive is None:
lib = None
else:
lib = self.archive.get_template_lib(template_path)
#lib = self.archive.get_lib(lib) if lib else None
template = Template(source, display_path, raw_path=template_path, lib=lib)
template.parse(self)
if self.cache.enabled:
self.cache.set(cache_name, template.dump(self))
self.templates[template_path] = template
return template
def removedir(self, path, recursive=False, force=False):
if self.isfile(path):
raise ResourceInvalidError(path)
if not self.isdir(path):
if not self.isdir(dirname(path)):
raise ParentDirectoryMissingError(path)
raise ResourceNotFoundError(path)
if abspath(path) == "/":
raise RemoveRootError(path)
if force:
for child_path in self.listdir(path, full=True, dirs_only=True):
self.removedir(child_path, force=force)
for child_path in self.listdir(path, full=True, files_only=True):
self.remove(child_path)
elif len(self.listdir(path)) > 0:
raise DirectoryNotEmptyError(path)
self.client.auth.service.files() \
.delete(fileId=self._ids[path]) \
.execute()
self._ids.pop(path)
if recursive and len(self.listdir(dirname(path))) == 0:
self.removedir(dirname(path), recursive=recursive)
