def makedir(self, path, permissions=None, recreate=False):
path = self.fix_path(path)
if self.exists(path) and not recreate:
raise errors.DirectoryExists(path)
if path == "/":
return SubFS(self, path)
if self.exists(path):
meta = self.getinfo(path)
if meta.is_dir:
if recreate == False:
raise errors.DirectoryExists(path)
else:
return SubFS(self, path)
if meta.is_file:
raise errors.DirectoryExpected(path)
ppath = self.get_parent(path)
if not self.exists(ppath):
raise errors.ResourceNotFound(ppath)
try:
folderMetadata = self.dropbox.files_create_folder_v2(path)
except ApiError as e:
raise errors.DirectoryExpected(path=path)
return SubFS(self, path)
def _reset_path(self, path, confirm=False):
if not confirm:
print(
"Are you sure you want to reset path '%s' - located at '%s' on Cloud Firestore?"
% (path, self._prep_path(path))
)
return False
with self._lock:
try:
_res = self._getresource(path)
except:
_res = fire_fs._getresource(self._prep_path(path))
if not _res or not _res.isdir():
raise errors.DirectoryExpected(path)
if len(_res.listdir()) < 1:
return self.opendir(path)
if not self._is_cached:
self._stop_cache(False)
_res.delete(recursive=True)
self._stop_cache(True)
else:
_res.delete(recursive=True)
_res = fire_fs.mkdir(self._prep_path(path))
return self.opendir(path)
def openbin(self, path, mode="r", buffering=-1, **options):
path = self.fix_path(path)
_mode = Mode(mode)
mode = _mode
_mode.validate_bin()
_path = self.validatepath(path)
log.debug("openbin: %s, %s", path, mode)
with self._lock:
try:
info = self.getinfo(_path)
log.debug("Info: %s", info)
except errors.ResourceNotFound:
if not _mode.create:
raise errors.ResourceNotFound(path)
# Check the parent is an existing directory
if not self.getinfo(self.get_parent(_path)).is_dir:
raise errors.DirectoryExpected(path)
else:
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
return DropboxFile(self.dropbox, path, mode)
def listdir(self, path: Text) -> List[Text]:
""" Get a list of resources in a directory. """
npath = self.normalize_path(path)
if not self.exists(npath):
raise errors.ResourceNotFound(path)
if not self.isdir(npath):
raise errors.DirectoryExpected(path)
qpath = npath + "/%"
if npath == "/":
qpath = "/%"
cursor = self.connection.cursor()
cursor.execute(
"SELECT name FROM sqlar WHERE name LIKE ?",
(qpath,)
)
rows = list(cursor.fetchall())
cursor.close()
children = []
for row in rows:
if row['name'] == npath or "/" in row['name'][len(npath):].strip("/"):
continue
children.append(basename(row['name']))
return children
def listdir(self, path):
# type: (Text) -> List[Text]
"""Get a list of the resource names in a directory.
This method will return a list of the resources in a directory.
A *resource* is a file, directory, or one of the other types
defined in `~fs.enums.ResourceType`.
Arguments:
path (str): A path to a directory on the filesystem
Returns:
list: list of names, relative to ``path``.
Raises:
fs.errors.DirectoryExpected: If ``path`` is not a directory.
fs.errors.ResourceNotFound: If ``path`` does not exist.
"""
with self._lock:
_res = self._getresource(path)
if not _res:
raise errors.ResourceNotFound(path)
if not _res.is_collection:
raise errors.DirectoryExpected(path)
return _res.get_member_names()
def listdir(self, path):
_path = self.validatepath(path)
_s3_key = self._path_to_dir_key(_path)
prefix_len = len(_s3_key)
paginator = self.client.get_paginator('list_objects')
with s3errors(path):
_paginate = paginator.paginate(
Bucket=self._bucket_name,
Prefix=_s3_key,
Delimiter=self.delimiter
)
_directory = []
for result in _paginate:
common_prefixes = result.get('CommonPrefixes', ())
for prefix in common_prefixes:
_prefix = prefix.get('Prefix')
_name = _prefix[prefix_len:]
if _name:
_directory.append(_name.rstrip(self.delimiter))
for obj in result.get('Contents', ()):
name = obj["Key"][prefix_len:]
if name:
_directory.append(name)
if not _directory:
if not self.getinfo(_path).is_dir:
raise errors.DirectoryExpected(path)
return _directory
def openbin(self, path, mode="r", buffering=-1, **options):
path = self.fix_path(path)
_mode = Mode(mode)
mode = _mode
_mode.validate_bin()
_path = self.validatepath(path)
log.debug("openbin: %s, %s", path, mode)
with self._lock:
try:
info = self.getinfo(_path)
except errors.ResourceNotFound:
if not _mode.create:
raise
# Target doesn't exist and we're in create mode. Ensure the
# parent is an existing directory before we try to create a file
# in it.
parent_path = self.get_parent(_path)
# Can't use self.isdir() because it doesn't crash if the
# directory doesn't exist, and we don't want to stat a file twice
# if we can avoid it.
info = self.getinfo(parent_path)
if not info.is_dir:
raise errors.DirectoryExpected(parent_path)
return DropboxFile(self.dropbox, path, mode)
# Target exists.
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
return DropboxFile(self.dropbox, path, mode)
def __init__(self, root_path=None, use_cache=True):
# self._meta = {}
super(FirestoreFS, self).__init__()
if root_path is None:
root_path = "/_firestore_fs_"
_root_path = self.validatepath(root_path)
if not _root_path.startswith("/"):
_root_path = "/" + _root_path
self._is_cached = True
if not use_cache:
self._stop_cache(True)
# Initialize Firestore filesystem if needed
fire_fs.initfs()
# Check if the requested root_path exists
_res = fire_fs._getresource(_root_path)
if _res:
if _res.isdir():
log.info("Root path exists %s" % _root_path)
else:
raise errors.DirectoryExpected(root_path)
else:
log.info("Creating root path %s" % _root_path)
_res = fire_fs.mkdir(_root_path)
log.info("Resource: %s" % _res)
self.root_path = _root_path
self.root_res = _res
def listdir(self, path):
info = self.getinfo(path)
if not info.is_dir:
raise errors.DirectoryExpected(path)
for i in info.raw['other']['files']:
yield i
return
def listdir(self, path):
_path = self.validatepath(path)
_type = self.gettype(_path)
if _type is not ResourceType.directory:
raise errors.DirectoryExpected(path)
result = self.pcloud.listfolder(path=_path)
return [item["name"] for item in result["metadata"]["contents"]]
def opendir(self, path: str, factory=None) -> SubFS[FS]:
# Implemented to support skipping the directory check if strict=False
_factory = factory or SubFS
if self.strict and not self.getbasic(path).is_dir:
raise errors.DirectoryExpected(path=path)
return _factory(self, path)
def removedir(self, path):
_path = self.validatepath(path)
if path in '/':
raise errors.RemoveRootError()
if not self.getinfo(path).is_dir:
raise errors.DirectoryExpected(path)
if not self.isempty(_path):
raise errors.DirectoryNotEmpty(path)
self.client.clean(_path.encode('utf-8'))
def scandir(
self,
path, # type: Text
namespaces=None, # type: Optional[Collection[Text]]
page=None, # type: Optional[Tuple[int, int]]
):
# type: (...) -> Iterator[Info]
"""Get an iterator of resource info.
Arguments:
path (str): A path to a directory on the filesystem.
namespaces (list, optional): A list of namespaces to include
in the resource information, e.g. ``['basic', 'access']``.
page (tuple, optional): May be a tuple of ``(<start>, <end>)``
indexes to return an iterator of a subset of the resource
info, or `None` to iterate over the entire directory.
Paging a directory scan may be necessary for very large
directories.
Returns:
~collections.abc.Iterator: an iterator of `Info` objects.
Raises:
fs.errors.DirectoryExpected: If ``path`` is not a directory.
fs.errors.ResourceNotFound: If ``path`` does not exist.
"""
namespaces = namespaces or ()
if path in ("/", "") or path is None:
return self._scandir_root(namespaces)
_res = self._getresource(path)
if not _res:
raise errors.ResourceNotFound(path)
if not _res.isdir():
raise errors.DirectoryExpected(path)
# info = (
# self.getinfo(join(_path, name), namespaces=namespaces)
# for name in self.listdir(path)
# )
# iter_info = iter(info)
# iter_info = self._scandir_from_resource(_res, namespaces)
if page is not None:
start, end = page
iter_info = self._scandir_from_resource(_res, namespaces,
end - start, start)
else:
limit = self._limit
offset = 0
iter_info = self._scandir_from_resource(_res, namespaces, limit,
offset)
# if page is not None:
# start, end = page
# iter_info = itertools.islice(iter_info, start, end)
return iter_info
def listdir(self, path): # noqa: D102
self.check()
_path = self.validatepath(path)
_type = self.gettype(_path)
if _type is not ResourceType.directory:
raise errors.DirectoryExpected(path)
with convert_sshfs_errors('listdir', path):
return self._sftp.listdir(_path)
def listdir(self, path):
_path = self.validatepath(path)
if not self.getinfo(_path).is_dir:
raise errors.DirectoryExpected(path)
dir_list = self.client.list(_path.encode('utf-8'))
if six.PY2:
dir_list = map(operator.methodcaller('decode', 'utf-8'), dir_list)
return list(map(operator.methodcaller('rstrip', '/'), dir_list))
def removedir(self, path):
self.check()
_path = self.validatepath(path)
if _path == '/':
raise errors.RemoveRootError()
info = self.getinfo(_path)
if not info.is_dir:
raise errors.DirectoryExpected(path)
if not self.isempty(path):
raise errors.DirectoryNotEmpty(path)
_key = self._path_to_dir_key(_path)
self.s3_delete(self._bucket_name, _key)
def removedir(self, path):
self.check()
_path = self.validatepath(path)
if _path == "/":
raise errors.RemoveRootError()
info = self.getinfo(_path)
if not info.is_dir:
raise errors.DirectoryExpected(path)
if not self.isempty(path):
raise errors.DirectoryNotEmpty(path)
_key = self._path_to_dir_key(_path)
self.client.delete_object(Bucket=self._bucket_name, Key=_key)
def removedir(self, path):
self.check()
_path = self.validatepath(path)
if _path == "/":
raise errors.RemoveRootError()
info = self.getinfo(_path)
if not info.is_dir:
raise errors.DirectoryExpected(path)
if not self.isempty(_path):
raise errors.DirectoryNotEmpty(path)
_key = self._path_to_dir_key(_path)
with dlkerrors(path):
self.dlk.rmdir(_key)
def removedir(self, path: Text) -> None:
""" Remove a directory. """
npath = self.normalize_path(path)
if npath == "/":
raise errors.RemoveRootError
if not self.exists(npath):
raise errors.ResourceNotFound(path)
if not self.isdir(npath):
raise errors.DirectoryExpected(path)
if self.listdir(npath):
raise errors.DirectoryNotEmpty(path)
cursor = self.connection.cursor()
cursor.execute("DELETE FROM sqlar WHERE name LIKE ?", (npath,))
cursor.close()
def scandir(self, path, namespaces=None, page=None):
_path = self.validatepath(path)
namespaces = namespaces or ()
_s3_key = self._path_to_dir_key(_path)
prefix_len = len(_s3_key)
info = self.getinfo(path)
if not info.is_dir:
raise errors.DirectoryExpected(path)
paginator = self.client.get_paginator('list_objects')
_paginate = paginator.paginate(
Bucket=self._bucket_name,
Prefix=_s3_key,
Delimiter=self.delimiter
)
def gen_info():
for result in _paginate:
common_prefixes = result.get('CommonPrefixes', ())
for prefix in common_prefixes:
_prefix = prefix.get('Prefix')
_name = _prefix[prefix_len:]
if _name:
info = {
"basic": {
"name": _name.rstrip(self.delimiter),
"is_dir": True
}
}
yield Info(info)
for _obj in result.get('Contents', ()):
name = _obj["Key"][prefix_len:]
if name:
with s3errors(path):
obj = self.s3.Object(
self._bucket_name, _obj["Key"]
)
info = self._info_from_object(obj, namespaces)
yield Info(info)
iter_info = iter(gen_info())
if page is not None:
start, end = page
iter_info = itertools.islice(iter_info, start, end)
for info in iter_info:
yield info
def removedir(self, path: str) -> None:
self.check()
_path = self.validatepath(path)
if _path == "/":
raise errors.RemoveRootError()
info = self.getinfo(_path)
if not info.is_dir:
raise errors.DirectoryExpected(path)
if not self.isempty(path):
raise errors.DirectoryNotEmpty(path)
_key = self._path_to_dir_key(_path)
try:
self.bucket.delete_blob(_key)
except google.cloud.exceptions.NotFound:
raise errors.ResourceNotFound(path)
def listdir(self, path):
_path = self.fix_path(path)
if _path == "/":
_path = ""
if not self.exists(_path):
raise errors.ResourceNotFound(path)
meta = self.getinfo(_path)
if meta.is_file:
raise errors.DirectoryExpected(path)
result = self.dropbox.files_list_folder(_path, include_media_info=True)
allEntries = result.entries
while result.has_more:
result = self.dropbox.files_list_folder_continue(result.cursor)
allEntries += result.entries
return [x.name for x in allEntries]
def removedir(self, path):
_path = self.fix_path(path)
if _path == "/":
raise errors.RemoveRootError()
try:
info = self.getinfo(path)
if not info.is_dir:
raise errors.DirectoryExpected(path=path)
if len(self.listdir(path)) > 0:
raise errors.DirectoryNotEmpty(path=path)
self.dropbox.files_delete_v2(_path)
except ApiError as e:
if isinstance(e.error._value, LookupError):
raise errors.ResourceNotFound(path=path)
raise errors.FileExpected(path=path, exc=e)
def listdir(self, path, limit=None, offset=0):
# type: (Text) -> List[Text]
"""Get a list of the resource names in a directory.
This method will return a list of the resources in a directory.
A *resource* is a file, directory, or one of the other types
defined in `~fs.enums.ResourceType`.
Arguments:
path (str): A path to a directory on the filesystem
Returns:
list: list of names, relative to ``path``.
Raises:
fs.errors.DirectoryExpected: If ``path`` is not a directory.
fs.errors.ResourceNotFound: If ``path`` does not exist.
"""
log.info(path)
if path in ("/", "") or path is None:
# return [kind for kind in self._kinds if kind not in self._metakinds]
return self._kinds
if path.startswith("/"):
path = path[1:]
parts = path.split("/")
kind = parts.pop(0)
if kind not in self._kinds:
raise errors.ResourceNotFound(path)
if len(parts) > 0:
# we should return this error here, but we could also make it easier to navigate...
raise errors.DirectoryExpected(path)
# id_or_name = "/".join(parts)
# ...
if limit is None:
limit = self._limit
# return [str(key.id_or_name) for key in db.list_entity_keys(kind, limit, offset)]
result = []
for key in db.list_entity_keys(kind, limit, offset):
name = self._key_to_path(key)
result.append(name)
return result
def openbin(self, path, mode='r', buffering=-1, **options):
_mode = Mode(mode)
_mode.validate_bin()
_path = self.validatepath(path)
log.debug("openbin: %s, %s", path, mode)
with self._lock:
try:
info = self.getinfo(_path)
log.debug("Info: %s", info)
except errors.ResourceNotFound:
if not _mode.create:
raise errors.ResourceNotFound(path)
# Check the parent is an existing directory
if self.gettype(dirname(_path)) is not ResourceType.directory:
raise errors.DirectoryExpected(dirname(path))
else:
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
return WebDAVFile(self, _path, _mode)
def removedir(self, path):
"""
Remove directory under `path`.
The directory must be empty.
:param str path: Path pointing to a directory.
"""
# type: (Text) -> None
path = ensureUnicode(path)
self.check()
_path = toAscii(self.validatepath(path))
if _path == "/":
raise errors.RemoveRootError()
info = self.getinfo(path)
if not info.is_dir:
raise errors.DirectoryExpected(path)
if not self.isempty(path):
raise errors.DirectoryNotEmpty(path)
self._odfs.unlink(_path)
def removedir(self, path):
# type: (Text) -> None
"""Remove a directory from the filesystem.
Arguments:
path (str): Path of the directory to remove.
Raises:
fs.errors.DirectoryNotEmpty: If the directory is not empty (
see `~fs.base.FS.removetree` for a way to remove the
directory contents.).
fs.errors.DirectoryExpected: If the path does not refer to
a directory.
fs.errors.ResourceNotFound: If no resource exists at the
given path.
fs.errors.RemoveRootError: If an attempt is made to remove
the root directory (i.e. ``'/'``)
"""
_path = self.validatepath(path)
if _path == "/" or _path == "" or _path is None:
raise errors.RemoveRootError()
if _path.endswith("/"):
_path = _path[:-1]
with self._lock:
_res = self._getresource(path)
if not _res:
raise errors.ResourceNotFound(path)
if not _res.is_collection:
raise errors.DirectoryExpected(path)
if len(_res.get_member_names()) > 0:
raise errors.DirectoryNotEmpty(path)
# _res.delete(recursive=False)
_res.delete()
def listdir(self, path: str) -> List[str]:
result = list(self._scandir(path))
if not result:
if not self.getinfo(path).is_dir:
raise errors.DirectoryExpected(path)
return result
def _scandir(
self,
path: str,
return_info: bool = False,
namespaces: List[str] = None
) -> Union[Iterator[str], Iterator[Info]]:
"""Returns all the resources in a directory
Args:
path: Path to the directory on the filesystem which shall be scanned
return_info: If `True` instances of the type fs.info.Info are being returned. If `False` only the names of the resources are being returned.
namespaces: A list of namespaces to include in the resource information. Only considered if `return_info=True`.
Returns:
Either an iterator of Info instances for each resource in the directory or an iterator of string names for each resource in the directory
"""
namespaces = namespaces or ()
_path = self.validatepath(path)
if namespaces and not return_info:
raise ValueError(
"The provided namespaces are only considered if return_info=True"
)
info = self.getinfo(_path)
if not info.is_dir:
raise errors.DirectoryExpected(_path)
dir_key = self._path_to_dir_key(_path)
if dir_key == "/":
# In case we want to list the root directory, no prefix is necessary
prefix = ""
else:
prefix = dir_key
prefix_len = len(prefix)
# Build set of root level directories
page_iterator = self.bucket.list_blobs(prefix=prefix,
delimiter=self.DELIMITER)
dir_prefixes = set()
for page in page_iterator.pages:
dir_prefixes.update(page.prefixes)
# Loop over all root level directories
for dir_prefix in dir_prefixes:
_name = dir_prefix[prefix_len:]
if return_info:
yield self._dir_info(_name)
else:
yield _name.rstrip(self.DELIMITER)
# Loop over all root level blobs
item_iterator = self.bucket.list_blobs(prefix=prefix,
delimiter=self.DELIMITER)
for blob in list(item_iterator):
if blob.name == dir_key: # Don't return root directory
continue
if return_info:
yield self._info_from_blob(blob, namespaces=namespaces)
else:
yield blob.name[prefix_len:]