def walk(self, src: str) -> List[Tuple[str, IOHandle]]:
"""Get list of all files at the given source path.
If the source path references a single file the returned list will
contain a single entry. If the source specifies a folder the result
contains a list of all files in that folder and the subfolders.
Parameters
----------
src: str
Source path specifying a file or folder.
Returns
-------
list of tuples (str, flowserv.volume.base.IOHandle)
"""
dirpath = util.filepath(key=src, sep=self.client.sep)
dirpath = self.client.sep.join([self.remotedir, dirpath
]) if dirpath else self.remotedir
files = self.client.walk(dirpath=dirpath)
if files is None:
# The source path references a single file.
filename = util.filepath(key=src, sep=self.client.sep)
filename = self.client.sep.join([self.remotedir, filename])
return [(src, SFTPFile(filename=filename, client=self.client))]
else:
# The source path references a directory.
result = list()
for key in files:
key = util.join(src, key) if src else key
filename = util.filepath(key=key, sep=self.client.sep)
filename = self.client.sep.join([self.remotedir, filename])
result.append(
(key, SFTPFile(filename=filename, client=self.client)))
return result
def delete(self, key: str):
"""Delete file or folder with the given key.
Parameters
----------
key: str
Path to a file object in the storage volume.
"""
sftp = self.client.sftp()
# Get recursive list of all files in the base folder and delete them.
dirpath = util.filepath(key=key, sep=self.client.sep)
dirpath = self.client.sep.join([self.remotedir, dirpath
]) if dirpath else self.remotedir
files = self.client.walk(dirpath=dirpath)
if files is None:
filename = util.filepath(key=key, sep=self.client.sep)
filename = self.client.sep.join([self.remotedir, filename])
sftp.remove(filename)
else:
# Collect sub-directories that need to be removed separately after
# the directories are empty.
directories = set()
for src in files:
filename = util.filepath(key=src, sep=self.client.sep)
filename = self.client.sep.join([self.remotedir, filename])
dirname = util.dirname(src)
if dirname:
directories.add(dirname)
sftp.remove(filename)
for dirpath in sorted(directories, reverse=True):
dirname = util.filepath(key=dirpath, sep=self.client.sep)
dirname = self.client.sep.join(
[self.remotedir, dirname]) if dirname else self.remotedir
sftp.rmdir(dirname)
def walk(self, src: str) -> List[Tuple[str, IOHandle]]:
"""Get list of all files at the given source path.
If the source path references a single file the returned list will
contain a single entry. If the source specifies a folder the result
contains a list of all files in that folder and the subfolders.
Parameters
----------
src: str
Source path specifying a file or folder.
Returns
-------
list of tuples (str, flowserv.volume.base.IOHandle)
"""
# The file key is a path expression that uses '/' as the path separator.
# If the local OS uses a different separator we need to replace it.
filename = os.path.join(self.basedir, util.filepath(
key=src)) if src else self.basedir
# The result is a list of file keys.
result = list()
if os.path.isfile(filename):
# If the source key references a file return a list with the key as
# the only element.
result.append((src, FSFile(filename=filename)))
elif os.path.isdir(filename):
# Recursively append all files in the referenced directory to the
# result list.
walkdir(filename, src, result)
return result
def mkdir(self, path: str):
"""Create the directory with the given (relative) path and all of its
parent directories.
Does not raise an error if the directory exists.
Parameters
----------
path: string
Relative path to a directory in the storage volume.
"""
dirname = os.path.join(self.basedir, util.filepath(key=path))
os.makedirs(dirname, exist_ok=True)
def store(self, file: IOHandle, dst: str):
"""Store a given file object at the destination path of this volume
store.
Parameters
----------
file: flowserv.volume.base.IOHandle
File-like object that is being stored.
dst: str
Destination path for the stored object.
"""
# The file key is a path expression that uses '/' as the path separator.
# If the local OS uses a different separator we need to replace it.
filename = os.path.join(self.basedir, util.filepath(key=dst))
os.makedirs(os.path.dirname(filename), exist_ok=True)
with open(filename, 'wb') as fout:
with file.open() as fin:
fout.write(fin.read())
def mkdir(self, path: str):
"""Create the directory with the given (relative) path and all of its
parent directories.
For bucket stores no directories need to be created prior to accessing
them.
Parameters
----------
path: string
Relative path to a directory in the storage volume.
"""
dirname = util.filepath(key=path, sep=self.client.sep)
dirname = self.client.sep.join([self.remotedir, dirname])
sftp = self.client.sftp()
try:
sftp_mkdir(client=sftp, dirpath=dirname)
finally:
sftp.close()
def load(self, key: str) -> IOHandle:
"""Load a file object at the source path of this volume store.
Returns a file handle that can be used to open and read the file.
Parameters
----------
key: str
Path to a file object in the storage volume.
Returns
--------
flowserv.volume.base.IOHandle
"""
# The file key is a path expression that uses '/' as the path separator.
# If the local OS uses a different separator we need to replace it.
filename = util.filepath(key=key, sep=self.client.sep)
filename = self.client.sep.join([self.remotedir, filename])
return SFTPFile(filename=filename, client=self.client)
def load(self, key: str) -> IOHandle:
"""Load a file object at the source path of this volume store.
Returns a file handle that can be used to open and read the file.
Parameters
----------
key: str
Path to a file object in the storage volume.
Returns
--------
flowserv.volume.base.IOHandle
"""
# The file key is a path expression that uses '/' as the path separator.
# If the local OS uses a different separator we need to replace it.
filename = os.path.join(self.basedir, util.filepath(key=key))
if not os.path.isfile(filename):
raise err.UnknownFileError(filename)
return FSFile(filename=filename)
def get_store_for_folder(self,
key: str,
identifier: Optional[str] = None
) -> StorageVolume:
"""Get storage volume for a sob-folder of the given volume.
Parameters
----------
key: string
Relative path to sub-folder. The concatenation of the base folder
for this storage volume and the given key will form te new base
folder for the returned storage volume.
identifier: string, default=None
Unique volume identifier.
Returns
-------
flowserv.volume.base.StorageVolume
"""
return FileSystemStorage(basedir=os.path.join(self.basedir,
util.filepath(key=key)),
identifier=identifier)
def store(self, file: IOHandle, dst: str):
"""Store a given file object at the destination path of this volume
store.
Parameters
----------
file: flowserv.volume.base.IOHandle
File-like object that is being stored.
dst: str
Destination path for the stored object.
"""
# The file key is a path expression that uses '/' as the path separator.
# If the local OS uses a different separator we need to replace it.
filename = util.filepath(key=dst, sep=self.client.sep)
filename = self.client.sep.join([self.remotedir, filename])
dirname = self.client.sep.join(filename.split(self.client.sep)[:-1])
sftp = self.client.sftp()
try:
sftp_mkdir(client=sftp, dirpath=dirname)
with sftp.open(filename, 'wb') as fout:
with file.open() as fin:
fout.write(fin.read())
finally:
sftp.close()
def get_store_for_folder(self,
key: str,
identifier: Optional[str] = None
) -> StorageVolume:
"""Get storage volume for a sob-folder of the given volume.
Parameters
----------
key: string
Relative path to sub-folder. The concatenation of the base folder
for this storage volume and the given key will form te new base
folder for the returned storage volume.
identifier: string, default=None
Unique volume identifier.
Returns
-------
flowserv.volume.base.StorageVolume
"""
dirpath = util.filepath(key=key, sep=self.client.sep)
return RemoteStorage(client=self.client,
remotedir=self.client.sep.join(
[self.remotedir, dirpath]),
identifier=identifier)
def test_filepath(key, sep, result):
"""Test the file path transformation function."""
assert util.filepath(key=key, sep=sep) == result
