def walk(self, src: str) -> List[Tuple[str, IOHandle]]: """Get list of all files at the given source path. If the source path references a single file the returned list will contain a single entry. If the source specifies a folder the result contains a list of all files in that folder and the subfolders. Parameters ---------- src: str Source path specifying a file or folder. Returns ------- list of tuples (str, flowserv.volume.base.IOHandle) """ dirpath = util.filepath(key=src, sep=self.client.sep) dirpath = self.client.sep.join([self.remotedir, dirpath ]) if dirpath else self.remotedir files = self.client.walk(dirpath=dirpath) if files is None: # The source path references a single file. filename = util.filepath(key=src, sep=self.client.sep) filename = self.client.sep.join([self.remotedir, filename]) return [(src, SFTPFile(filename=filename, client=self.client))] else: # The source path references a directory. result = list() for key in files: key = util.join(src, key) if src else key filename = util.filepath(key=key, sep=self.client.sep) filename = self.client.sep.join([self.remotedir, filename]) result.append( (key, SFTPFile(filename=filename, client=self.client))) return result
def delete(self, key: str): """Delete file or folder with the given key. Parameters ---------- key: str Path to a file object in the storage volume. """ sftp = self.client.sftp() # Get recursive list of all files in the base folder and delete them. dirpath = util.filepath(key=key, sep=self.client.sep) dirpath = self.client.sep.join([self.remotedir, dirpath ]) if dirpath else self.remotedir files = self.client.walk(dirpath=dirpath) if files is None: filename = util.filepath(key=key, sep=self.client.sep) filename = self.client.sep.join([self.remotedir, filename]) sftp.remove(filename) else: # Collect sub-directories that need to be removed separately after # the directories are empty. directories = set() for src in files: filename = util.filepath(key=src, sep=self.client.sep) filename = self.client.sep.join([self.remotedir, filename]) dirname = util.dirname(src) if dirname: directories.add(dirname) sftp.remove(filename) for dirpath in sorted(directories, reverse=True): dirname = util.filepath(key=dirpath, sep=self.client.sep) dirname = self.client.sep.join( [self.remotedir, dirname]) if dirname else self.remotedir sftp.rmdir(dirname)
def walk(self, src: str) -> List[Tuple[str, IOHandle]]: """Get list of all files at the given source path. If the source path references a single file the returned list will contain a single entry. If the source specifies a folder the result contains a list of all files in that folder and the subfolders. Parameters ---------- src: str Source path specifying a file or folder. Returns ------- list of tuples (str, flowserv.volume.base.IOHandle) """ # The file key is a path expression that uses '/' as the path separator. # If the local OS uses a different separator we need to replace it. filename = os.path.join(self.basedir, util.filepath( key=src)) if src else self.basedir # The result is a list of file keys. result = list() if os.path.isfile(filename): # If the source key references a file return a list with the key as # the only element. result.append((src, FSFile(filename=filename))) elif os.path.isdir(filename): # Recursively append all files in the referenced directory to the # result list. walkdir(filename, src, result) return result
def mkdir(self, path: str): """Create the directory with the given (relative) path and all of its parent directories. Does not raise an error if the directory exists. Parameters ---------- path: string Relative path to a directory in the storage volume. """ dirname = os.path.join(self.basedir, util.filepath(key=path)) os.makedirs(dirname, exist_ok=True)
def store(self, file: IOHandle, dst: str): """Store a given file object at the destination path of this volume store. Parameters ---------- file: flowserv.volume.base.IOHandle File-like object that is being stored. dst: str Destination path for the stored object. """ # The file key is a path expression that uses '/' as the path separator. # If the local OS uses a different separator we need to replace it. filename = os.path.join(self.basedir, util.filepath(key=dst)) os.makedirs(os.path.dirname(filename), exist_ok=True) with open(filename, 'wb') as fout: with file.open() as fin: fout.write(fin.read())
def mkdir(self, path: str): """Create the directory with the given (relative) path and all of its parent directories. For bucket stores no directories need to be created prior to accessing them. Parameters ---------- path: string Relative path to a directory in the storage volume. """ dirname = util.filepath(key=path, sep=self.client.sep) dirname = self.client.sep.join([self.remotedir, dirname]) sftp = self.client.sftp() try: sftp_mkdir(client=sftp, dirpath=dirname) finally: sftp.close()
def load(self, key: str) -> IOHandle: """Load a file object at the source path of this volume store. Returns a file handle that can be used to open and read the file. Parameters ---------- key: str Path to a file object in the storage volume. Returns -------- flowserv.volume.base.IOHandle """ # The file key is a path expression that uses '/' as the path separator. # If the local OS uses a different separator we need to replace it. filename = util.filepath(key=key, sep=self.client.sep) filename = self.client.sep.join([self.remotedir, filename]) return SFTPFile(filename=filename, client=self.client)
def load(self, key: str) -> IOHandle: """Load a file object at the source path of this volume store. Returns a file handle that can be used to open and read the file. Parameters ---------- key: str Path to a file object in the storage volume. Returns -------- flowserv.volume.base.IOHandle """ # The file key is a path expression that uses '/' as the path separator. # If the local OS uses a different separator we need to replace it. filename = os.path.join(self.basedir, util.filepath(key=key)) if not os.path.isfile(filename): raise err.UnknownFileError(filename) return FSFile(filename=filename)
def get_store_for_folder(self, key: str, identifier: Optional[str] = None ) -> StorageVolume: """Get storage volume for a sob-folder of the given volume. Parameters ---------- key: string Relative path to sub-folder. The concatenation of the base folder for this storage volume and the given key will form te new base folder for the returned storage volume. identifier: string, default=None Unique volume identifier. Returns ------- flowserv.volume.base.StorageVolume """ return FileSystemStorage(basedir=os.path.join(self.basedir, util.filepath(key=key)), identifier=identifier)
def store(self, file: IOHandle, dst: str): """Store a given file object at the destination path of this volume store. Parameters ---------- file: flowserv.volume.base.IOHandle File-like object that is being stored. dst: str Destination path for the stored object. """ # The file key is a path expression that uses '/' as the path separator. # If the local OS uses a different separator we need to replace it. filename = util.filepath(key=dst, sep=self.client.sep) filename = self.client.sep.join([self.remotedir, filename]) dirname = self.client.sep.join(filename.split(self.client.sep)[:-1]) sftp = self.client.sftp() try: sftp_mkdir(client=sftp, dirpath=dirname) with sftp.open(filename, 'wb') as fout: with file.open() as fin: fout.write(fin.read()) finally: sftp.close()
def get_store_for_folder(self, key: str, identifier: Optional[str] = None ) -> StorageVolume: """Get storage volume for a sob-folder of the given volume. Parameters ---------- key: string Relative path to sub-folder. The concatenation of the base folder for this storage volume and the given key will form te new base folder for the returned storage volume. identifier: string, default=None Unique volume identifier. Returns ------- flowserv.volume.base.StorageVolume """ dirpath = util.filepath(key=key, sep=self.client.sep) return RemoteStorage(client=self.client, remotedir=self.client.sep.join( [self.remotedir, dirpath]), identifier=identifier)
def test_filepath(key, sep, result): """Test the file path transformation function.""" assert util.filepath(key=key, sep=sep) == result