def delete_file(self, group_id, file_id):
"""Delete uploaded group file with given identifier. Raises an error if
the group or file does not exist.
Parameters
----------
group_id: string
Unique group identifier
file_id: string
Unique file identifier
Raises
------
flowserv.error.UnknownWorkflowGroupError
flowserv.error.UnknownFileError
"""
# Get the group to ensure that it exists.
group = self.get_group(group_id)
# Get handle for the file.
file_key = None
for i, file in enumerate(group.uploads):
if file.file_id == file_id:
del group.uploads[i]
file_key = file.key
# No file with matching identifier was found.
if file_key is None:
raise err.UnknownFileError(file_id)
# If deleting the database record was successful delete the file on
# disk.
self.fs.delete(key=file_key)
def get_uploaded_file(self, group_id: str, file_id: str) -> FileHandle:
"""Get handle for an uploaded group file with the given identifier.
Raises an error if the group or the file does not exists.
Returns the file handle and an object that provides read access to the
file contents. The object may either be the path to the file on disk
or a IOHandle.
Parameters
----------
group_id: string
Unique group identifier
file_id: string
Unique file identifier
Returns
-------
flowserv.model.files.FileHandle
Raises
------
flowserv.error.UnknownWorkflowGroupError
flowserv.error.UnknownFileError
"""
group = self.get_group(group_id)
for fh in group.uploads:
if fh.file_id == file_id:
return FileHandle(name=fh.name,
mime_type=fh.mime_type,
fileobj=self.fs.load(key=fh.key))
# No file with matching identifier was found.
raise err.UnknownFileError(file_id)
def get_result_file(self, workflow_id: str, file_id: str) -> FileHandle:
"""Get file handle for a file that was generated as the result of a
successful post-processing workflow run.
Parameters
----------
workflow_id: string
Unique workflow identifier.
file_id: string
Unique resource file identifier.
Returns
-------
flowserv.model.files.base.FileHandle
Raises
------
flowserv.error.UnknownWorkflowError
flowserv.error.UnknownFileError
"""
# Get the workflow handle. This will raise an error if the workflow
# does not exist.
workflow = self.workflow_repo.get_workflow(workflow_id)
# Ensure that the post-processing run exists and that it is in SUCCESS
# state.
if workflow.postproc_run_id is None:
raise err.UnknownFileError(file_id)
# Return the result file from the run manager.
return self.run_manager.get_runfile(run_id=workflow.postproc_run_id,
file_id=file_id)
def get_result_archive(self, workflow_id: str) -> FileHandle:
"""Get compressed tar-archive containing all result files that were
generated by the most recent post-processing workflow. If the workflow
does not have a post-processing step or if the post-processing workflow
run is not in SUCCESS state, a unknown resource error is raised.
Parameters
----------
workflow_id: string
Unique workflow identifier.
Returns
-------
flowserv.model.files.base.FileHandle
Raises
------
flowserv.error.UnknownWorkflowError
flowserv.error.UnknownFileError
"""
# Get the workflow handle. This will raise an error if the workflow
# does not exist.
workflow = self.workflow_repo.get_workflow(workflow_id)
# Ensure that the post-processing run exists and that it is in SUCCESS
# state
if workflow.postproc_run_id is None:
raise err.UnknownFileError('no post-processing workflow')
return self.run_manager.get_runarchive(run_id=workflow.postproc_run_id)
def __init__(self, filename: str, raise_error: Optional[bool] = True):
"""Initialize the file name that points to a file on disk.
Parameters
----------
filename: string
Path to an existing file on disk.
raise_error: bool, default=True
Raise error if the given file name does not reference an existing
file.
"""
if raise_error and not os.path.isfile(filename):
raise err.UnknownFileError(filename)
self.filename = filename
def open(self) -> IO:
"""Get file contents as a BytesIO buffer.
Returns
-------
io.BytesIO
Raises
------
flowserv.error.UnknownFileError
"""
if not os.path.isfile(self.filename):
raise err.UnknownFileError(self.filename)
return util.read_buffer(self.filename)
def open(self) -> IO:
"""Get file contents as a BytesIO buffer.
Returns
-------
io.BytesIO
"""
# Load object into a new bytes buffer.
data = BytesIO()
try:
self.bucket.download_fileobj(self.key, data)
except botocore.exceptions.ClientError:
raise err.UnknownFileError(self.key)
# Ensure to reset the read pointer of the buffer before returning it.
data.seek(0)
return data
def open(self) -> IO:
"""Get file contents as a BytesIO buffer.
Returns
-------
io.BytesIO
"""
blob = self.client.bucket(self.bucket_name).blob(self.key)
# Load object into a new bytes buffer.
from google.cloud import exceptions
try:
data = BytesIO(blob.download_as_bytes())
except exceptions.NotFound:
raise err.UnknownFileError(self.key)
# Ensure to reset the read pointer of the buffer before returning it.
data.seek(0)
return data
def get_runfile(
self, run_id: str, file_id: str = None, key: str = None
) -> FileHandle:
"""Get handle and file object for a given run result file. The file is
either identified by the unique file identifier or the file key. Raises
an error if the specified file does not exist.
Parameters
----------
run_id: string
Unique run identifier.
file_id: string
Unique file identifier.
Returns
-------
flowserv.model.files.base.FileHandle
Raises
------
flowserv.error.UnknownFileError
ValueError
"""
# Raise an error if both or neither file_id and key are given.
if file_id is None and key is None:
raise ValueError('no arguments for file_id or key')
elif not (file_id is None or key is None):
raise ValueError('invalid arguments for file_id and key')
run = self.get_run(run_id)
if file_id:
fh = run.get_file(by_id=file_id)
else:
fh = run.get_file(by_key=key)
if fh is None:
raise err.UnknownFileError(file_id)
# Return file handle for resource file
workflow_id = run.workflow.workflow_id
rundir = self.fs.run_basedir(workflow_id=workflow_id, run_id=run_id)
return FileHandle(
name=fh.name,
mime_type=fh.mime_type,
fileobj=self.fs.load_file(os.path.join(rundir, fh.key))
)
def load(self, key: str) -> IOHandle:
"""Load a file object at the source path of this volume store.
Returns a file handle that can be used to open and read the file.
Parameters
----------
key: str
Path to a file object in the storage volume.
Returns
--------
flowserv.volume.base.IOHandle
"""
# The file key is a path expression that uses '/' as the path separator.
# If the local OS uses a different separator we need to replace it.
filename = os.path.join(self.basedir, util.filepath(key=key))
if not os.path.isfile(filename):
raise err.UnknownFileError(filename)
return FSFile(filename=filename)
