def __init__(
self, created_at, started_at=None, stopped_at=None, messages=None
):
"""Initialize the timestamps that are associated with the workflow
state and the optional error messages.
Parameters
----------
created_at: string
Timestamp of workflow creation
started_at: string
Timestamp when the workflow started running
stopped_at: string, optional
Timestamp when workflow error occurred or the when the workflow was
canceled
messages: list(string), optional
Optional list of error messages
"""
super(StateError, self).__init__(
type_id=STATE_ERROR,
created_at=created_at
)
self.started_at = started_at if started_at is not None else created_at
self.stopped_at = stopped_at if stopped_at is not None else utc_now()
self.messages = messages if messages is not None else list()
def test_workflow_leaderboard_serialization(database, tmpdir):
"""Test serialization of a workflow leaderboard."""
config = Config().basedir(tmpdir)
schema = validator('WorkflowLeaderboard')
view = WorkflowSerializer()
with database.session() as session:
manager = WorkflowManager(session=session, fs=FileSystemStore(config))
workflow = manager.create_workflow(source=BENCHMARK_DIR,
name='Test',
specfile=SPEC_FILE)
ts = util.utc_now()
ranking = [
RunResult(run_id='0',
group_id='1',
group_name='A',
created_at=ts,
started_at=ts,
finished_at=ts,
values={
'len': 1,
'count': 10
})
]
doc = view.workflow_leaderboard(workflow, ranking=ranking)
schema.validate(doc)
def __init__(self,
created_at: str,
started_at: str,
finished_at: Optional[str] = None,
files: Optional[List[str]] = None):
"""Initialize the timestamps that are associated with the workflow
state and the list of created files.
Parameters
----------
created_at: string
Timestamp of workflow creation
started_at: string
Timestamp when the workflow started running
finished_at: string, optional
Timestamp when workflow execution completed
files: list(string), default=None
Optional list of created files (relative path).
"""
super(StateSuccess, self).__init__(type_id=STATE_SUCCESS,
created_at=created_at)
self.started_at = started_at
self.finished_at = finished_at if finished_at is not None else utc_now(
) # noqa: E501
self.files = files if files is not None else list()
def test_obsolete_runs(fscls, database, tmpdir):
"""Test deleting runs that were created before a given date."""
# -- Setup ----------------------------------------------------------------
fs = fscls(env=Config().basedir(tmpdir))
# Create two runs (one SUCCESS and one ERROR) before a timestamp t1
_, _, run_1, _ = success_run(database, fs, tmpdir)
_, _, run_2 = error_run(database, fs, ['There were errors'])
time.sleep(1)
t1 = util.utc_now()
# Create another SUCCESS run after timestamp t1
_, _, run_3, _ = success_run(database, fs, tmpdir)
# -- Test delete run with state filter ------------------------------------
with database.session() as session:
runs = RunManager(session=session, fs=fs)
assert runs.delete_obsolete_runs(date=t1, state=st.STATE_ERROR) == 1
# After deleting the error run the two success runs still exist.
runs.get_run(run_id=run_1)
with pytest.raises(err.UnknownRunError):
runs.get_run(run_id=run_2)
runs.get_run(run_id=run_3)
# -- Test delete all runs prior to a given date ---------------------------
with database.session() as session:
runs = RunManager(session=session, fs=fs)
assert runs.delete_obsolete_runs(date=t1) == 1
# After deleting the run the only one success runs still exist.
with pytest.raises(err.UnknownRunError):
runs.get_run(run_id=run_1)
runs.get_run(run_id=run_3)
def run_success(run_manager, run_id, store, values):
"""Set given run into success state with the given result data."""
store.store(file=io_file(values), dst=RESULT_FILE_ID)
ts = util.utc_now()
run_manager.update_run(run_id=run_id,
state=st.StateSuccess(created_at=ts,
started_at=ts,
finished_at=ts,
files=[RESULT_FILE_ID]),
runstore=store)
def __init__(self, type_id, created_at=None):
"""Initialize the type identifier and the 'created at' timestamp.
Parameters
----------
type_id: string
Type identifier
"""
self.type_id = type_id
self.created_at = created_at if created_at is not None else utc_now()
def upload_file(self, group_id: str, file: IOHandle, name: str):
"""Upload a new file for a workflow group. This will create a copy of
the given file in the file store that is associated with the group. The
file will be places in a unique folder inside the groups upload folder.
Raises an error if the given file name is invalid.
Parameters
----------
group_id: string
Unique group identifier
file: flowserv.model.files.base.IOHandle
File object (e.g., uploaded via HTTP request)
name: string
Name of the file
Returns
-------
flowserv.model.base.UploadFile
Raises
------
flowserv.error.ConstraintViolationError
flowserv.error.UnknownWorkflowGroupError
"""
# Get the group object to ensure that the group exists.
group = self.get_group(group_id)
# Ensure that the given file name is valid
constraint.validate_name(name)
# Create a new unique identifier for the file and save the file object
# to the new file path.
file_id = util.get_unique_identifier()
uploaddir = self.fs.group_uploaddir(
workflow_id=group.workflow_id,
group_id=group.group_id
)
# Get file size.
file_size = file.size()
# Attempt to guess the Mime type for the uploaded file from the file
# name.
mime_type, _ = mimetypes.guess_type(url=name)
self.fs.store_files(files=[(file, file_id)], dst=uploaddir)
# Insert information into database and return handle for uploaded file.
fileobj = UploadFile(
file_id=file_id,
created_at=util.utc_now(),
key=os.path.join(uploaddir, file_id),
name=name,
mime_type=mime_type,
size=file_size
)
group.uploads.append(fileobj)
return fileobj
def __init__(self, type_id: str, created_at: Optional[str] = None):
"""Initialize the type identifier and the 'created at' timestamp.
Parameters
----------
type_id: string
Workflow state type identifier.
created_at: str, default=None
Timestamp when the workflow run was first created.
"""
self.type_id = type_id
self.created_at = created_at if created_at is not None else utc_now()
def __init__(self, created_at: str, started_at: Optional[str] = None):
"""Initialize the timestamps that are associated with the workflow
state.
Parameters
----------
created_at: string
Timestamp of workflow creation
started_at: string
Timestamp when the workflow started running
"""
super(StateRunning, self).__init__(type_id=STATE_RUNNING,
created_at=created_at)
self.started_at = started_at if started_at is not None else utc_now()
def run_success(run_manager, run_id, rundir, values):
"""Set given run into success state with the given result data."""
os.makedirs(rundir)
filename = os.path.join(rundir, RESULT_FILE_ID)
util.write_object(filename=filename, obj=values)
ts = util.utc_now()
run_manager.update_run(
run_id=run_id,
state=st.StateSuccess(
created_at=ts,
started_at=ts,
finished_at=ts,
files=[RESULT_FILE_ID]
),
rundir=rundir
)
def cancel(self, messages: Optional[List[str]] = None) -> StateCanceled:
"""Get instance of canceled state for a pending wokflow.
Since the workflow did not start to run the started_at timestamp is set
to the current time just like the stopped_at timestamp.
Parameters
----------
messages: list(string), optional
Optional list of messages
Returns
-------
flowserv.model.workflow.state.StateCanceled
"""
ts = utc_now()
return StateCanceled(created_at=self.created_at,
started_at=ts,
stopped_at=ts,
messages=messages)
def error(self, messages: Optional[List[str]] = None) -> StateError:
"""Get instance of error state for a pending wokflow. If the exception
that caused the workflow execution to terminate is given it will be
used to create the list of error messages.
Since the workflow did not start to run the started_at timestamp is set
to the current time just like the stopped_at timestamp.
Parameters
----------
messages: list(string), optional
Optional list of error messages
Returns
-------
flowserv.model.workflow.state.StateError
"""
ts = utc_now()
return StateError(created_at=self.created_at,
started_at=ts,
stopped_at=ts,
messages=messages)
def __init__(self,
created_at: str,
started_at: Optional[str] = None,
stopped_at: Optional[str] = None,
messages: Optional[List[str]] = None):
"""Initialize the timestamps that are associated with the workflow
state and the optional messages.
Parameters
----------
created_at: string
Timestamp of workflow creation
started_at: string
Timestamp when the workflow started running
stopped_at: string, optional
Timestamp when workflow was canceled
messages: list(string), optional
Optional list of messages
"""
super(StateCanceled, self).__init__(type_id=STATE_CANCELED,
created_at=created_at)
self.started_at = started_at if started_at is not None else created_at
self.stopped_at = stopped_at if stopped_at is not None else utc_now()
self.messages = messages if messages is not None else CANCELED