def run_workflow(run_id: str, rundir: str, state: WorkflowState,
output_files: List[str], steps: List[ContainerStep],
arguments: Dict,
workers: WorkerFactory) -> Tuple[str, str, Dict]:
"""Execute a list of workflow steps synchronously.
This is the worker function for asynchronous workflow executions. Returns a
tuple containing the run identifier, the folder with the run files, and a
serialization of the workflow state.
Parameters
----------
run_id: string
Unique run identifier
rundir: string
Path to the working directory of the workflow run
state: flowserv.model.workflow.state.WorkflowState
Current workflow state (to access the timestamps)
output_files: list(string)
Relative path of output files that are generated by the workflow run
steps: list of flowserv.model.workflow.step.WorkflowStep
Steps in the serial workflow that are executed in the given context.
arguments: dict
Dictionary of argument values for parameters in the template.
workers: flowserv.controller.worker.factory.WorkerFactory, default=None
Factory for :class:`flowserv.model.workflow.step.ContainerStep` steps.
Returns
-------
(string, string, dict)
"""
logging.info('start run {}'.format(run_id))
try:
run_result = exec_workflow(steps=steps,
workers=workers,
rundir=rundir,
result=RunResult(arguments=arguments))
if run_result.returncode != 0:
# Return error state. Include STDERR in result
messages = run_result.log
result_state = state.error(messages=messages)
doc = serialize.serialize_state(result_state)
return run_id, rundir, doc
# Create list of output files that were generated.
files = list()
for relative_path in output_files:
if os.path.exists(os.path.join(rundir, relative_path)):
files.append(relative_path)
# Workflow executed successfully
result_state = state.success(files=files)
except Exception as ex:
logging.error(ex)
strace = util.stacktrace(ex)
logging.debug('\n'.join(strace))
result_state = state.error(messages=strace)
logging.info('finished run {}: {}'.format(run_id, result_state.type_id))
return run_id, rundir, serialize.serialize_state(result_state)
def run_workflow(run_id: str, state: WorkflowState, output_files: List[str],
steps: List[ContainerStep], arguments: Dict,
volumes: VolumeManager,
workers: WorkerPool) -> Tuple[str, str, Dict]:
"""Execute a list of workflow steps synchronously.
This is the worker function for asynchronous workflow executions. Returns a
tuple containing the run identifier, the folder with the run files, and a
serialization of the workflow state.
Parameters
----------
run_id: string
Unique run identifier
state: flowserv.model.workflow.state.WorkflowState
Current workflow state (to access the timestamps)
output_files: list(string)
Relative path of output files that are generated by the workflow run
steps: list of flowserv.model.workflow.step.WorkflowStep
Steps in the serial workflow that are executed in the given context.
arguments: dict
Dictionary of argument values for parameters in the template.
volumes: flowserv.volume.manager.VolumeManager
Factory for storage volumes.
workers: flowserv.controller.worker.manager.WorkerPool
Factory for :class:`flowserv.model.workflow.step.ContainerStep` steps.
Returns
-------
(string, string, dict)
"""
logging.info('start run {}'.format(run_id))
runstore = volumes.get(DEFAULT_STORE)
try:
run_result = exec_workflow(steps=steps,
workers=workers,
volumes=volumes,
result=RunResult(arguments=arguments))
if run_result.returncode != 0:
# Return error state. Include STDERR in result
messages = run_result.log
result_state = state.error(messages=messages)
doc = serialize.serialize_state(result_state)
return run_id, runstore.to_dict(), doc
# Workflow executed successfully
result_state = state.success(files=output_files)
except Exception as ex:
logging.error(ex, exc_info=True)
strace = util.stacktrace(ex)
logging.debug('\n'.join(strace))
result_state = state.error(messages=strace)
logging.info('finished run {}: {}'.format(run_id, result_state.type_id))
return run_id, runstore.to_dict(), serialize.serialize_state(result_state)
def test_running_state():
"""Test serialization/deserialization of running states."""
s = state.StateRunning(created_at=util.to_datetime(CREATED_AT),
started_at=util.to_datetime(STARTED_AT))
s = state.deserialize_state(state.serialize_state(s))
assert s.is_running()
validate_date(s.created_at, util.to_datetime(CREATED_AT))
validate_date(s.started_at, util.to_datetime(STARTED_AT))
def test_error_state():
"""Test serialization/deserialization of error states."""
s = state.StateError(created_at=util.to_datetime(CREATED_AT),
started_at=util.to_datetime(STARTED_AT),
stopped_at=util.to_datetime(FINISHED_AT),
messages=['there', 'were', 'errors'])
s = state.deserialize_state(state.serialize_state(s))
assert s.is_error()
assert s.messages == ['there', 'were', 'errors']
validate_date(s.created_at, util.to_datetime(CREATED_AT))
validate_date(s.started_at, util.to_datetime(STARTED_AT))
validate_date(s.stopped_at, util.to_datetime(FINISHED_AT))
def test_success_state(tmpdir):
"""Test serialization/deserialization of success states."""
filename = os.path.join(str(tmpdir), 'results.json')
util.write_object(filename=filename, obj={'A': 1})
s = state.StateSuccess(created_at=util.to_datetime(CREATED_AT),
started_at=util.to_datetime(STARTED_AT),
finished_at=util.to_datetime(FINISHED_AT),
files=['myfile1', 'myfile2'])
s = state.deserialize_state(state.serialize_state(s))
assert s.is_success()
assert len(s.files) == 2
assert 'myfile1' in s.files
assert 'myfile2' in s.files
validate_date(s.created_at, util.to_datetime(CREATED_AT))
validate_date(s.started_at, util.to_datetime(STARTED_AT))
validate_date(s.finished_at, util.to_datetime(FINISHED_AT))
def test_pending_state():
"""Test serialization/deserialization of pending states."""
s = state.StatePending(created_at=util.to_datetime(CREATED_AT))
s = state.deserialize_state(state.serialize_state(s))
assert s.is_pending()
validate_date(s.created_at, util.to_datetime(CREATED_AT))