def test_list_runs(fscls, database, tmpdir):
"""Test retrieving a list of run descriptors."""
# -- Setup ----------------------------------------------------------------
#
# Create two runs: one in running state and one in error state.
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
group = groups.get_group(group_id)
# Run 1 in running state
r = runs.create_run(group=group)
run_1 = r.run_id
runs.update_run(run_id=run_1, state=r.state().start())
r = runs.create_run(group=group)
run_2 = r.run_id
runs.update_run(run_id=run_2, state=r.state().error())
# -- Test get listing -----------------------------------------------------
with database.session() as session:
runs = RunManager(session=session, fs=fs)
run_index = dict()
for run in runs.list_runs(group_id):
run_index[run.run_id] = run
assert len(run_index) == 2
assert run_index[run_1].state().is_running()
assert run_index[run_2].state().is_error()
# -- Test polling runs ----------------------------------------------------
with database.session() as session:
runs = RunManager(session=session, fs=fs)
assert len(runs.list_runs(group_id)) == 2
assert len(runs.list_runs(group_id, state=st.STATE_ERROR)) == 1
assert len(runs.list_runs(group_id, state=st.STATE_SUCCESS)) == 0
def success_run(database: DB, fs: StorageVolume,
basedir: str) -> Tuple[str, str, str, str]:
"""Create a successful run with two result files:
- A.json
- results/B.json
Returns the identifier of the created workflow, group, run, and user.
"""
# Setup temporary run folder.
runfs = FileSystemStorage(basedir=os.path.join(basedir, 'tmprun'))
runfs.store(file=io_file({'A': 1}), dst='A.json')
runfs.store(file=io_file({'B': 1}), dst=util.join('results', 'B.json'))
with database.session() as session:
user_id = create_user(session, active=True)
workflow_id = create_workflow(session)
group_id = create_group(session, workflow_id, users=[user_id])
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(
run_id=run_id,
state=state.start().success(files=['A.json', 'results/B.json']),
runstore=runfs)
return workflow_id, group_id, run_id, user_id
def success_run(database: DB, fs: FileStore,
basedir: str) -> Tuple[str, str, str, str]:
"""Create a successful run with two result files:
- A.json
- run/results/B.json
Returns the identifier of the created workflow, group, run, and user.
"""
# Setup temporary run folder.
tmprundir = os.path.join(basedir, 'tmprun')
tmpresultsdir = os.path.join(tmprundir, 'run', 'results')
os.makedirs(tmprundir)
os.makedirs(tmpresultsdir)
f1 = os.path.join(tmprundir, 'A.json')
util.write_object(f1, {'A': 1})
f2 = os.path.join(tmpresultsdir, 'B.json')
util.write_object(f2, {'B': 1})
with database.session() as session:
user_id = create_user(session, active=True)
workflow_id = create_workflow(session)
group_id = create_group(session, workflow_id, users=[user_id])
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(
run_id,
state.start().success(files=['A.json', 'run/results/B.json']),
rundir=tmprundir)
assert not os.path.exists(tmprundir)
return workflow_id, group_id, run_id, user_id
def test_cancel_run(fscls, database, tmpdir):
"""Test setting run state to canceled."""
# -- Setup ----------------------------------------------------------------
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# -- Test set run to error state ------------------------------------------
with database.session() as session:
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(run_id=run_id, state=state.cancel())
with database.session() as session:
runs = RunManager(session=session, fs=fs)
run = runs.get_run(run_id)
state = run.state()
assert not state.is_active()
assert not state.is_pending()
assert not state.is_running()
assert state.is_canceled()
assert not state.is_error()
assert not state.is_success()
assert len(state.messages) == 1
def test_invalid_state_transitions(fscls, database, tmpdir):
"""Test error cases for invalid state transitions."""
# -- Setup ----------------------------------------------------------------
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# -- Test set active run to pending ---------------------------------------
with database.session() as session:
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(run_id=run_id, state=state.start())
with pytest.raises(err.ConstraintViolationError):
runs.update_run(run_id=run_id, state=st.StatePending())
# Cancel run
with database.session() as session:
runs = RunManager(session=session, fs=fs)
runs.update_run(run_id=run_id, state=state.cancel())
# -- Test cannot set run to any of the inactive states --------------------
with database.session() as session:
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
assert runs.update_run(run_id=run_id, state=state.cancel()) is None
with pytest.raises(err.ConstraintViolationError):
runs.update_run(run_id=run_id, state=state.error())
with pytest.raises(err.ConstraintViolationError):
runs.update_run(run_id=run_id, state=state.success())
def test_run_parameters(database, tmpdir):
"""Test creating run with template arguments."""
# -- Setup ----------------------------------------------------------------
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# Prepare run arguments
filename = os.path.join(str(tmpdir), 'results.json')
util.write_object(filename=filename, obj={'A': 1})
arguments = [{'id': 'A', 'value': 10}, {'id': 'B', 'value': True}]
# -- Test create run with arguments ---------------------------------------
with database.session() as session:
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(
group=groups.get_group(group_id),
arguments=arguments
)
run_id = run.run_id
with database.session() as session:
runs = RunManager(session=session, fs=fs)
run = runs.get_run(run_id)
assert run.arguments == arguments
def test_run_serialization(database, tmpdir):
"""Test serialization of run handles and run listings."""
config = Config().basedir(tmpdir)
view = RunSerializer()
fs = FileSystemStore(config)
# Setup temporary run folder.
tmprundir = os.path.join(tmpdir, 'tmprun')
tmpresultsdir = os.path.join(tmprundir, 'run', 'results')
os.makedirs(tmprundir)
os.makedirs(tmpresultsdir)
f1 = os.path.join(tmprundir, 'A.json')
util.write_object(f1, {'A': 1})
# Create runs.
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# Create successful run.
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(
run_id,
state.start().success(files=['A.json', 'run/results/B.json']),
rundir=tmprundir)
run = runs.get_run(run_id)
doc = view.run_handle(run)
validator('RunHandle').validate(doc)
# Create error run.
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(run_id=run_id, state=state)
messages = ['There', 'were', 'many errors']
runs.update_run(run_id=run_id, state=state.error(messages))
run = runs.get_run(run_id)
doc = view.run_handle(run)
validator('RunHandle').validate(doc)
# Validate run listing.
doc = view.run_listing(runs=runs.list_runs(group_id))
validator('RunListing').validate(doc)
assert len(doc[labels.RUN_LIST]) == 2
def run_postproc_workflow(postproc_spec: Dict, workflow: WorkflowObject,
ranking: List, runs: List, run_manager: RunManager,
backend: WorkflowController):
"""Run post-processing workflow for a workflow template."""
workflow_spec = postproc_spec.get('workflow')
pp_inputs = postproc_spec.get('inputs', {})
pp_files = pp_inputs.get('files', [])
# Prepare temporary directory with result files for all
# runs in the ranking. The created directory is the only
# run argument
strace = None
try:
datadir = postutil.prepare_postproc_data(input_files=pp_files,
ranking=ranking,
run_manager=run_manager)
dst = pp_inputs.get('runs', postbase.RUNS_DIR)
run_args = {
postbase.PARA_RUNS: InputFile(source=FSFile(datadir), target=dst)
}
arg_list = [
serialize_arg(postbase.PARA_RUNS, serialize_fh(datadir, dst))
]
except Exception as ex:
logging.error(ex)
strace = util.stacktrace(ex)
run_args = dict()
arg_list = []
# Create a new run for the workflow. The identifier for the run group is
# None.
run = run_manager.create_run(workflow=workflow,
arguments=arg_list,
runs=runs)
if strace is not None:
# If there were data preparation errors set the created run into an
# error state and return.
run_manager.update_run(run_id=run.run_id,
state=run.state().error(messages=strace))
else:
# Execute the post-processing workflow asynchronously if
# there were no data preparation errors.
postproc_state, rundir = backend.exec_workflow(
run=run,
template=WorkflowTemplate(workflow_spec=workflow_spec,
parameters=postbase.PARAMETERS),
arguments=run_args,
config=workflow.engine_config)
# Update the post-processing workflow run state if it is
# no longer pending for execution.
if not postproc_state.is_pending():
run_manager.update_run(run_id=run.run_id,
state=postproc_state,
rundir=rundir)
# Remove the temporary input folder
shutil.rmtree(datadir)
def error_run(database, fs, messages):
"""Create a run that is in error state. Returns the identifier of the
created workflow, group, and run.
"""
# Setup temporary run folder.
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(run_id=run_id, state=state)
messages = ['There', 'were', 'many errors']
runs.update_run(run_id=run_id, state=state.error(messages))
return workflow_id, group_id, run_id
def test_create_run_errors(fscls, database, tmpdir):
"""Test error cases for create_run parameter combinations."""
# -- Setup ----------------------------------------------------------------
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# -- Test create_run with invalid arguments -------------------------------
with database.session() as session:
wfrepo = WorkflowManager(session=session, fs=fs)
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
workflow = wfrepo.get_workflow(workflow_id)
group = groups.get_group(group_id)
with pytest.raises(ValueError):
runs.create_run()
with pytest.raises(ValueError):
runs.create_run(workflow=workflow, group=group)
with pytest.raises(ValueError):
runs.create_run(group=group, runs=['A'])
def run_postproc_workflow(workflow: WorkflowObject, ranking: List[RunResult],
keys: List[str], run_manager: RunManager,
tmpstore: StorageVolume, staticfs: StorageVolume,
backend: WorkflowController):
"""Run post-processing workflow for a workflow template.
Parameters
----------
workflow: flowserv.model.base.WorkflowObject
Handle for the workflow that triggered the post-processing workflow run.
ranking: list(flowserv.model.ranking.RunResult)
List of runs in the current result ranking.
keys: list of string
Sorted list of run identifier for runs in the ranking.
run_manager: flowserv.model.run.RunManager
Manager for workflow runs
tmpstore: flowserv.volume.base.StorageVolume
Temporary storage volume where the created post-processing files are
stored. This volume will be erased after the workflow is started.
staticfs: flowserv.volume.base.StorageVolume
Storage volume that contains the static files from the workflow
template.
backend: flowserv.controller.base.WorkflowController
Backend that is used to execute the post-processing workflow.
"""
# Get workflow specification and the list of input files from the
# post-processing statement.
postproc_spec = workflow.postproc_spec
workflow_spec = postproc_spec.get('workflow')
pp_inputs = postproc_spec.get('inputs', {})
pp_files = pp_inputs.get('files', [])
# Prepare temporary directory with result files for all
# runs in the ranking. The created directory is the only
# run argument
strace = None
try:
prepare_postproc_data(input_files=pp_files,
ranking=ranking,
run_manager=run_manager,
store=tmpstore)
dst = pp_inputs.get('runs', RUNS_DIR)
run_args = {PARA_RUNS: InputDirectory(store=tmpstore, target=RUNS_DIR)}
arg_list = [serialize_arg(PARA_RUNS, dst)]
except Exception as ex:
logging.error(ex, exc_info=True)
strace = util.stacktrace(ex)
run_args = dict()
arg_list = []
# Create a new run for the workflow. The identifier for the run group is
# None.
run = run_manager.create_run(workflow=workflow,
arguments=arg_list,
runs=keys)
if strace is not None:
# If there were data preparation errors set the created run into an
# error state and return.
run_manager.update_run(run_id=run.run_id,
state=run.state().error(messages=strace))
else:
# Execute the post-processing workflow asynchronously if
# there were no data preparation errors.
try:
postproc_state, runstore = backend.exec_workflow(
run=run,
template=WorkflowTemplate(workflow_spec=workflow_spec,
parameters=PARAMETERS),
arguments=run_args,
staticfs=staticfs,
config=workflow.engine_config)
except Exception as ex:
# Make sure to catch exceptions and set the run into an error state.
postproc_state = run.state().error(messages=util.stacktrace(ex))
runstore = None
# Update the post-processing workflow run state if it is
# no longer pending for execution.
if not postproc_state.is_pending():
run_manager.update_run(run_id=run.run_id,
state=postproc_state,
runstore=runstore)
# Erase the temporary storage volume.
tmpstore.erase()