def test_workflow_leaderboard_serialization(database, tmpdir):
"""Test serialization of a workflow leaderboard."""
config = Config().basedir(tmpdir)
schema = validator('WorkflowLeaderboard')
view = WorkflowSerializer()
with database.session() as session:
manager = WorkflowManager(session=session, fs=FileSystemStore(config))
workflow = manager.create_workflow(source=BENCHMARK_DIR,
name='Test',
specfile=SPEC_FILE)
ts = util.utc_now()
ranking = [
RunResult(run_id='0',
group_id='1',
group_name='A',
created_at=ts,
started_at=ts,
finished_at=ts,
values={
'len': 1,
'count': 10
})
]
doc = view.workflow_leaderboard(workflow, ranking=ranking)
schema.validate(doc)
def test_empty_ranking(database, tmpdir):
"""The rankings for workflows without completed runs are empty."""
# -- Setup ----------------------------------------------------------------
workflows = init(database, tmpdir)
fs = FileSystemStorage(basedir=tmpdir)
# -- Test empty listing with no successful runs ---------------------------
with database.session() as session:
wfrepo = WorkflowManager(session=session, fs=fs)
rankings = RankingManager(session=session)
for workflow_id, _ in workflows:
wf = wfrepo.get_workflow(workflow_id)
assert len(rankings.get_ranking(wf)) == 0
def test_create_workflow_with_alt_manifest(database, tmpdir):
"""Test creating 'Hello World' workflow with a different manifest file."""
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(source=BENCHMARK_DIR,
manifestfile=ALT_MANIFEST)
assert wf.name == 'Hello World'
assert wf.description is None
assert wf.instructions is None
template = wf.get_template()
assert template.result_schema is not None
def test_workflow_listing_serialization(database, tmpdir):
"""Test serialization of workflow listings."""
config = Config().basedir(tmpdir)
schema = validator('WorkflowListing')
view = WorkflowSerializer()
with database.session() as session:
manager = WorkflowManager(session=session, fs=FileSystemStore(config))
model.create_workflow(session)
model.create_workflow(session)
workflows = manager.list_workflows()
doc = view.workflow_listing(workflows)
schema.validate(doc)
assert len(doc[labels.WORKFLOW_LIST]) == 2
def test_get_workflow(database, tmpdir):
"""Test retrieving workflows from the repository."""
# -- Setup ----------------------------------------------------------------
#
# Create two workflows.
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(name='A', source=BENCHMARK_DIR)
workflow_1 = wf.workflow_id
wf = manager.create_workflow(name='B',
description='Workflow B',
source=BENCHMARK_DIR,
instructions=INSTRUCTION_FILE,
specfile=TEMPLATE_WITHOUT_SCHEMA)
workflow_2 = wf.workflow_id
# -- Test getting workflow handles ----------------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.get_workflow(workflow_1)
assert wf.name == 'A'
assert wf.description == 'Hello World Demo'
assert wf.instructions is not None
template = wf.get_template()
assert template.result_schema is not None
wf = manager.get_workflow(workflow_2)
assert wf.name == 'B'
assert wf.description == 'Workflow B'
assert wf.instructions == '# Hello World'
template = wf.get_template()
assert template.result_schema is None
def test_create_workflow_with_alt_manifest(fscls, database, tmpdir):
"""Test creating 'Hello World' workflow with a different manifest file."""
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(source=BENCHMARK_DIR,
manifestfile=ALT_MANIFEST)
assert wf.name == 'Hello World'
assert wf.description == 'Hello World Demo'
assert wf.instructions == '# Hello World'
template = wf.get_template()
assert template.result_schema is not None
staticdir = os.path.join(tmpdir, fs.workflow_staticdir(wf.workflow_id))
assert os.path.isfile(os.path.join(staticdir, 'code/helloworld.py'))
assert not os.path.isfile(os.path.join(staticdir, 'data/names.txt'))
def test_workflow_handle_serialization(database, tmpdir):
"""Test serialization of workflow handles."""
config = Config().basedir(tmpdir)
schema = validator('WorkflowHandle')
view = WorkflowSerializer()
with database.session() as session:
manager = WorkflowManager(session=session, fs=FileSystemStore(config))
workflow = manager.create_workflow(source=BENCHMARK_DIR,
name='Test',
specfile=SPEC_FILE)
doc = view.workflow_handle(workflow)
schema.validate(doc)
workflow = manager.get_workflow(workflow.workflow_id)
schema.validate(doc)
assert doc[labels.WORKFLOW_NAME] == 'Test'
def test_create_workflow_with_error(database, tmpdir):
"""Error cases when creating a workflow."""
# -- Setup ----------------------------------------------------------------
fs = FileSystemStorage(basedir=tmpdir)
# -- Invalid name ---------------------------------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
with pytest.raises(err.ConstraintViolationError):
manager.create_workflow(name=' ', source=BENCHMARK_DIR)
manager.create_workflow(name='a' * 512, source=BENCHMARK_DIR)
with pytest.raises(err.ConstraintViolationError):
manager.create_workflow(name='a' * 513, source=BENCHMARK_DIR)
# - Invalid template
with pytest.raises(err.UnknownParameterError):
manager.create_workflow(name='A benchmark',
source=BENCHMARK_DIR,
specfile=TEMPLATE_WITH_ERROR)
def test_workflow_name(database, tmpdir):
"""Test creating workflows with existing names."""
# -- Setup ----------------------------------------------------------------
# Initialize the repository. Create two workflows, one with name 'Workflow'
# and the other with name 'Workflow (2)'
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
manager.create_workflow(name='Workflow', source=BENCHMARK_DIR)
manager.create_workflow(name='Workflow (2)', source=BENCHMARK_DIR)
# Creating another workflow with name 'Workflow' will result in a workflow
# with name 'Workflow (1)' and the 'Workflow (3)'
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(name='Workflow', source=BENCHMARK_DIR)
assert wf.name == 'Workflow (1)'
wf = manager.create_workflow(name='Workflow', source=BENCHMARK_DIR)
assert wf.name == 'Workflow (3)'
def test_list_workflow(database, tmpdir):
"""Test deleting a workflows from the repository."""
# -- Setup ----------------------------------------------------------------
#
# Create two workflows.
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
manager.create_workflow(source=BENCHMARK_DIR)
manager.create_workflow(source=BENCHMARK_DIR)
manager.create_workflow(source=BENCHMARK_DIR)
# -- Test list workflows --------------------------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
workflows = manager.list_workflows()
assert len(workflows) == 3
def test_create_run_errors(fscls, database, tmpdir):
"""Test error cases for create_run parameter combinations."""
# -- Setup ----------------------------------------------------------------
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# -- Test create_run with invalid arguments -------------------------------
with database.session() as session:
wfrepo = WorkflowManager(session=session, fs=fs)
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
workflow = wfrepo.get_workflow(workflow_id)
group = groups.get_group(group_id)
with pytest.raises(ValueError):
runs.create_run()
with pytest.raises(ValueError):
runs.create_run(workflow=workflow, group=group)
with pytest.raises(ValueError):
runs.create_run(group=group, runs=['A'])
def test_multi_success_runs(database, tmpdir):
"""Test rankings for workflows where each group has multiple successful
runs.
"""
# -- Setup ----------------------------------------------------------------
# Create database with two workflows and four grous each. Each group has
# three active runs. Then set all runs for the first workflow into success
# state. Increase a counter for the avg_len value as we update runs.
workflows = init(database, tmpdir)
fs = FileSystemStorage(basedir=tmpdir)
workflow_id, groups = workflows[0]
count = 0
asc_order = list()
count_order = list()
desc_order = list()
with database.session() as session:
for group_id, runs in groups:
for i, run_id in enumerate(runs):
tmprundir = os.path.join(tmpdir, 'runs', run_id)
run_success(run_manager=RunManager(session=session, fs=fs),
run_id=run_id,
store=fs.get_store_for_folder(key=tmprundir),
values={
'count': count,
'avg': 1.0,
'name': run_id
})
count += 1
if i == 0:
asc_order.append(run_id)
count_order.append(run_id)
desc_order.append(run_id)
# -- Test get ranking with one result per group ---------------------------
with database.session() as session:
wfrepo = WorkflowManager(session=session, fs=fs)
rankings = RankingManager(session=session)
wf = wfrepo.get_workflow(workflow_id)
ranking = rankings.get_ranking(wf)
rank_order = [e.run_id for e in ranking]
assert rank_order == desc_order[::-1]
ranking = rankings.get_ranking(
wf, order_by=[SortColumn(column_id='count', sort_desc=False)])
rank_order = [e.run_id for e in ranking]
assert rank_order == asc_order
# Run execution time
assert type(ranking[0].exectime()) == timedelta
# -- Test get ranking with all results per group --------------------------
with database.session() as session:
wfrepo = WorkflowManager(session=session, fs=fs)
rankings = RankingManager(session=session)
wf = wfrepo.get_workflow(workflow_id)
ranking = rankings.get_ranking(wf, include_all=True)
rank_order = [e.run_id for e in ranking]
assert rank_order == count_order[::-1]
ranking = rankings.get_ranking(
wf,
order_by=[SortColumn(column_id='count', sort_desc=False)],
include_all=True)
rank_order = [e.run_id for e in ranking]
assert rank_order == count_order
def test_create_workflow(database, tmpdir):
"""Test creating workflows with different levels of detail."""
# -- Setup ----------------------------------------------------------------
fs = FileSystemStorage(basedir=tmpdir)
# -- Add workflow with minimal information --------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(source=BENCHMARK_DIR, identifier='WF001')
assert wf.workflow_id == 'WF001'
assert wf.name == 'Hello World'
assert wf.description == 'Hello World Demo'
assert wf.instructions is not None
template = wf.get_template()
assert template.result_schema is not None
# Ensure that the static files where copied to the workflow folder.
staticfs = fs.get_store_for_folder(
dirs.workflow_staticdir(workflow_id='WF001'))
files = {key for key, _ in staticfs.walk(src=None)}
assert files == {
'instructions.md', 'data/names.txt', 'code/analyze.py',
'code/postproc.py', 'code/helloworld.py', 'notebooks/HelloWorld.ipynb'
}
# -- Add workflow with user-provided metadata -----------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(
name='My benchmark',
description='My description',
instructions=INSTRUCTION_FILE,
source=BENCHMARK_DIR,
engine_config={'workers': {
'test': {
'worker': 'docker'
}
}})
assert wf.name == 'My benchmark'
assert wf.description == 'My description'
assert wf.instructions == '# Hello World'
wf.engine_config == {'workers': {'test': {'worker': 'docker'}}}
template = wf.get_template()
assert template.result_schema is not None
def test_create_workflow(fscls, identifier, database, tmpdir):
"""Test creating workflows with different levels of detail."""
# -- Setup ----------------------------------------------------------------
fs = fscls(env=Config().basedir(tmpdir))
# -- Add workflow with minimal information --------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(source=BENCHMARK_DIR,
identifier=identifier)
assert wf.workflow_id == identifier
assert wf.name == 'Hello World'
assert wf.description is None
assert wf.instructions is None
template = wf.get_template()
assert template.result_schema is not None
staticdir = os.path.join(tmpdir, fs.workflow_staticdir(wf.workflow_id))
assert os.path.isfile(os.path.join(staticdir, 'code/helloworld.py'))
assert os.path.isfile(os.path.join(staticdir, 'data/names.txt'))
# -- Add workflow with user-provided metadata -----------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(
name='My benchmark',
description='My description',
instructions=INSTRUCTION_FILE,
source=BENCHMARK_DIR,
engine_config={'workers': {
'test': {
'worker': 'docker'
}
}})
assert wf.name == 'My benchmark'
assert wf.description == 'My description'
assert wf.instructions == '# Hello World'
wf.engine_config == {'workers': {'test': {'worker': 'docker'}}}
template = wf.get_template()
assert template.result_schema is not None
staticdir = os.path.join(tmpdir, fs.workflow_staticdir(wf.workflow_id))
assert os.path.isfile(os.path.join(staticdir, 'code/helloworld.py'))
assert os.path.isfile(os.path.join(staticdir, 'data/names.txt'))
def test_create_workflow_with_alt_spec(database, tmpdir):
"""Test creating workflows with alternative specification files."""
# -- Setup ----------------------------------------------------------------
fs = FileSystemStorage(basedir=tmpdir)
# -- Template without schema ----------------------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(source=BENCHMARK_DIR,
specfile=TEMPLATE_WITHOUT_SCHEMA)
workflow_id = wf.workflow_id
assert wf.name == 'Hello World'
template = wf.get_template()
assert template.result_schema is None
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.get_workflow(workflow_id=workflow_id)
assert wf.name == 'Hello World'
template = wf.get_template()
assert template.result_schema is None
# -- Template with post-processing step -----------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(name='Top Tagger',
source=BENCHMARK_DIR,
specfile=TEMPLATE_TOPTAGGER)
workflow_id = wf.workflow_id
assert wf.get_template().postproc_spec is not None
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.get_workflow(workflow_id=workflow_id)
assert wf.get_template().postproc_spec is not None
def test_update_workflow_name(database, tmpdir):
"""Test updating workflow names."""
# -- Setup ----------------------------------------------------------------
#
# Create two workflow templates. Workflow 1 does not have a description
# and instructions while workflow 2 has.
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
# Initialize the repository
wf = manager.create_workflow(name='A', source=BENCHMARK_DIR)
workflow_1 = wf.workflow_id
wf = manager.create_workflow(name='My benchmark',
description='desc',
instructions=INSTRUCTION_FILE,
source=BENCHMARK_DIR)
workflow_2 = wf.workflow_id
# -- Test update workflow name --------------------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.update_workflow(workflow_id=workflow_1, name='B')
assert wf.name == 'B'
# It is possible to change the name to an existing name only if it is
# the same workflow.
wf = manager.update_workflow(workflow_id=workflow_2,
name='My benchmark')
assert wf.name == 'My benchmark'
# -- Error cases ----------------------------------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
# Cannot change name to existing name.
with pytest.raises(err.ConstraintViolationError):
manager.update_workflow(workflow_id=workflow_2, name='B')
def test_update_workflow_description(database, tmpdir):
"""Test updating the name and description of a workflow."""
# -- Setup ----------------------------------------------------------------
#
# Create one workflow without description and instructions.
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
# Initialize the repository
wf = manager.create_workflow(name='A', source=BENCHMARK_DIR)
workflow_id = wf.workflow_id
# -- Test update description and instruction text -------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.update_workflow(workflow_id=workflow_id,
description='The description',
instructions='The instructions')
assert wf.description == 'The description'
assert wf.instructions == 'The instructions'
# -- Test no update -------------------------------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.update_workflow(workflow_id=workflow_id)
assert wf.name == 'A'
assert wf.description == 'The description'
assert wf.instructions == 'The instructions'
# -- Test update all three properties -------------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.update_workflow(workflow_id=workflow_id,
name='A name',
description='A description',
instructions='Some instructions')
assert wf.name == 'A name'
assert wf.description == 'A description'
assert wf.instructions == 'Some instructions'
def __enter__(self) -> API:
"""Create a new instance of the local API when the context manager is
entered.
"""
# Open a new database session.
self._session = self._db.session()
session = self._session.open()
# Shortcuts for local variables.
env = self._env
fs = self._fs
engine = self._engine
# Start by creating the authorization component and setting the
# identifier for and authenticated user.
user_id = self._user_id
username = None
if env[AUTH] == config.AUTH_OPEN:
auth = OpenAccessAuth(session)
user_id = config.DEFAULT_USER if user_id is None else user_id
else:
auth = DefaultAuthPolicy(session)
access_token = self._access_token if self._access_token is not None else env.get(
ACCESS_TOKEN)
if access_token and user_id is None:
# If an access token is given we retrieve the user that is
# associated with the token. Authentication may raise an error.
# Here, we ignore that error since the token may be an outdated
# token that is stored in the environment.
try:
user = auth.authenticate(access_token)
# Set the user name for the authenticated user (to be
# included in the service descriptor).
username = user.name
user_id = user.user_id
except err.UnauthenticatedAccessError:
pass
# Create the individual components of the API.
ttl = env.get(config.FLOWSERV_AUTH_LOGINTTL, config.DEFAULT_LOGINTTL)
user_manager = UserManager(session=session, token_timeout=ttl)
run_manager = RunManager(session=session, fs=fs)
group_manager = WorkflowGroupManager(session=session,
fs=fs,
users=user_manager)
ranking_manager = RankingManager(session=session)
workflow_repo = WorkflowManager(session=session, fs=fs)
return API(
service=ServiceDescriptor.from_config(env=env, username=username),
workflow_service=LocalWorkflowService(
workflow_repo=workflow_repo,
ranking_manager=ranking_manager,
group_manager=group_manager,
run_manager=run_manager,
user_id=user_id),
group_service=LocalWorkflowGroupService(
group_manager=group_manager,
workflow_repo=workflow_repo,
backend=engine,
run_manager=run_manager,
auth=auth,
user_id=user_id),
upload_service=LocalUploadFileService(group_manager=group_manager,
auth=auth,
user_id=user_id),
run_service=LocalRunService(run_manager=run_manager,
group_manager=group_manager,
ranking_manager=ranking_manager,
backend=engine,
auth=auth,
user_id=user_id),
user_service=LocalUserService(manager=user_manager, auth=auth))
def test_delete_workflow(database, tmpdir):
"""Test deleting a workflows from the repository."""
# -- Setup ----------------------------------------------------------------
#
# Create two workflows.
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
wf = manager.create_workflow(name='A', source=BENCHMARK_DIR)
workflow_1 = wf.workflow_id
wf = manager.create_workflow(name='B', source=BENCHMARK_DIR)
workflow_2 = wf.workflow_id
# -- Test delete first workflow -------------------------------------------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
manager.delete_workflow(workflow_1)
with database.session() as session:
# The second workflow still exists.
manager = WorkflowManager(session=session, fs=fs)
manager.get_workflow(workflow_2) is not None
# -- Deleting the same repository multiple times raises an error ----------
with database.session() as session:
manager = WorkflowManager(session=session, fs=fs)
with pytest.raises(err.UnknownWorkflowError):
manager.delete_workflow(workflow_id=workflow_1)
