def success_run(database: DB, fs: StorageVolume,
basedir: str) -> Tuple[str, str, str, str]:
"""Create a successful run with two result files:
- A.json
- results/B.json
Returns the identifier of the created workflow, group, run, and user.
"""
# Setup temporary run folder.
runfs = FileSystemStorage(basedir=os.path.join(basedir, 'tmprun'))
runfs.store(file=io_file({'A': 1}), dst='A.json')
runfs.store(file=io_file({'B': 1}), dst=util.join('results', 'B.json'))
with database.session() as session:
user_id = create_user(session, active=True)
workflow_id = create_workflow(session)
group_id = create_group(session, workflow_id, users=[user_id])
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(
run_id=run_id,
state=state.start().success(files=['A.json', 'results/B.json']),
runstore=runfs)
return workflow_id, group_id, run_id, user_id
def test_list_files(fscls, database, tmpdir):
"""Test listing uploaded files."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with two groups for a single workflow. The first group
# has one uploaded file and the second group has one file.
file = io_file(data={'A': 1})
fn = 'data.json'
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_1 = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_1 = model.create_group(session, workflow_id, users=[user_1])
group_2 = model.create_group(session, workflow_id, users=[user_1])
manager = WorkflowGroupManager(session=session, fs=fs)
manager.upload_file(group_id=group_1, file=file, name=fn)
manager.upload_file(group_id=group_1, file=file, name=fn)
manager.upload_file(group_id=group_2, file=file, name=fn)
# -- Test list files for groups -------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
files = manager.list_uploaded_files(group_id=group_1)
assert len(files) == 2
files = manager.list_uploaded_files(group_id=group_2)
assert len(files) == 1
def test_invalid_state_transitions(fscls, database, tmpdir):
"""Test error cases for invalid state transitions."""
# -- Setup ----------------------------------------------------------------
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# -- Test set active run to pending ---------------------------------------
with database.session() as session:
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(run_id=run_id, state=state.start())
with pytest.raises(err.ConstraintViolationError):
runs.update_run(run_id=run_id, state=st.StatePending())
# Cancel run
with database.session() as session:
runs = RunManager(session=session, fs=fs)
runs.update_run(run_id=run_id, state=state.cancel())
# -- Test cannot set run to any of the inactive states --------------------
with database.session() as session:
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
assert runs.update_run(run_id=run_id, state=state.cancel()) is None
with pytest.raises(err.ConstraintViolationError):
runs.update_run(run_id=run_id, state=state.error())
with pytest.raises(err.ConstraintViolationError):
runs.update_run(run_id=run_id, state=state.success())
def test_cancel_run(fscls, database, tmpdir):
"""Test setting run state to canceled."""
# -- Setup ----------------------------------------------------------------
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# -- Test set run to error state ------------------------------------------
with database.session() as session:
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(run_id=run_id, state=state.cancel())
with database.session() as session:
runs = RunManager(session=session, fs=fs)
run = runs.get_run(run_id)
state = run.state()
assert not state.is_active()
assert not state.is_pending()
assert not state.is_running()
assert state.is_canceled()
assert not state.is_error()
assert not state.is_success()
assert len(state.messages) == 1
def success_run(database: DB, fs: FileStore,
basedir: str) -> Tuple[str, str, str, str]:
"""Create a successful run with two result files:
- A.json
- run/results/B.json
Returns the identifier of the created workflow, group, run, and user.
"""
# Setup temporary run folder.
tmprundir = os.path.join(basedir, 'tmprun')
tmpresultsdir = os.path.join(tmprundir, 'run', 'results')
os.makedirs(tmprundir)
os.makedirs(tmpresultsdir)
f1 = os.path.join(tmprundir, 'A.json')
util.write_object(f1, {'A': 1})
f2 = os.path.join(tmpresultsdir, 'B.json')
util.write_object(f2, {'B': 1})
with database.session() as session:
user_id = create_user(session, active=True)
workflow_id = create_workflow(session)
group_id = create_group(session, workflow_id, users=[user_id])
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(
run_id,
state.start().success(files=['A.json', 'run/results/B.json']),
rundir=tmprundir)
assert not os.path.exists(tmprundir)
return workflow_id, group_id, run_id, user_id
def test_list_runs(fscls, database, tmpdir):
"""Test retrieving a list of run descriptors."""
# -- Setup ----------------------------------------------------------------
#
# Create two runs: one in running state and one in error state.
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
group = groups.get_group(group_id)
# Run 1 in running state
r = runs.create_run(group=group)
run_1 = r.run_id
runs.update_run(run_id=run_1, state=r.state().start())
r = runs.create_run(group=group)
run_2 = r.run_id
runs.update_run(run_id=run_2, state=r.state().error())
# -- Test get listing -----------------------------------------------------
with database.session() as session:
runs = RunManager(session=session, fs=fs)
run_index = dict()
for run in runs.list_runs(group_id):
run_index[run.run_id] = run
assert len(run_index) == 2
assert run_index[run_1].state().is_running()
assert run_index[run_2].state().is_error()
# -- Test polling runs ----------------------------------------------------
with database.session() as session:
runs = RunManager(session=session, fs=fs)
assert len(runs.list_runs(group_id)) == 2
assert len(runs.list_runs(group_id, state=st.STATE_ERROR)) == 1
assert len(runs.list_runs(group_id, state=st.STATE_SUCCESS)) == 0
def test_run_parameters(database, tmpdir):
"""Test creating run with template arguments."""
# -- Setup ----------------------------------------------------------------
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# Prepare run arguments
filename = os.path.join(str(tmpdir), 'results.json')
util.write_object(filename=filename, obj={'A': 1})
arguments = [{'id': 'A', 'value': 10}, {'id': 'B', 'value': True}]
# -- Test create run with arguments ---------------------------------------
with database.session() as session:
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(
group=groups.get_group(group_id),
arguments=arguments
)
run_id = run.run_id
with database.session() as session:
runs = RunManager(session=session, fs=fs)
run = runs.get_run(run_id)
assert run.arguments == arguments
def test_group_handle_serialization(database, tmpdir):
"""Test serialization of workflow group handles."""
config = Config().basedir(tmpdir)
view = WorkflowGroupSerializer()
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=FileSystemStore(config))
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
manager.upload_file(group_id=group_id, file=io_file(data={'A': 1}), name='a.json')
group = manager.get_group(group_id)
doc = view.group_handle(group)
validator('UserGroupHandle').validate(doc)
assert len(doc[labels.GROUP_MEMBERS]) == 1
def test_upload_file(database, tmpdir):
"""Test uploading files."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with two groups for a single workflow. Upload one file
# for each group.
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
user_1 = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_1 = model.create_group(session, workflow_id, users=[user_1])
# -- Test upload file -----------------------------------------------------
data = {'A': 1}
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
fh = manager.upload_file(group_id=group_1,
file=io_file(data={'A': 1}),
name='A.json')
assert fh.name == 'A.json'
assert fh.mime_type == 'application/json'
fh = manager.get_uploaded_file(group_id=group_1, file_id=fh.file_id)
assert json.load(fh.open()) == data
# -- Test error case ------------------------------------------------------
data = {'A': 1}
with database.session() as session:
with pytest.raises(err.ConstraintViolationError):
manager.upload_file(group_id=group_1,
file=io_file(data={'A': 1}),
name=' ')
with pytest.raises(err.UnknownWorkflowGroupError):
manager.upload_file(group_id='UNKNOWN',
file=io_file(data={'A': 1}),
name=' ')
def test_group_listing_serialization(database, tmpdir):
"""Test serialization of workflow group listing."""
config = Config().basedir(tmpdir)
view = WorkflowGroupSerializer()
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=FileSystemStore(config))
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
model.create_group(session, workflow_id, users=[user_id])
model.create_group(session, workflow_id, users=[user_id])
groups = manager.list_groups(workflow_id=workflow_id, user_id=user_id)
assert len(groups) == 2
doc = view.group_listing(groups)
validator('UserGroupListing').validate(doc)
assert len(doc[labels.GROUP_LIST]) == 2
def error_run(database, fs, messages):
"""Create a run that is in error state. Returns the identifier of the
created workflow, group, and run.
"""
# Setup temporary run folder.
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(run_id=run_id, state=state)
messages = ['There', 'were', 'many errors']
runs.update_run(run_id=run_id, state=state.error(messages))
return workflow_id, group_id, run_id
def test_run_serialization(database, tmpdir):
"""Test serialization of run handles and run listings."""
config = Config().basedir(tmpdir)
view = RunSerializer()
fs = FileSystemStore(config)
# Setup temporary run folder.
tmprundir = os.path.join(tmpdir, 'tmprun')
tmpresultsdir = os.path.join(tmprundir, 'run', 'results')
os.makedirs(tmprundir)
os.makedirs(tmpresultsdir)
f1 = os.path.join(tmprundir, 'A.json')
util.write_object(f1, {'A': 1})
# Create runs.
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# Create successful run.
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(
run_id,
state.start().success(files=['A.json', 'run/results/B.json']),
rundir=tmprundir)
run = runs.get_run(run_id)
doc = view.run_handle(run)
validator('RunHandle').validate(doc)
# Create error run.
run = runs.create_run(group=groups.get_group(group_id))
run_id = run.run_id
state = run.state()
runs.update_run(run_id=run_id, state=state)
messages = ['There', 'were', 'many errors']
runs.update_run(run_id=run_id, state=state.error(messages))
run = runs.get_run(run_id)
doc = view.run_handle(run)
validator('RunHandle').validate(doc)
# Validate run listing.
doc = view.run_listing(runs=runs.list_runs(group_id))
validator('RunListing').validate(doc)
assert len(doc[labels.RUN_LIST]) == 2
def test_file_listing_serialization(database, tmpdir):
"""Test serialization of file handles."""
view = UploadFileSerializer()
filename = 'data.json'
with database.session() as session:
manager = WorkflowGroupManager(session=session,
fs=FileSystemStorage(basedir=tmpdir))
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
fh = manager.upload_file(group_id=group_id,
file=io_file(data={'A': 1}),
name=filename)
doc = view.file_handle(group_id=group_id, fh=fh)
assert doc[labels.FILE_NAME] == filename
validator('FileHandle').validate(doc)
doc = view.file_listing(
group_id=group_id,
files=manager.list_uploaded_files(group_id=group_id))
validator('FileListing').validate(doc)
def test_create_run_errors(fscls, database, tmpdir):
"""Test error cases for create_run parameter combinations."""
# -- Setup ----------------------------------------------------------------
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
# -- Test create_run with invalid arguments -------------------------------
with database.session() as session:
wfrepo = WorkflowManager(session=session, fs=fs)
groups = WorkflowGroupManager(session=session, fs=fs)
runs = RunManager(session=session, fs=fs)
workflow = wfrepo.get_workflow(workflow_id)
group = groups.get_group(group_id)
with pytest.raises(ValueError):
runs.create_run()
with pytest.raises(ValueError):
runs.create_run(workflow=workflow, group=group)
with pytest.raises(ValueError):
runs.create_run(group=group, runs=['A'])
def test_list_groups(database, tmpdir):
"""Test listing groups by user or by workflow."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with three groups for a two workflow. Group 1 has
# user 1 as only member, group 2 has user 2 and 3 as member, group 3 has
# user 1 and 3 as members.
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
user_1 = model.create_user(session, active=True)
user_2 = model.create_user(session, active=True)
user_3 = model.create_user(session, active=True)
workflow_1 = model.create_workflow(session)
workflow_2 = model.create_workflow(session)
members_1 = [user_1]
group_1 = model.create_group(session, workflow_1, users=members_1)
members_2 = [user_2, user_3]
group_2 = model.create_group(session, workflow_1, users=members_2)
members_3 = [user_1, user_3]
group_3 = model.create_group(session, workflow_2, users=members_3)
# -- Test list all groups -------------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
assert len(manager.list_groups()) == 3
assert len(manager.list_groups(workflow_1)) == 2
assert len(manager.list_groups(workflow_2)) == 1
# -- Test list groups for users -------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
# User 1 is member of group 1 and 3.
groups = manager.list_groups(user_id=user_1)
assert len(groups) == 2
assert [g.name for g in groups] == [group_1, group_3]
# User 2 is member of group 2.
groups = manager.list_groups(user_id=user_2)
assert len(groups) == 1
assert [g.name for g in groups] == [group_2]
# User 3 is member of group 2 and 3.
groups = manager.list_groups(user_id=user_3)
assert len(groups) == 2
assert [g.name for g in groups] == [group_2, group_3]
def test_delete_file(database, tmpdir):
"""Test deleting an uploaded file."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with two groups for a single workflow. Upload one file
# for each group.
fn = 'data.json'
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
user_1 = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_1 = model.create_group(session, workflow_id, users=[user_1])
group_2 = model.create_group(session, workflow_id, users=[user_1])
manager = WorkflowGroupManager(session=session, fs=fs)
fh = manager.upload_file(group_id=group_1,
file=io_file(data={'A': 1}),
name=fn)
file_1 = fh.file_id
fh = manager.upload_file(group_id=group_2,
file=io_file(data={'A': 1}),
name=fn)
file_2 = fh.file_id
# -- Test delete file -----------------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
fh = manager.get_uploaded_file(group_id=group_1, file_id=file_1)
manager.delete_file(group_id=group_1, file_id=file_1)
# File 1 can no longer be accessed while file 2 is still present.
with pytest.raises(err.UnknownFileError):
manager.get_uploaded_file(group_id=group_1, file_id=file_1).open()
fh = manager.get_uploaded_file(group_id=group_2, file_id=file_2)
# -- Error cases ----------------------------------------------------------
with database.session() as session:
# - Delete unknown file
manager = WorkflowGroupManager(session=session, fs=fs)
with pytest.raises(err.UnknownFileError):
manager.delete_file(group_id=group_1, file_id=file_1)
def __enter__(self) -> API:
"""Create a new instance of the local API when the context manager is
entered.
"""
# Open a new database session.
self._session = self._db.session()
session = self._session.open()
# Shortcuts for local variables.
env = self._env
fs = self._fs
engine = self._engine
# Start by creating the authorization component and setting the
# identifier for and authenticated user.
user_id = self._user_id
username = None
if env[AUTH] == config.AUTH_OPEN:
auth = OpenAccessAuth(session)
user_id = config.DEFAULT_USER if user_id is None else user_id
else:
auth = DefaultAuthPolicy(session)
access_token = self._access_token if self._access_token is not None else env.get(
ACCESS_TOKEN)
if access_token and user_id is None:
# If an access token is given we retrieve the user that is
# associated with the token. Authentication may raise an error.
# Here, we ignore that error since the token may be an outdated
# token that is stored in the environment.
try:
user = auth.authenticate(access_token)
# Set the user name for the authenticated user (to be
# included in the service descriptor).
username = user.name
user_id = user.user_id
except err.UnauthenticatedAccessError:
pass
# Create the individual components of the API.
ttl = env.get(config.FLOWSERV_AUTH_LOGINTTL, config.DEFAULT_LOGINTTL)
user_manager = UserManager(session=session, token_timeout=ttl)
run_manager = RunManager(session=session, fs=fs)
group_manager = WorkflowGroupManager(session=session,
fs=fs,
users=user_manager)
ranking_manager = RankingManager(session=session)
workflow_repo = WorkflowManager(session=session, fs=fs)
return API(
service=ServiceDescriptor.from_config(env=env, username=username),
workflow_service=LocalWorkflowService(
workflow_repo=workflow_repo,
ranking_manager=ranking_manager,
group_manager=group_manager,
run_manager=run_manager,
user_id=user_id),
group_service=LocalWorkflowGroupService(
group_manager=group_manager,
workflow_repo=workflow_repo,
backend=engine,
run_manager=run_manager,
auth=auth,
user_id=user_id),
upload_service=LocalUploadFileService(group_manager=group_manager,
auth=auth,
user_id=user_id),
run_service=LocalRunService(run_manager=run_manager,
group_manager=group_manager,
ranking_manager=ranking_manager,
backend=engine,
auth=auth,
user_id=user_id),
user_service=LocalUserService(manager=user_manager, auth=auth))
def test_delete_group(database, tmpdir):
"""Test creating and deleting workflow groups."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with two groups for a single workflow.
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
user_id = model.create_user(session, active=True)
wf_id = model.create_workflow(session)
manager = WorkflowGroupManager(session=session, fs=fs)
group_1 = manager.create_group(workflow_id=wf_id,
name='A',
user_id=user_id,
parameters=ParameterIndex(),
workflow_spec=dict()).group_id
group_2 = manager.create_group(workflow_id=wf_id,
name='B',
user_id=user_id,
parameters=ParameterIndex(),
workflow_spec=dict()).group_id
# -- Delete group ---------------------------------------------------------
with database.session() as session:
# Ensure that group directores are deleted.
manager = WorkflowGroupManager(session=session, fs=fs)
manager.delete_group(group_1)
# Access to group 1 raises error while group 2 is still accessible.
with pytest.raises(err.UnknownWorkflowGroupError):
manager.get_group(group_1)
assert manager.get_group(group_2) is not None
def test_update_groups(database, tmpdir):
"""Test updating group name and group members."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with two groups for one workflow. Group 1 has user 1 as
# only member, group 2 has user 2 and 3 as member.
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
user_1 = model.create_user(session, active=True)
user_2 = model.create_user(session, active=True)
user_3 = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
members_1 = [user_1]
group_1 = model.create_group(session, workflow_id, users=members_1)
members_2 = [user_2, user_3]
group_2 = model.create_group(session, workflow_id, users=members_2)
# -- Test add member ------------------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
members_1 = [user_1, user_3]
manager.update_group(group_1, members=members_1)
members = [m.user_id for m in manager.get_group(group_1).members]
assert set(members) == set(members_1)
members = [m.user_id for m in manager.get_group(group_2).members]
assert set(members) == set(members_2)
# -- Test rename group ----------------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
manager.update_group(group_2, name='My Group')
assert manager.get_group(group_1).name == group_1
assert manager.get_group(group_2).name == 'My Group'
# -- Test rename group and change members ---------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
members_2 = [user_1] + members_2
manager.update_group(group_2, name='The Group', members=members_2)
members = [m.user_id for m in manager.get_group(group_1).members]
assert set(members) == set(members_1)
members = [m.user_id for m in manager.get_group(group_2).members]
assert set(members) == set(members_2)
assert manager.get_group(group_1).name == group_1
assert manager.get_group(group_2).name == 'The Group'
# -- Test no changes ------------------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
manager.update_group(group_2, name='The Group', members=members_2)
members = [m.user_id for m in manager.get_group(group_1).members]
assert set(members) == set(members_1)
members = [m.user_id for m in manager.get_group(group_2).members]
assert set(members) == set(members_2)
assert manager.get_group(group_1).name == group_1
assert manager.get_group(group_2).name == 'The Group'
def test_create_group(database, tmpdir):
"""Test creating and retrieving new workflow groups."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with a single workflow.
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
# -- Test create group ----------------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
group = manager.create_group(workflow_id=workflow_id,
name='Group 1',
user_id=user_id,
parameters=ParameterIndex(),
workflow_spec=dict())
assert group.name == 'Group 1'
assert group.owner_id == user_id
assert group.engine_config is None
assert len(group.members) == 1
assert isinstance(group.parameters, dict)
assert len(group.parameters) == 0
assert isinstance(group.workflow_spec, dict)
assert len(group.workflow_spec) == 0
# Retrieve the group from the database
group = manager.get_group(group.group_id)
assert group.name == 'Group 1'
assert group.owner_id == user_id
assert len(group.members) == 1
assert isinstance(group.parameters, dict)
assert len(group.parameters) == 0
assert isinstance(group.workflow_spec, dict)
assert len(group.workflow_spec) == 0
# -- Test create group with duplicate members and different config --------
engine_config = {'workers': {'test': {'worker': 'docker'}}}
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
group = manager.create_group(workflow_id=workflow_id,
name='Group 2',
user_id=user_id,
parameters=ParameterIndex(),
workflow_spec=dict(),
members=[user_id, user_id, user_id],
engine_config=engine_config)
assert len(group.members) == 1
assert group.engine_config == engine_config
# Retrieve the group from the database
group = manager.get_group(group.group_id)
assert len(group.members) == 1
# -- Test error cases -----------------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
# - Invalid name
with pytest.raises(err.ConstraintViolationError):
manager.create_group(workflow_id=workflow_id,
name='A' * 513,
user_id=user_id,
parameters=ParameterIndex(),
workflow_spec=dict())
# - Duplicate name
with pytest.raises(err.ConstraintViolationError):
manager.create_group(workflow_id=workflow_id,
name='Group 1',
user_id=user_id,
parameters=ParameterIndex(),
workflow_spec=dict())
# - Unknown user
with pytest.raises(err.UnknownUserError):
manager.create_group(workflow_id=workflow_id,
name='D',
user_id=user_id,
parameters=ParameterIndex(),
workflow_spec=dict(),
members=[user_id, 'not a user'])
# - Missing user
with pytest.raises(err.UnknownUserError):
manager.create_group(workflow_id=workflow_id,
name='D',
user_id=None,
parameters=ParameterIndex(),
workflow_spec=dict(),
members=[user_id, 'not a user'])
def test_get_file(database, tmpdir):
"""Test accessing uploaded files."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with two groups for a single workflow. Upload one file
# for each group.
data_1 = {'A': 1}
data_2 = {'B': 2}
f1 = io_file(data=data_1)
f2 = io_file(data=data_2)
fn = 'data.json'
fs = FileSystemStorage(basedir=tmpdir)
with database.session() as session:
user_1 = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_1 = model.create_group(session, workflow_id, users=[user_1])
group_2 = model.create_group(session, workflow_id, users=[user_1])
mngr = WorkflowGroupManager(session=session, fs=fs)
file_1 = mngr.upload_file(group_id=group_1, file=f1, name=fn).file_id
file_2 = mngr.upload_file(group_id=group_2, file=f2, name=fn).file_id
files = [(group_1, file_1, data_1), (group_2, file_2, data_2)]
# -- Test get file --------------------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
for g_id, f_id, data in files:
fh = manager.get_uploaded_file(group_id=g_id, file_id=f_id)
assert fh.name == fn
assert fh.mime_type == 'application/json'
assert json.load(fh.open()) == data
# -- Test error cases -----------------------------------------------------
# - File handle is unknown for s2
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
with pytest.raises(err.UnknownFileError):
manager.get_uploaded_file(group_id=group_2, file_id=file_1).open()
# - Access file with unknown file identifier
with pytest.raises(err.UnknownFileError):
manager.get_uploaded_file(group_id=group_1, file_id='UNK').open()