def test_upload_file(fscls, database, tmpdir):
"""Test uploading files."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with two groups for a single workflow. Upload one file
# for each group.
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_1 = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_1 = model.create_group(session, workflow_id, users=[user_1])
# -- Test upload file -----------------------------------------------------
data = {'A': 1}
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
fh = manager.upload_file(group_id=group_1,
file=io_file(data={'A': 1}),
name='A.json')
assert fh.name == 'A.json'
assert fh.mime_type == 'application/json'
fh = manager.get_uploaded_file(group_id=group_1, file_id=fh.file_id)
assert json.load(fh.open()) == data
# -- Test error case ------------------------------------------------------
data = {'A': 1}
with database.session() as session:
with pytest.raises(err.ConstraintViolationError):
manager.upload_file(group_id=group_1,
file=io_file(data={'A': 1}),
name=' ')
with pytest.raises(err.UnknownWorkflowGroupError):
manager.upload_file(group_id='UNKNOWN',
file=io_file(data={'A': 1}),
name=' ')
def test_run_helloworld_sync(sync_service, specfile, state):
"""Execute the helloworld example."""
# -- Setup ----------------------------------------------------------------
#
# Start a new run for the workflow template.
with sync_service() as api:
workflow_id = create_workflow(api,
source=TEMPLATE_DIR,
specfile=specfile)
user_id = create_user(api)
with sync_service(user_id=user_id) as api:
group_id = create_group(api, workflow_id)
names = io_file(data=['Alice', 'Bob'], format='plain/text')
file_id = upload_file(api, group_id, names)
args = [
serialize_arg('names', serialize_fh(file_id, 'data/names.txt')),
serialize_arg('sleeptime', 3)
]
run_id = start_run(api, group_id, arguments=args)
# -- Validate the run handle against the expected state -------------------
with sync_service(user_id=user_id) as api:
r = api.runs().get_run(run_id)
serialize.validate_run_handle(r, state=state)
if state == st.STATE_SUCCESS:
# The run should have the greetings.txt file as a result.
files = dict()
for obj in r['files']:
files[obj['name']] = obj['id']
assert len(files) == 1
fh = api.runs().get_result_file(
run_id=run_id, file_id=files['results/greetings.txt'])
value = fh.open().read().decode('utf-8').strip()
assert 'Hello Alice!' in value
assert 'Hello Bob!' in value
def start_hello_world(api, group_id):
"""Start a new run for the Hello World template. Returns the run identifier
and the identifier for the input file.
Parameters
----------
api: flowserv.service.api.API
Service API manager.
group_id: string
Unique group identifier.
Returns
-------
string, string
"""
file_id = api.uploads().upload_file(group_id=group_id,
file=io_file(data=['Alice', 'Bob'],
format='txt/plain'),
name='n.txt')['id']
run_id = api.runs().start_run(group_id=group_id,
arguments=[{
'name':
'names',
'value':
serialize_fh(file_id=file_id)
}])['id']
api.runs().backend.start(run_id)
return run_id, file_id
def test_cancel_run_helloworld(async_service):
"""Test cancelling a helloworld run."""
# -- Setup ----------------------------------------------------------------
#
# Start a new run for the workflow template.
with async_service() as api:
workflow_id = create_workflow(api, source=TEMPLATE_DIR)
user_id = create_user(api)
with async_service(user_id=user_id) as api:
group_id = create_group(api, workflow_id)
names = io_file(data=['Alice', 'Bob', 'Zoe'], format='plain/text')
file_id = upload_file(api, group_id, names)
args = [
serialize_arg('names', serialize_fh(file_id)),
serialize_arg('sleeptime', 10),
serialize_arg('greeting', 'Hi')
]
run_id = start_run(api, group_id, arguments=args)
# Poll run after sleeping for one second.
time.sleep(1)
with async_service(user_id=user_id) as api:
run = api.runs().get_run(run_id=run_id)
assert run['state'] in st.ACTIVE_STATES
# -- Cancel the active run ------------------------------------------------
with async_service(user_id=user_id) as api:
run = api.runs().cancel_run(
run_id=run_id,
reason='done'
)
assert run['state'] == st.STATE_CANCELED
assert run['messages'][0] == 'done'
with async_service(user_id=user_id) as api:
run = api.runs().get_run(run_id=run_id)
assert run['state'] == st.STATE_CANCELED
assert run['messages'][0] == 'done'
def test_delete_file(fscls, database, tmpdir):
"""Test deleting an uploaded file."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with two groups for a single workflow. Upload one file
# for each group.
file = io_file(data={'A': 1})
fn = 'data.json'
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_1 = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_1 = model.create_group(session, workflow_id, users=[user_1])
group_2 = model.create_group(session, workflow_id, users=[user_1])
manager = WorkflowGroupManager(session=session, fs=fs)
fh = manager.upload_file(group_id=group_1, file=file, name=fn)
file_1 = fh.file_id
fh = manager.upload_file(group_id=group_2, file=file, name=fn)
file_2 = fh.file_id
# -- Test delete file -----------------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
fh = manager.get_uploaded_file(group_id=group_1, file_id=file_1)
manager.delete_file(group_id=group_1, file_id=file_1)
# File 1 can no longer be accessed while file 2 is still present.
with pytest.raises(err.UnknownFileError):
manager.get_uploaded_file(group_id=group_1, file_id=file_1).open()
fh = manager.get_uploaded_file(group_id=group_2, file_id=file_2)
# -- Error cases ----------------------------------------------------------
with database.session() as session:
# - Delete unknown file
manager = WorkflowGroupManager(session=session, fs=fs)
with pytest.raises(err.UnknownFileError):
manager.delete_file(group_id=group_1, file_id=file_1)
def test_list_files(fscls, database, tmpdir):
"""Test listing uploaded files."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with two groups for a single workflow. The first group
# has one uploaded file and the second group has one file.
file = io_file(data={'A': 1})
fn = 'data.json'
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_1 = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_1 = model.create_group(session, workflow_id, users=[user_1])
group_2 = model.create_group(session, workflow_id, users=[user_1])
manager = WorkflowGroupManager(session=session, fs=fs)
manager.upload_file(group_id=group_1, file=file, name=fn)
manager.upload_file(group_id=group_1, file=file, name=fn)
manager.upload_file(group_id=group_2, file=file, name=fn)
# -- Test list files for groups -------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
files = manager.list_uploaded_files(group_id=group_1)
assert len(files) == 2
files = manager.list_uploaded_files(group_id=group_2)
assert len(files) == 1
def test_upload_group_file_local(local_service, hello_world):
"""Test uploading files for a workflow group."""
# -- Setup ----------------------------------------------------------------
#
# Create one group with minimal metadata for the 'Hello World' workflow.
with local_service() as api:
user_id = create_user(api)
workflow = hello_world(api, name='W1')
workflow_id = workflow.workflow_id
with local_service(user_id=user_id) as api:
group_id = create_group(api, workflow_id=workflow_id)
# -- Upload first file for the group --------------------------------------
with local_service(user_id=user_id) as api:
r = api.uploads().upload_file(group_id=group_id,
file=io_file(data={
'group': 1,
'file': 1
}),
name='group1.json')
file_id = r['id']
serialize.validate_file_handle(r)
assert r['name'] == 'group1.json'
# -- Get serialized handle for the file and the group ---------------------
for uid in [user_id, None]:
with local_service(user_id=uid) as api:
fcont = api.uploads().get_uploaded_file(group_id, file_id).read()
assert fcont == b'{"group": 1, "file": 1}'
gh = api.groups().get_group(group_id=group_id)
serialize.validate_group_handle(gh)
def test_list_group_files_local(local_service, hello_world):
"""Test getting a listing of uploaded files for a workflow group."""
# -- Setup ----------------------------------------------------------------
#
# Upload two files for a workflow group.
with local_service() as api:
user_id = create_user(api)
workflow = hello_world(api, name='W1')
workflow_id = workflow.workflow_id
with local_service(user_id=user_id) as api:
group_id = create_group(api, workflow_id=workflow_id)
for i in range(2):
upload_file(api=api,
group_id=group_id,
file=io_file(data={
'group': 1,
'file': i
}))
# -- Get file listing -----------------------------------------------------
with local_service(user_id=user_id) as api:
files = api.uploads().list_uploaded_files(group_id=group_id)
serialize.validate_file_listing(files, 2)
# -- Error when listing files as unknonw user -----------------------------
with local_service(user_id='UNKNOWN') as api:
with pytest.raises(err.UnauthorizedAccessError):
api.uploads().list_uploaded_files(group_id=group_id)
def test_delete_group_file_local(local_service, hello_world):
"""Test deleting an uploaded file for a workflow group."""
# -- Setup ----------------------------------------------------------------
#
# Upload one file for a workflow group.
with local_service() as api:
user_id = create_user(api)
workflow = hello_world(api, name='W1')
workflow_id = workflow.workflow_id
with local_service(user_id=user_id) as api:
group_id = create_group(api, workflow_id=workflow_id)
file_id = upload_file(api=api,
group_id=group_id,
file=io_file(data={
'group': 1,
'file': 1
}))
# -- Error when unknown user attempts to delete the file ------------------
with local_service(user_id='UNKNNOWN') as api:
with pytest.raises(err.UnauthorizedAccessError):
api.uploads().delete_file(group_id, file_id)
# -- Delete the uploaded file ---------------------------------------------
with local_service(user_id=user_id) as api:
api.uploads().delete_file(group_id, file_id)
# After deletion the file cannot be accessed anymore.
with local_service(user_id=user_id) as api:
with pytest.raises(err.UnknownFileError):
api.uploads().get_uploaded_file(group_id, file_id)
def write_results(rundir: str, files: Tuple[Union[dict, list], str, str]):
"""Create a result files for a workflow run.
Parameters
----------
rundir: string
Path to the temporary run directory.
run_id: string
Unique run identifier.
files: list
List of 3-tuples containing the file data, format, and relative path.
"""
for data, format, rel_path in files:
filename = os.path.join(rundir, rel_path)
os.makedirs(os.path.dirname(filename), exist_ok=True)
io_file(data=data, format=format).store(filename)
def test_get_file(fscls, database, tmpdir):
"""Test accessing uploaded files."""
# -- Setup ----------------------------------------------------------------
#
# Create a database with two groups for a single workflow. Upload one file
# for each group.
data_1 = {'A': 1}
data_2 = {'B': 2}
f1 = io_file(data=data_1)
f2 = io_file(data=data_2)
fn = 'data.json'
fs = fscls(env=Config().basedir(tmpdir))
with database.session() as session:
user_1 = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_1 = model.create_group(session, workflow_id, users=[user_1])
group_2 = model.create_group(session, workflow_id, users=[user_1])
mngr = WorkflowGroupManager(session=session, fs=fs)
file_1 = mngr.upload_file(group_id=group_1, file=f1, name=fn).file_id
file_2 = mngr.upload_file(group_id=group_2, file=f2, name=fn).file_id
files = [(group_1, file_1, data_1), (group_2, file_2, data_2)]
# -- Test get file --------------------------------------------------------
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
for g_id, f_id, data in files:
fh = manager.get_uploaded_file(group_id=g_id, file_id=f_id)
assert fh.name == fn
assert fh.mime_type == 'application/json'
assert json.load(fh.open()) == data
# -- Test error cases -----------------------------------------------------
# - File handle is unknown for s2
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=fs)
with pytest.raises(err.UnknownFileError):
manager.get_uploaded_file(group_id=group_2, file_id=file_1).open()
# - Access file with unknown file identifier
with pytest.raises(err.UnknownFileError):
manager.get_uploaded_file(group_id=group_1, file_id='UNK').open()
def test_group_handle_serialization(database, tmpdir):
"""Test serialization of workflow group handles."""
config = Config().basedir(tmpdir)
view = WorkflowGroupSerializer()
with database.session() as session:
manager = WorkflowGroupManager(session=session, fs=FileSystemStore(config))
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
manager.upload_file(group_id=group_id, file=io_file(data={'A': 1}), name='a.json')
group = manager.get_group(group_id)
doc = view.group_handle(group)
validator('UserGroupHandle').validate(doc)
assert len(doc[labels.GROUP_MEMBERS]) == 1
def run_erroneous_workflow(service, specfile):
"""Execute the modified helloworld example."""
with service() as api:
# Create workflow template, user, and the workflow group.
workflow_id = create_workflow(
api,
source=TEMPLATE_DIR,
specfile=specfile
)
user_id = create_user(api)
with service(user_id=user_id) as api:
group_id = create_group(api, workflow_id)
# Upload the names file.
names = io_file(data=NAMES, format='txt/plain')
file_id = upload_file(api, group_id, names)
# Run the workflow.
arguments = [
serialize_arg('names', serialize_fh(file_id)),
serialize_arg('greeting', 'Hi')
]
run_id = start_run(api, group_id, arguments=arguments)
# Poll workflow state every second.
run = poll_run(service, run_id, user_id)
assert run['state'] == st.STATE_SUCCESS
with service() as api:
wh = api.workflows().get_workflow(workflow_id=workflow_id)
attmpts = 0
while 'postproc' not in wh:
time.sleep(1)
with service() as api:
wh = api.workflows().get_workflow(workflow_id=workflow_id)
attmpts += 1
if attmpts > 60:
break
assert 'postproc' in wh
serialize.validate_workflow_handle(wh)
attmpts = 0
while wh['postproc']['state'] in st.ACTIVE_STATES:
time.sleep(1)
with service() as api:
wh = api.workflows().get_workflow(workflow_id=workflow_id)
attmpts += 1
if attmpts > 60:
break
assert wh['postproc']['state'] not in st.ACTIVE_STATES
serialize.validate_workflow_handle(wh)
assert wh['postproc']['state'] == st.STATE_ERROR
def test_run_helloworld_async(async_service, fsconfig, target):
"""Execute the helloworld example."""
# -- Setup ----------------------------------------------------------------
#
# Start a new run for the workflow template.
with async_service() as api:
workflow_id = create_workflow(api, source=TEMPLATE_DIR)
user_id = create_user(api)
with async_service(user_id=user_id) as api:
group_id = create_group(api, workflow_id)
names = io_file(data=['Alice', 'Bob', 'Zoe'], format='plain/text')
file_id = upload_file(api, group_id, names)
args = [
serialize_arg('names', serialize_fh(file_id, target)),
serialize_arg('sleeptime', 1),
serialize_arg('greeting', 'Hi')
]
run_id = start_run(api, group_id, arguments=args)
# Poll workflow state every second.
with async_service(user_id=user_id) as api:
run = api.runs().get_run(run_id=run_id)
watch_dog = 30
while run['state'] in st.ACTIVE_STATES and watch_dog:
time.sleep(1)
watch_dog -= 1
with async_service(user_id=user_id) as api:
run = api.runs().get_run(run_id=run_id)
assert run['state'] == st.STATE_SUCCESS
files = dict()
for f in run['files']:
files[f['name']] = f['id']
fh = api.runs().get_result_file(
run_id=run_id,
file_id=files['results/greetings.txt']
)
greetings = fh.open().read().decode('utf-8').strip()
assert 'Hi Alice' in greetings
assert 'Hi Bob' in greetings
assert 'Hi Zoe' in greetings
fh = api.runs().get_result_file(
run_id=run_id,
file_id=files['results/analytics.json']
)
assert json.load(fh.open()) is not None
def test_file_listing_serialization(database, tmpdir):
"""Test serialization of file handles."""
config = Config().basedir(tmpdir)
view = UploadFileSerializer()
filename = 'data.json'
with database.session() as session:
manager = WorkflowGroupManager(session=session,
fs=FileSystemStore(config))
user_id = model.create_user(session, active=True)
workflow_id = model.create_workflow(session)
group_id = model.create_group(session, workflow_id, users=[user_id])
fh = manager.upload_file(group_id=group_id,
file=io_file(data={'A': 1}),
name=filename)
doc = view.file_handle(group_id=group_id, fh=fh)
assert doc[labels.FILE_NAME] == filename
validator('FileHandle').validate(doc)
doc = view.file_listing(
group_id=group_id,
files=manager.list_uploaded_files(group_id=group_id))
validator('FileListing').validate(doc)
def test_postproc_workflow(fsconfig, tmpdir):
"""Execute the modified helloworld example."""
# -- Setup ----------------------------------------------------------------
#
# It is important here that we do not use the SQLite in-memory database
# since this fails (for unknown reason; presumably due to different threads)
# when the post-processing run is updated.
# --
env = Config().basedir(tmpdir).run_async().auth()
env.update(fsconfig)
service = LocalAPIFactory(env=env)
# Start a new run for the workflow template.
with service() as api:
# Need to set the file store in the backend to the new instance as
# well. Otherwise, the post processing workflow may attempt to use
# the backend which was initialized prior with a different file store.
workflow_id = create_workflow(
api,
source=TEMPLATE_DIR,
specfile=SPEC_FILE
)
user_id = create_user(api)
# Create four groups and run the workflow with a slightly different input
# file
for i in range(4):
with service(user_id=user_id) as api:
group_id = create_group(api, workflow_id)
names = io_file(data=NAMES[:(i + 1)], format='plain/text')
file_id = upload_file(api, group_id, names)
# Set the template argument values
arguments = [
serialize_arg('names', serialize_fh(file_id)),
serialize_arg('greeting', 'Hi')
]
run_id = start_run(api, group_id, arguments=arguments)
# Poll workflow state every second.
run = poll_run(service, run_id, user_id)
assert run['state'] == st.STATE_SUCCESS
with service() as api:
wh = api.workflows().get_workflow(workflow_id=workflow_id)
attmpts = 0
while 'postproc' not in wh:
time.sleep(1)
with service() as api:
wh = api.workflows().get_workflow(workflow_id=workflow_id)
attmpts += 1
if attmpts > 60:
break
assert 'postproc' in wh
serialize.validate_workflow_handle(wh)
attmpts = 0
while wh['postproc']['state'] in st.ACTIVE_STATES:
time.sleep(1)
with service() as api:
wh = api.workflows().get_workflow(workflow_id=workflow_id)
attmpts += 1
if attmpts > 60:
break
serialize.validate_workflow_handle(wh)
with service() as api:
ranking = api.workflows().get_ranking(workflow_id=workflow_id)
serialize.validate_ranking(ranking)
for fobj in wh['postproc']['files']:
if fobj['name'] == 'results/compare.json':
file_id = fobj['id']
with service(user_id=user_id) as api:
fh = api.workflows().get_result_file(
workflow_id=workflow_id,
file_id=file_id
)
compare = util.read_object(fh.open())
assert len(compare) == (i + 1)
# Access the post-processing result files.
with service() as api:
fh = api.workflows().get_result_archive(workflow_id=workflow_id)
assert fh.name.startswith('run')
assert fh.mime_type == 'application/gzip'
