def test_run_helloworld_with_diff_inputs(tmpdir):
"""Run the hello world workflow in the test environment with different
types of input files.
"""
basedir = os.path.join(tmpdir, 'flowserv')
db = Flowserv(basedir=basedir, open_access=True, clear=True)
files = list()
files.append(BytesIO(b'Alice\nBob\nClaire'))
filename = os.path.join(tmpdir, 'names.txt')
with open(filename, 'w') as f:
f.write('Alice\nBob\nClaire')
files.append(filename)
files.append(FSFile(filename))
# -- Install and run the workflow -----------------------------------------
app_id = db.install(source=BENCHMARK_DIR, ignore_postproc=True)
wf = db.open(app_id)
for file in files:
run = wf.start_run({
'names': file,
'greeting': 'Hey',
'sleeptime': 0.0
})
assert run.is_success()
# -- Error for invalid file type ------------------------------------------
with pytest.raises(err.InvalidArgumentError):
wf.start_run({
'names': ['A', 'B'],
'greeting': 'Hey',
'sleeptime': 0.1
})
# -- Error for unknown parameter ------------------------------------------
with pytest.raises(err.UnknownParameterError):
wf.start_run({'names': filename, 'greeting': 'Hey', 'sleep': 0.1})
def test_run_helloworld_with_postproc(tmpdir):
"""Run the hello world workflow in the test environment including the
post-processing workflow.
"""
db = Flowserv(basedir=os.path.join(tmpdir, 'flowserv'), open_access=True)
# -- Install and run the workflow -----------------------------------------
app_id = db.install(source='helloworld', ignore_postproc=False)
wf = db.open(app_id)
run = wf.start_run({
'names': StringIO('Alice\nBob\nClaire'),
'greeting': 'Hey',
'sleeptime': 0.1
})
assert run.is_success()
run = wf.start_run({
'names': StringIO('Xenia\nYolanda\nZoe'),
'greeting': 'Hello',
'sleeptime': 0.1
})
assert run.is_success()
postproc = wf.get_postproc_results()
columns, rows = postproc.get_file('results/ngrams.csv').data()
assert columns == ['3-gram', 'Count']
assert rows is not None
columns.append('X')
# Access the data object again to receive the buffered data
columns, rows = postproc.get_file('results/ngrams.csv').data()
assert columns == ['3-gram', 'Count', 'X']
def test_install_app(tmpdir):
"""Test initializing the Flowserv client with the full set of arguments."""
basedir = os.path.join(tmpdir, 'test')
Flowserv(basedir=basedir,
database=TEST_URL,
open_access=True,
run_async=True,
clear=True)
def get_app(source: str, specfile: str, manifestfile: str,
name: str) -> Workflow:
"""Get the application handle for the given workflow template.
Creates a fresh database in the application cache directory and installes
the given workflow.
Parameters
----------
source: string
Path to local template, name or URL of the template in the
repository.
specfile: string
Path to the workflow template specification file (absolute or
relative to the workflow directory)
manifestfile: string
Path to manifest file. If not given an attempt is made to read one
of the default manifest file names in the base directory.
name: string
Name of the application that will determine the base folder for all
workflow files.
Returns
-------
flowserv.client.app.workflow.Workflow
"""
# Use the application cache directory for storing all workflow files.
homedir = os.path.join(user_cache_dir(appname='flowapp'), name)
util.cleardir(homedir)
# Create new database.
dburl = SQLITE_DB(dirname=homedir, filename='flowapp.db')
DB(connect_url=dburl).init()
# Install workflow template and return app handle.
env = config\
.env()\
.basedir(homedir)\
.database(dburl)\
.volume(FStore(basedir=homedir, identifier=DEFAULT_STORE))\
.open_access()\
.run_sync()
client = Flowserv(env=env)
workflow_id = client.install(source=source,
specfile=specfile,
manifestfile=manifestfile)
return client.open(identifier=workflow_id)
def test_run_helloworld_in_env(tmpdir):
"""Run the hello world workflow in the test environment."""
basedir = os.path.join(tmpdir, 'flowserv')
db = Flowserv(basedir=basedir, open_access=True, clear=True)
# -- Install and run the workflow -----------------------------------------
app_id = db.install(source=BENCHMARK_DIR,
specfile=BENCHMARK_FILE,
ignore_postproc=True)
wf = db.open(app_id)
run = wf.start_run({
'names': StringIO('Alice\nBob\nClaire'),
'greeting': 'Hey',
'sleeptime': 0.1
})
assert run.is_success()
assert not run.is_active()
assert str(run) == st.STATE_SUCCESS
assert len(run.files()) == 2
text = run.get_file('greetings').text()
assert 'Hey Alice' in text
assert 'Hey Bob' in text
assert 'Hey Claire' in text
# There should no by any post-processing results
assert wf.get_postproc_results() is None
# -- Polling the run should return a valid result -------------------------
run = wf.poll_run(run.run_id)
assert run.is_success()
assert not run.is_active()
assert str(run) == st.STATE_SUCCESS
assert len(run.files()) == 2
file_handles = dict()
for f in run.files():
file_handles[f.name] = f
assert file_handles['greetings'].name == 'greetings'
assert file_handles['greetings'].format == {'type': 'plaintext'}
assert file_handles[
'results/analytics.json'].name == 'results/analytics.json'
assert file_handles['results/analytics.json'].caption is None
assert file_handles['results/analytics.json'].format == {'type': 'json'}
# -- Cancelling a finished run raises an error ----------------------------
with pytest.raises(err.InvalidRunStateError):
wf.cancel_run(run.run_id)
# -- Uninstall workflow ---------------------------------------------------
db.uninstall(wf.identifier)
# Running the workflow again will raise an error.
with pytest.raises(err.UnknownWorkflowGroupError):
run = wf.start_run({'names': StringIO('Alice')})
# Erase the workflow folser.
db.erase()
assert not os.path.exists(os.path.join(tmpdir, 'flowserv'))
def test_run_helloworld_with_postproc(tmpdir):
"""Run the hello world workflow in the test environment including the
post-processing workflow.
"""
db = Flowserv(basedir=os.path.join(tmpdir, 'flowserv'), open_access=True)
# -- Install and run the workflow -----------------------------------------
app_id = db.install(source=BENCHMARK_DIR,
specfile=POSTPROC_SPEC,
ignore_postproc=False)
wf = db.open(app_id)
run = wf.start_run({
'names': StringIO('Alice\nBob\nClaire'),
'greeting': 'Hey'
})
assert run.is_success()
run = wf.start_run({
'names': StringIO('Xenia\nYolanda\nZoe'),
'greeting': 'Hello'
})
assert run.is_success()
postproc = wf.get_postproc_results()
doc = postproc.get_file('results/compare.json').json()
assert doc == [{
'avg_count': 12.0,
'total_count': 36,
'max_len': 14,
'max_line': 'Hello Yolanda!'
}]
# Access the data object again to receive the buffered data
doc = postproc.get_file('results/compare.json').json()
assert doc == [{
'avg_count': 12.0,
'total_count': 36,
'max_len': 14,
'max_line': 'Hello Yolanda!'
}]
def install_application(ctx, key, name, description, instructions, specfile,
manifest, template, ignore_postproc, verbose):
"""Install workflow from local folder or repository."""
# Install the application from the given workflow template.
# Create a new workflow for the application from the specified template.
app_key = Flowserv(open_access=True).install(
source=template,
identifier=key,
name=name,
description=description,
instructions=instructions,
specfile=specfile,
manifestfile=manifest,
ignore_postproc=ignore_postproc,
verbose=verbose)
click.echo('export {}={}'.format(ctx.obj.vars['workflow'], app_key))
def test_install_app(tmpdir):
"""Install and uninstall a workflow application via the app client."""
basedir = os.path.join(tmpdir, 'test')
env = {config.FLOWSERV_BASEDIR: basedir}
client = Flowserv(env=env)
# Install workflow as application.
app_id = client.install(source=TEMPLATE_DIR)
# Get the application object.
app = client.open(app_id)
assert app.name() == 'Hello World'
assert app.workflow_id == app.group_id
# Use the open_app function to get a fresh instance of the workflow.
app = open_app(env=app.service, identifier=app_id)
assert app.name() == 'Hello World'
assert app.workflow_id == app.group_id
# Uninstall the app.
client.uninstall(app_id)
# Erase the nase directory.
client.erase()
assert not os.path.isdir(basedir)
def uninstall_application(force, appkey):
"""Uninstall workflow with the given key."""
if not force: # pragma: no cover
click.echo('This will erase all workflow files and run results.')
click.confirm('Continue?', default=True, abort=True)
Flowserv(open_access=True).uninstall(identifier=appkey)
def test_install_app(tmpdir):
"""Install and uninstall a workflow application via the app client."""
basedir = os.path.join(tmpdir, 'test')
env = Config()\
.basedir(tmpdir)\
.volume(FStore(basedir=str(tmpdir)))
client = Flowserv(env=env)
# Install workflow as application.
app_id = client.install(source=BENCHMARK_DIR)
# Get the application object.
app = client.open(app_id)
assert app.name() == 'Hello World'
assert app.workflow_id == app.group_id
# Use the open function to get a fresh instance of the workflow.
app = Flowserv(env=app.service).open(identifier=app_id)
assert app.name() == 'Hello World'
assert app.workflow_id == app.group_id
# Uninstall the app.
client.uninstall(app_id)
# Erase the nase directory.
client.erase()
assert not os.path.isdir(basedir)