def create_workflow(self, run, template, arguments):
"""Create a new instance of a workflow from the given workflow
template and user-provided arguments.
Parameters
----------
run: flowserv.model.base.RunObject
Handle for the run that is being executed.
template: flowserv.model.template.base.WorkflowTemplate
Workflow template containing the parameterized specification and
the parameter declarations.
arguments: dict
Dictionary of argument values for parameters in the template.
Returns
-------
flowserv.model.workflow.remote.RemoteWorkflowObject
"""
# Create a serial workfow to have a workflow handle.
_, _, output_file = parser.parse_template(template=template,
arguments=arguments)
self.state = StatePending()
self._pollcount = 0
return RemoteWorkflowObject(workflow_id=run.run_id,
state=self.state,
output_files=output_file)
def get_workflow_state(workflow):
"""Get workflow state."""
try:
state = REANAClient().get_workflow_state(workflow, StatePending())
click.echo('in state {}'.format(state))
except Exception as ex:
click.echo('Error: {}'.format(ex))
def test_run_workflow_error(tmpdir):
"""Test error edge case for the run_workflow function."""
_, _, result = run_workflow(run_id='0',
rundir=tmpdir,
state=StatePending(),
output_files=[],
steps=None,
arguments=dict(),
workers=None)
assert result['type'] == STATE_ERROR
def test_run_workflow_error(tmpdir):
"""Test error edge case for the run_workflow function."""
# This will cause a NonType exception when trying to access the steps in
# the exec_workflow method.
_, _, result = run_workflow(run_id='0',
state=StatePending(),
output_files=[],
steps=None,
arguments=dict(),
volumes=DefaultVolume(basedir=tmpdir),
workers=None)
assert result['type'] == STATE_ERROR
def test_remote_run_success(tmpdir):
"""Test monitoring a successful workflow run."""
client = RemoteTestClient()
workflow = RemoteWorkflowHandle(
run_id='R0',
workflow_id='W0',
state=StatePending(),
output_files=list(),
runstore=FileSystemStorage(basedir=tmpdir),
client=client
)
state = monitor_workflow(workflow=workflow, poll_interval=0.1)
assert state.is_success()
def test_remote_run_error(tmpdir):
"""Test monitoring an erroneous workflow run."""
# Create client that will raise an error after the default rounds of polling.
client = RemoteTestClient(error='some error')
workflow = RemoteWorkflowHandle(
run_id='R0',
workflow_id='W0',
state=StatePending(),
output_files=list(),
runstore=FileSystemStorage(basedir=tmpdir),
client=client
)
state = monitor_workflow(workflow=workflow, poll_interval=0.1)
assert state.is_error()
def __init__(self, runcount=5, error=None):
"""Initialize the internal state that maintains the created workflow.
The client only supports execution for a single workflow at a time.
Parameters
----------
runcount: int, default=5
Number of poll counts before the workflow state changes. This is
used to simulate asynchronous workflow excution.
error: string
Error message. If given the resulting workflow run will be in
error state and this string will be the only error message.
data: list or dict, default=['no data']
Result file content for successful workflow runs. Writes this data
item to the result file when the poll counter reaches the run count
and the error message is None.
"""
self.runcount = runcount
self.error = error
self.state = None
# Count the number of times that the get_workflow_state() method has
# been called.
self.state = StatePending()
self._pollcount = 0
def run_workflow(spec):
"""Create a new workflow run for the given specification."""
doc = util.read_object(filename=spec)
rundir = os.path.dirname(spec)
if not rundir:
rundir = '.'
run = RunHandle(
identifier='0000',
workflow_id='0000',
group_id='0000',
state=StatePending(),
arguments=dict(),
rundir=rundir
)
template = WorkflowTemplate(workflow_spec=doc, sourcedir=rundir)
wf = REANAClient().create_workflow(run, template, dict())
click.echo('created workflow {} ({})'.format(wf.identifier, wf.state))
class RemoteTestClient(RemoteClient):
"""Implementation of the remote workflow engine client. Simulates the
execution of a workflow. The remote workflow initially is in pending state.
The first call to the get_workflow_state method will return a workflow in
running state without actually stating any workflow execution. The next N
calls to get_workflow_state will then simulate a runnign workflow. When
the method is then called next either successful run or an error run is
returned.
"""
def __init__(self, runcount=5, error=None, data=['no data']):
"""Initialize the internal state that maintains the created workflow.
The client only supports execution for a single workflow at a time.
Parameters
----------
runcount: int, default=5
Number of poll counts before the workflow state changes. This is
used to simulate asynchronous workflow excution.
error: string
Error message. If given the resulting workflow run will be in
error state and this string will be the only error message.
data: list or dict, default=['no data']
Result file content for successful workflow runs. Writes this data
item to the result file when the poll counter reaches the run count
and the error message is None.
"""
self.runcount = runcount
self.error = error
self.data = data
self.state = None
# Count the number of times that the get_workflow_state() method has
# been called.
self._pollcount = None
def create_workflow(self, run, template, arguments):
"""Create a new instance of a workflow from the given workflow
template and user-provided arguments.
Parameters
----------
run: flowserv.model.base.RunObject
Handle for the run that is being executed.
template: flowserv.model.template.base.WorkflowTemplate
Workflow template containing the parameterized specification and
the parameter declarations.
arguments: dict
Dictionary of argument values for parameters in the template.
Returns
-------
flowserv.model.workflow.remote.RemoteWorkflowObject
"""
# Create a serial workfow to have a workflow handle.
_, _, output_file = parser.parse_template(template=template,
arguments=arguments)
self.state = StatePending()
self._pollcount = 0
return RemoteWorkflowObject(workflow_id=run.run_id,
state=self.state,
output_files=output_file)
def download_file(self, workflow_id, source, target):
"""Download file from relative location in the base directory to a
given target path. Since the workflow is executed in the run directory
no files need to be copied.
Parameters
----------
workflow_id: string
Unique workflow identifier.
source: string
Relative path to source file in workflow workspace.
target: string
Path to target file on local disk.
"""
os.makedirs(os.path.dirname(target), exist_ok=True)
util.write_object(obj=self.data, filename=target)
def get_workflow_state(self, workflow_id, current_state):
"""Get information about the current state of a given workflow.
Note, if the returned result is SUCCESS the workflow resource files may
not have been initialized properly. This will be done by the workflow
controller. The timestamps, however, should be set accurately.
Parameters
----------
workflow_id: string
Unique workflow identifier
current_state: flowserv.model.workflw.state.WorkflowState
Last known state of the workflow by the workflow controller
Returns
-------
flowserv.model.workflw.state.WorkflowState
"""
if self.state is None:
raise ValueError('unknown workflow')
# Set the workflow to running state if this is the first call to the
# method.
if self._pollcount == 0:
self.state = self.state.start()
elif self._pollcount > self.runcount:
if self.error:
self.state = self.state.error(messages=[self.error])
else:
self.state = self.state.success()
self._pollcount += 1
return self.state
def stop_workflow(self, workflow_id):
"""Stop the execution of the workflow with the given identifier. Set
the get state counter to a negative value to avoid that the workflow is
executed.
Parameters
----------
workflow_id: string
Unique workflow identifier
"""
self.state = None
def test_pending_state():
"""Test creating instances of the pending state class."""
created_at = dt.datetime.now()
state = StatePending(created_at)
assert state.is_pending()
assert state.is_active()
assert not state.is_canceled()
assert not state.is_error()
assert not state.is_running()
assert not state.is_success()
assert state.created_at == created_at
running = state.start()
assert state.created_at == running.created_at
assert running.started_at is not None
# Cancel pending run
canceled = state.cancel()
assert canceled.is_canceled()
assert len(canceled.messages) == 1
canceled = state.cancel(messages=['by', 'user'])
assert canceled.is_canceled()
assert len(canceled.messages) == 2
# Set pending run into error state
state = StatePending(created_at)
error = state.error(messages=['there', 'was', 'a', 'error'])
assert error.is_error()
assert len(error.messages) == 4
class RemoteTestClient(RemoteClient):
"""Implementation of the remote workflow engine client. Simulates the
execution of a workflow. The remote workflow initially is in pending state.
The first call to the get_workflow_state method will return a workflow in
running state without actually stating any workflow execution. The next N
calls to get_workflow_state will then simulate a runnign workflow. When
the method is then called next either successful run or an error run is
returned.
"""
def __init__(self, runcount=5, error=None):
"""Initialize the internal state that maintains the created workflow.
The client only supports execution for a single workflow at a time.
Parameters
----------
runcount: int, default=5
Number of poll counts before the workflow state changes. This is
used to simulate asynchronous workflow excution.
error: string
Error message. If given the resulting workflow run will be in
error state and this string will be the only error message.
data: list or dict, default=['no data']
Result file content for successful workflow runs. Writes this data
item to the result file when the poll counter reaches the run count
and the error message is None.
"""
self.runcount = runcount
self.error = error
self.state = None
# Count the number of times that the get_workflow_state() method has
# been called.
self.state = StatePending()
self._pollcount = 0
def create_workflow(
self, run: RunObject, template: WorkflowTemplate, arguments: Dict,
staticfs: StorageVolume
) -> RemoteWorkflowHandle:
"""Create a new instance of a workflow from the given workflow
template and user-provided arguments.
Parameters
----------
run: flowserv.model.base.RunObject
Handle for the run that is being executed.
template: flowserv.model.template.base.WorkflowTemplate
Workflow template containing the parameterized specification and
the parameter declarations.
arguments: dict
Dictionary of argument values for parameters in the template.
staticfs: flowserv.volume.base.StorageVolume
Storage volume that contains the static files from the workflow
template.
Returns
-------
flowserv.controller.remote.client.RemoteWorkflowHandle
"""
# Create a serial workfow to have a workflow handle.
return RemoteWorkflowHandle(
run_id=run.run_id,
workflow_id=run.run_id,
state=self.state,
output_files=[],
runstore=staticfs.get_store_for_folder(util.join('runs', run.run_id)),
client=self
)
def get_workflow_state(
self, workflow_id: str, current_state: WorkflowState
) -> WorkflowState:
"""Get information about the current state of a given workflow.
Note, if the returned result is SUCCESS the workflow resource files may
not have been initialized properly. This will be done by the workflow
controller. The timestamps, however, should be set accurately.
Parameters
----------
workflow_id: string
Unique workflow identifier
current_state: flowserv.model.workflw.state.WorkflowState
Last known state of the workflow by the workflow controller
Returns
-------
flowserv.model.workflw.state.WorkflowState
"""
if self.state is None:
raise ValueError('unknown workflow')
# Set the workflow to running state if this is the first call to the
# method.
if self._pollcount == 0:
self.state = self.state.start()
elif self._pollcount > self.runcount:
if self.error:
self.state = self.state.error(messages=[self.error])
else:
self.state = self.state.success()
self._pollcount += 1
return self.state
def stop_workflow(self, workflow_id: str):
"""Stop the execution of the workflow with the given identifier.
Sets the state to None to raise an error the next time the workflow
state is polled.
Parameters
----------
workflow_id: string
Unique workflow identifier
"""
self.state = None