def run(self):
"""Poll the remote server continuously until execution is finished."""
try:
monitor_workflow(workflow=self.workflow,
poll_interval=self.poll_interval,
service=self.service)
except Exception as ex:
logging.error(ex, exc_info=True)
strace = util.stacktrace(ex)
logging.debug('\n'.join(strace))
state = self.workflow.state.error(messages=strace)
if self.service is not None:
with self.service() as api:
try:
api.runs().update_run(run_id=self.workflow.run_id,
state=state,
runstore=self.workflow.runstore)
except err.ConstraintViolationError:
pass
# Remove the workflow information form the task list.
try:
del self.tasks[self.run_id]
except Exception as ex:
logging.error(ex, exc_info=True)
logging.debug('\n'.join(util.stacktrace(ex)))
def cancel_run(self, run_id: str):
"""Request to cancel execution of the given run. This method is usually
called by the workflow engine that uses this controller for workflow
execution. It is threfore assumed that the state of the workflow run
is updated accordingly by the caller.
Parameters
----------
run_id: string
Unique run identifier.
"""
# Ensure that the run has not been removed already
if run_id in self.tasks:
workflow_id = self.tasks[run_id]
# Stop workflow execution at the engine. Ignore any errors that
# may be raised.
try:
self.client.stop_workflow(workflow_id)
except Exception as ex:
logging.error(ex, exc_info=True)
logging.debug('\n'.join(util.stacktrace(ex)))
# Delete the task from the dictionary. The state of the
# respective run will be updated by the workflow engine that
# uses this controller for workflow execution
del self.tasks[run_id]
def callback_function(result, lock, tasks, service):
"""Callback function for executed tasks.Removes the task from the task
index and updates the run state in the underlying database.
Parameters
----------
result: (string, dict)
Tuple of task identifier and serialized state of the workflow run
lock: multiprocessing.Lock
Lock for concurrency control
tasks: dict
Task index of the backend
service: contextlib,contextmanager
Context manager to create an instance of the service API.
"""
run_id, rundir, state_dict = result
logging.info('finished run {} with {}'.format(run_id, state_dict))
with lock:
if run_id in tasks:
# Close the pool and remove the entry from the task index
pool, _ = tasks[run_id]
pool.close()
del tasks[run_id]
state = serialize.deserialize_state(state_dict)
try:
with service() as api:
api.runs().update_run(run_id=run_id, state=state, rundir=rundir)
except Exception as ex:
logging.error(ex)
logging.debug('\n'.join(util.stacktrace(ex)))
def run_workflow(run_id: str, rundir: str, state: WorkflowState,
output_files: List[str], steps: List[ContainerStep],
arguments: Dict,
workers: WorkerFactory) -> Tuple[str, str, Dict]:
"""Execute a list of workflow steps synchronously.
This is the worker function for asynchronous workflow executions. Returns a
tuple containing the run identifier, the folder with the run files, and a
serialization of the workflow state.
Parameters
----------
run_id: string
Unique run identifier
rundir: string
Path to the working directory of the workflow run
state: flowserv.model.workflow.state.WorkflowState
Current workflow state (to access the timestamps)
output_files: list(string)
Relative path of output files that are generated by the workflow run
steps: list of flowserv.model.workflow.step.WorkflowStep
Steps in the serial workflow that are executed in the given context.
arguments: dict
Dictionary of argument values for parameters in the template.
workers: flowserv.controller.worker.factory.WorkerFactory, default=None
Factory for :class:`flowserv.model.workflow.step.ContainerStep` steps.
Returns
-------
(string, string, dict)
"""
logging.info('start run {}'.format(run_id))
try:
run_result = exec_workflow(steps=steps,
workers=workers,
rundir=rundir,
result=RunResult(arguments=arguments))
if run_result.returncode != 0:
# Return error state. Include STDERR in result
messages = run_result.log
result_state = state.error(messages=messages)
doc = serialize.serialize_state(result_state)
return run_id, rundir, doc
# Create list of output files that were generated.
files = list()
for relative_path in output_files:
if os.path.exists(os.path.join(rundir, relative_path)):
files.append(relative_path)
# Workflow executed successfully
result_state = state.success(files=files)
except Exception as ex:
logging.error(ex)
strace = util.stacktrace(ex)
logging.debug('\n'.join(strace))
result_state = state.error(messages=strace)
logging.info('finished run {}: {}'.format(run_id, result_state.type_id))
return run_id, rundir, serialize.serialize_state(result_state)
def run(self, step: ContainerStep, env: Dict, rundir: str) -> ExecResult:
"""Execute a list of shell commands in a workflow step synchronously.
Stops execution if one of the commands fails. Returns the combined
result from all the commands that were executed.
Parameters
----------
step: flowserv.controller.serial.workflow.ContainerStep
Step in a serial workflow.
env: dict, default=None
Default settings for environment variables when executing workflow
steps. May be None.
rundir: string
Path to the working directory of the workflow run.
Returns
-------
flowserv.controller.serial.workflow.result.ExecResult
"""
logging.info('run step with subprocess worker')
# Keep output to STDOUT and STDERR for all executed commands in the
# respective attributes of the returned execution result.
result = ExecResult(step=step)
# Windows-specific fix. Based on https://github.com/appveyor/ci/issues/1995
if 'SYSTEMROOT' in os.environ:
env = dict(env) if env else dict()
env['SYSTEMROOT'] = os.environ.get('SYSTEMROOT')
try:
# Run each command in the the workflow step. Each command is
# expected to be a shell command that is executed using the
# subprocess package. The subprocess.run() method is preferred for
# capturing output.
for cmd in step.commands:
logging.info('{}'.format(cmd))
proc = subprocess.run(
cmd,
cwd=rundir,
shell=True,
capture_output=True,
env=env
)
# Append output to STDOUT and STDERR to the respecive lists.
append(result.stdout, proc.stdout.decode('utf-8'))
append(result.stderr, proc.stderr.decode('utf-8'))
if proc.returncode != 0:
# Stop execution if the command failed.
result.returncode = proc.returncode
break
except Exception as ex:
logging.error(ex)
strace = '\n'.join(util.stacktrace(ex))
logging.debug(strace)
result.stderr.append(strace)
result.exception = ex
result.returncode = 1
return result
def run_postproc_workflow(postproc_spec: Dict, workflow: WorkflowObject,
ranking: List, runs: List, run_manager: RunManager,
backend: WorkflowController):
"""Run post-processing workflow for a workflow template."""
workflow_spec = postproc_spec.get('workflow')
pp_inputs = postproc_spec.get('inputs', {})
pp_files = pp_inputs.get('files', [])
# Prepare temporary directory with result files for all
# runs in the ranking. The created directory is the only
# run argument
strace = None
try:
datadir = postutil.prepare_postproc_data(input_files=pp_files,
ranking=ranking,
run_manager=run_manager)
dst = pp_inputs.get('runs', postbase.RUNS_DIR)
run_args = {
postbase.PARA_RUNS: InputFile(source=FSFile(datadir), target=dst)
}
arg_list = [
serialize_arg(postbase.PARA_RUNS, serialize_fh(datadir, dst))
]
except Exception as ex:
logging.error(ex)
strace = util.stacktrace(ex)
run_args = dict()
arg_list = []
# Create a new run for the workflow. The identifier for the run group is
# None.
run = run_manager.create_run(workflow=workflow,
arguments=arg_list,
runs=runs)
if strace is not None:
# If there were data preparation errors set the created run into an
# error state and return.
run_manager.update_run(run_id=run.run_id,
state=run.state().error(messages=strace))
else:
# Execute the post-processing workflow asynchronously if
# there were no data preparation errors.
postproc_state, rundir = backend.exec_workflow(
run=run,
template=WorkflowTemplate(workflow_spec=workflow_spec,
parameters=postbase.PARAMETERS),
arguments=run_args,
config=workflow.engine_config)
# Update the post-processing workflow run state if it is
# no longer pending for execution.
if not postproc_state.is_pending():
run_manager.update_run(run_id=run.run_id,
state=postproc_state,
rundir=rundir)
# Remove the temporary input folder
shutil.rmtree(datadir)
def run_workflow(run_id: str, state: WorkflowState, output_files: List[str],
steps: List[ContainerStep], arguments: Dict,
volumes: VolumeManager,
workers: WorkerPool) -> Tuple[str, str, Dict]:
"""Execute a list of workflow steps synchronously.
This is the worker function for asynchronous workflow executions. Returns a
tuple containing the run identifier, the folder with the run files, and a
serialization of the workflow state.
Parameters
----------
run_id: string
Unique run identifier
state: flowserv.model.workflow.state.WorkflowState
Current workflow state (to access the timestamps)
output_files: list(string)
Relative path of output files that are generated by the workflow run
steps: list of flowserv.model.workflow.step.WorkflowStep
Steps in the serial workflow that are executed in the given context.
arguments: dict
Dictionary of argument values for parameters in the template.
volumes: flowserv.volume.manager.VolumeManager
Factory for storage volumes.
workers: flowserv.controller.worker.manager.WorkerPool
Factory for :class:`flowserv.model.workflow.step.ContainerStep` steps.
Returns
-------
(string, string, dict)
"""
logging.info('start run {}'.format(run_id))
runstore = volumes.get(DEFAULT_STORE)
try:
run_result = exec_workflow(steps=steps,
workers=workers,
volumes=volumes,
result=RunResult(arguments=arguments))
if run_result.returncode != 0:
# Return error state. Include STDERR in result
messages = run_result.log
result_state = state.error(messages=messages)
doc = serialize.serialize_state(result_state)
return run_id, runstore.to_dict(), doc
# Workflow executed successfully
result_state = state.success(files=output_files)
except Exception as ex:
logging.error(ex, exc_info=True)
strace = util.stacktrace(ex)
logging.debug('\n'.join(strace))
result_state = state.error(messages=strace)
logging.info('finished run {}: {}'.format(run_id, result_state.type_id))
return run_id, runstore.to_dict(), serialize.serialize_state(result_state)
def exec(self, step: CodeStep, context: Dict,
store: FileSystemStorage) -> ExecResult:
"""Execute a workflow step of type :class:`flowserv.model.workflow.step.CodeStep`
in a given context.
Captures output to STDOUT and STDERR and includes them in the returned
execution result.
Note that the code worker expects a file system storage volume.
Parameters
----------
step: flowserv.model.workflow.step.CodeStep
Code step in a serial workflow.
context: dict
Context for the executed code.
store: flowserv.volume.fs.FileSystemStorage
Storage volume that contains the workflow run files.
Returns
-------
flowserv.controller.serial.workflow.result.ExecResult
"""
result = ExecResult(step=step)
out = sys.stdout
err = sys.stderr
sys.stdout = OutputStream(stream=result.stdout)
sys.stderr = OutputStream(stream=result.stderr)
# Change working directory temporarily.
cwd = os.getcwd()
os.chdir(store.basedir)
try:
step.exec(context=context)
except Exception as ex:
logging.error(ex, exc_info=True)
strace = '\n'.join(util.stacktrace(ex))
logging.debug(strace)
result.stderr.append(strace)
result.exception = ex
result.returncode = 1
finally:
# Make sure to reverse redirection of output streams
sys.stdout = out
sys.stderr = err
# Reset working directory.
os.chdir(cwd)
return result
def exec(self, step: NotebookStep, context: Dict,
store: FileSystemStorage) -> ExecResult:
"""Execute a given notebook workflow step in the current workflow
context.
The notebook engine expects a file system storage volume that provides
access to the notebook file and any other aditional input files.
Parameters
----------
step: flowserv.model.workflow.step.NotebookStep
Notebook step in a serial workflow.
context: dict
Dictionary of variables that represent the current workflow state.
store: flowserv.volume.fs.FileSystemStorage
Storage volume that contains the workflow run files.
Returns
-------
flowserv.controller.serial.workflow.result.ExecResult
"""
result = ExecResult(step=step)
# Create Docker image including papermill and notebook requirements.
try:
image, logs = docker_build(name=step.name,
requirements=step.requirements)
if logs:
result.stdout.append('\n'.join(logs))
except Exception as ex:
logging.error(ex, exc_info=True)
strace = '\n'.join(util.stacktrace(ex))
logging.debug(strace)
result.stderr.append(strace)
result.exception = ex
result.returncode = 1
return result
# Run notebook in Docker container.
cmd = step.cli_command(context=context)
result.stdout.append(f'run: {cmd}')
return docker_run(image=image,
commands=[cmd],
env=self.env,
rundir=store.basedir,
result=result)
def exec_func(step: FunctionStep, context: Dict, rundir: str) -> ExecResult:
"""Execute a workflow step of type :class:`flowserv.model.workflow.step.FunctionStep` in a given context.
Captures output to STDOUT and STDERR and includes them in the returned
execution result.
Parameters
----------
step: flowserv.model.workflow.step.FunctionStep
Code step in a serial workflow.
context: dict
Context for the executed code.
Returns
-------
flowserv.controller.serial.workflow.result.ExecResult
"""
result = ExecResult(step=step)
out = sys.stdout
err = sys.stderr
sys.stdout = OutputStream(stream=result.stdout)
sys.stderr = OutputStream(stream=result.stderr)
# Change working direcotry temporarily.
cwd = os.getcwd()
os.chdir(rundir)
try:
step.exec(context=context)
except Exception as ex:
logging.error(ex)
strace = '\n'.join(util.stacktrace(ex))
logging.debug(strace)
result.stderr.append(strace)
result.exception = ex
result.returncode = 1
finally:
# Make sure to reverse redirection of output streams
sys.stdout = out
sys.stderr = err
# Reset working directory.
os.chdir(cwd)
return result
def run(self, step: ContainerStep, env: Dict, rundir: str) -> ExecResult:
"""Execute a list of commands from a workflow steps synchronously using
the Docker engine.
Stops execution if one of the commands fails. Returns the combined
result from all the commands that were executed.
Parameters
----------
step: flowserv.controller.serial.workflow.ContainerStep
Step in a serial workflow.
env: dict, default=None
Default settings for environment variables when executing workflow
steps. May be None.
rundir: string
Path to the working directory of the workflow run that this step
belongs to.
Returns
-------
flowserv.controller.serial.workflow.result.ExecResult
"""
logging.info('run step with Docker worker')
# Keep output to STDOUT and STDERR for all executed commands in the
# respective attributes of the returned execution result.
result = ExecResult(step=step)
# Setup the workflow environment by obtaining volume information for
# all directories in the run folder.
volumes = dict()
for filename in os.listdir(rundir):
abs_file = os.path.abspath(os.path.join(rundir, filename))
if os.path.isdir(abs_file):
volumes[abs_file] = {
'bind': '/{}'.format(filename),
'mode': 'rw'
}
# Run the individual commands using the local Docker deamon. Import
# docker package here to avoid errors for installations that do not
# intend to use Docker and therefore did not install the package.
import docker
from docker.errors import ContainerError, ImageNotFound, APIError
client = docker.from_env()
try:
for cmd in step.commands:
logging.info('{}'.format(cmd))
logs = client.containers.run(image=step.image,
command=cmd,
volumes=volumes,
remove=True,
environment=env,
stdout=True)
if logs:
result.stdout.append(logs.decode('utf-8'))
except (ContainerError, ImageNotFound, APIError) as ex:
logging.error(ex)
strace = '\n'.join(util.stacktrace(ex))
logging.debug(strace)
result.stderr.append(strace)
result.exception = ex
result.returncode = 1
return result
def exec_workflow(
self, run: RunObject, template: WorkflowTemplate, arguments: Dict,
staticfs: StorageVolume, config: Optional[Dict] = None
) -> Tuple[WorkflowState, StorageVolume]:
"""Initiate the execution of a given workflow template for a set of
argument values. This will start a new process that executes a serial
workflow asynchronously. Returns the state of the workflow after the
process is stated (the state will therefore be RUNNING).
The set of arguments is not further validated. It is assumed that the
validation has been performed by the calling code (e.g., the run
service manager).
If the state of the run handle is not pending, an error is raised.
Parameters
----------
run: flowserv.model.base.RunObject
Handle for the run that is being executed.
template: flowserv.model.template.base.WorkflowTemplate
Workflow template containing the parameterized specification and
the parameter declarations.
arguments: dict
Dictionary of argument values for parameters in the template.
staticfs: flowserv.volume.base.StorageVolume
Storage volume that contains the static files from the workflow
template.
config: dict, default=None
Optional configuration settings are currently ignored. Included for
API completeness.
Returns
-------
flowserv.model.workflow.state.WorkflowState, flowserv.volume.base.StorageVolume
"""
# Get the run state. Ensure that the run is in pending state.
if not run.is_pending():
raise RuntimeError("invalid run state '{}'".format(run.state()))
try:
# Create a workflow on the remote engine. This will also upload all
# necessary files to the remote engine. Workflow execution may not
# be started (indicated by the state property of the returned
# handle for the remote workflow).
workflow = self.client.create_workflow(
run=run,
template=template,
arguments=arguments,
staticfs=staticfs
)
workflow_id = workflow.workflow_id
# Run the workflow. Depending on the values of the is_async flag
# the process will either block execution while monitoring the
# workflow state or not.
if self.is_async:
self.tasks[run.run_id] = workflow_id
# Start monitor tread for asynchronous monitoring.
monitor.WorkflowMonitor(
workflow=workflow,
poll_interval=self.poll_interval,
service=self.service,
tasks=self.tasks
).start()
return workflow.state, workflow.runstore
else:
# Run workflow synchronously. This will lock the calling thread
# while waiting (i.e., polling the remote engine) for the
# workflow execution to finish.
state = monitor.monitor_workflow(
workflow=workflow,
poll_interval=self.poll_interval
)
return state, workflow.runstore
except Exception as ex:
# Set the workflow runinto an ERROR state
logging.error(ex, exc_info=True)
strace = util.stacktrace(ex)
logging.debug('\n'.join(strace))
return run.state().error(messages=strace), None
def docker_run(image: str, commands: List[str], env: Dict, rundir: str,
result: ExecResult) -> ExecResult:
"""Helper function that executes a list of commands inside a Docker container.
Parameters
----------
image: string
Identifier of the Docker image to run.
commands: string or list of string
Commands that are executed inside the Docker container.
result: flowserv.controller.serial.workflow.result.ExecResult
Result object that will contain the run outputs and status code.
Returns
-------
flowserv.controller.serial.workflow.result.ExecResult
"""
# Setup the workflow environment by obtaining volume information for
# all directories in the run folder.
volumes = dict()
for filename in os.listdir(rundir):
abs_file = os.path.abspath(os.path.join(rundir, filename))
if os.path.isdir(abs_file):
volumes[abs_file] = {'bind': '/{}'.format(filename), 'mode': 'rw'}
# Run the individual commands using the local Docker deamon. Import
# docker package here to avoid errors for installations that do not
# intend to use Docker and therefore did not install the package.
import docker
from docker.errors import ContainerError, ImageNotFound, APIError
client = docker.from_env()
try:
for cmd in commands:
logging.info('{}'.format(cmd))
# Run detached container to be able to capture output to
# both, STDOUT and STDERR. DO NOT remove the container yet
# in order to be able to get the captured outputs.
container = client.containers.run(image=image,
command=cmd,
volumes=volumes,
remove=False,
environment=env,
detach=True)
# Wait for container to finish. The returned dictionary will
# contain the container's exit code ('StatusCode').
r = container.wait()
# Add container logs to the standard outputs for the workflow
# results.
logs = container.logs()
if logs:
result.stdout.append(logs.decode('utf-8'))
# Remove container if the remove flag is set to True.
container.remove()
# Check exit code for the container. If the code is not zero
# an error occurred and we exit the commands loop.
status_code = r.get('StatusCode')
if status_code != 0:
result.returncode = status_code
break
except (ContainerError, ImageNotFound, APIError) as ex:
logging.error(ex, exc_info=True)
strace = '\n'.join(util.stacktrace(ex))
logging.debug(strace)
result.stderr.append(strace)
result.exception = ex
result.returncode = 1
client.close()
return result
def exec_workflow(
self,
run: RunObject,
template: WorkflowTemplate,
arguments: Dict,
config: Optional[Dict] = None) -> Tuple[WorkflowState, str]:
"""Initiate the execution of a given workflow template for a set of
argument values. This will start a new process that executes a serial
workflow asynchronously.
The serial workflow engine executes workflows on the local machine and
therefore uses the file system to store temporary run files. The path
to the run folder is returned as the second value in the result tuple.
The first value in the result tuple is the state of the workflow after
the process is stated. If the workflow is executed asynchronously the
state will be RUNNING. Otherwise, the run state should be an inactive
state.
The set of arguments is not further validated. It is assumed that the
validation has been performed by the calling code (e.g., the run
service manager).
The optional configuration object can be used to override the worker
configuration that was provided at object instantiation. Expects a
dictionary with an element `workers` that contains a mapping of container
identifier to a container worker configuration object.
If the state of the run handle is not pending, an error is raised.
Parameters
----------
run: flowserv.model.base.RunObject
Handle for the run that is being executed.
template: flowserv.model.template.base.WorkflowTemplate
Workflow template containing the parameterized specification and
the parameter declarations.
arguments: dict
Dictionary of argument values for parameters in the template.
config: dict, default=None
Optional object to overwrite the worker configuration settings.
Returns
-------
flowserv.model.workflow.state.WorkflowState, string
Raises
------
flowserv.error.DuplicateRunError
"""
# Get the run state. Ensure that the run is in pending state
if not run.is_pending():
raise RuntimeError("invalid run state '{}'".format(run.state))
state = run.state()
rundir = os.path.join(self.runsdir, run.run_id)
# Get the worker configuration.
worker_config = self.worker_config if not config else config.get(
'workers')
# Get the source directory for static workflow files.
sourcedir = self.fs.workflow_staticdir(run.workflow.workflow_id)
# Get the list of workflow steps and the generated output files.
steps, run_args, outputs = parser.parse_template(template=template,
arguments=arguments)
try:
# Copy template files to the run folder.
self.fs.copy_folder(key=sourcedir, dst=rundir)
# Store any given file arguments in the run folder.
for key, para in template.parameters.items():
if para.is_file() and key in arguments:
file = arguments[key]
file.source().store(os.path.join(rundir, file.target()))
# Create top-level folder for all expected result files.
util.create_directories(basedir=rundir, files=outputs)
# Start a new process to run the workflow. Make sure to catch all
# exceptions to set the run state properly
state = state.start()
if self.is_async:
# Raise an error if the service manager is not given.
if self.service is None:
raise ValueError('service manager not given')
# Run steps asynchronously in a separate process
pool = Pool(processes=1)
task_callback_function = partial(callback_function,
lock=self.lock,
tasks=self.tasks,
service=self.service)
with self.lock:
self.tasks[run.run_id] = (pool, state)
pool.apply_async(run_workflow,
args=(run.run_id, rundir, state, outputs,
steps, run_args,
WorkerFactory(config=worker_config)),
callback=task_callback_function)
return state, rundir
else:
# Run steps synchronously and block the controller until done
_, _, state_dict = run_workflow(
run_id=run.run_id,
rundir=rundir,
state=state,
output_files=outputs,
steps=steps,
arguments=run_args,
workers=WorkerFactory(config=worker_config))
return serialize.deserialize_state(state_dict), rundir
except Exception as ex:
# Set the workflow runinto an ERROR state
logging.error(ex)
return state.error(messages=util.stacktrace(ex)), rundir
def run_postproc_workflow(workflow: WorkflowObject, ranking: List[RunResult],
keys: List[str], run_manager: RunManager,
tmpstore: StorageVolume, staticfs: StorageVolume,
backend: WorkflowController):
"""Run post-processing workflow for a workflow template.
Parameters
----------
workflow: flowserv.model.base.WorkflowObject
Handle for the workflow that triggered the post-processing workflow run.
ranking: list(flowserv.model.ranking.RunResult)
List of runs in the current result ranking.
keys: list of string
Sorted list of run identifier for runs in the ranking.
run_manager: flowserv.model.run.RunManager
Manager for workflow runs
tmpstore: flowserv.volume.base.StorageVolume
Temporary storage volume where the created post-processing files are
stored. This volume will be erased after the workflow is started.
staticfs: flowserv.volume.base.StorageVolume
Storage volume that contains the static files from the workflow
template.
backend: flowserv.controller.base.WorkflowController
Backend that is used to execute the post-processing workflow.
"""
# Get workflow specification and the list of input files from the
# post-processing statement.
postproc_spec = workflow.postproc_spec
workflow_spec = postproc_spec.get('workflow')
pp_inputs = postproc_spec.get('inputs', {})
pp_files = pp_inputs.get('files', [])
# Prepare temporary directory with result files for all
# runs in the ranking. The created directory is the only
# run argument
strace = None
try:
prepare_postproc_data(input_files=pp_files,
ranking=ranking,
run_manager=run_manager,
store=tmpstore)
dst = pp_inputs.get('runs', RUNS_DIR)
run_args = {PARA_RUNS: InputDirectory(store=tmpstore, target=RUNS_DIR)}
arg_list = [serialize_arg(PARA_RUNS, dst)]
except Exception as ex:
logging.error(ex, exc_info=True)
strace = util.stacktrace(ex)
run_args = dict()
arg_list = []
# Create a new run for the workflow. The identifier for the run group is
# None.
run = run_manager.create_run(workflow=workflow,
arguments=arg_list,
runs=keys)
if strace is not None:
# If there were data preparation errors set the created run into an
# error state and return.
run_manager.update_run(run_id=run.run_id,
state=run.state().error(messages=strace))
else:
# Execute the post-processing workflow asynchronously if
# there were no data preparation errors.
try:
postproc_state, runstore = backend.exec_workflow(
run=run,
template=WorkflowTemplate(workflow_spec=workflow_spec,
parameters=PARAMETERS),
arguments=run_args,
staticfs=staticfs,
config=workflow.engine_config)
except Exception as ex:
# Make sure to catch exceptions and set the run into an error state.
postproc_state = run.state().error(messages=util.stacktrace(ex))
runstore = None
# Update the post-processing workflow run state if it is
# no longer pending for execution.
if not postproc_state.is_pending():
run_manager.update_run(run_id=run.run_id,
state=postproc_state,
runstore=runstore)
# Erase the temporary storage volume.
tmpstore.erase()
def exec_workflow(
self,
run: RunObject,
template: WorkflowTemplate,
arguments: Dict,
staticfs: StorageVolume,
config: Optional[Dict] = None
) -> Tuple[WorkflowState, StorageVolume]:
"""Initiate the execution of a given workflow template for a set of
argument values. This will start a new process that executes a serial
workflow asynchronously.
The serial workflow engine executes workflows on the local machine and
therefore uses the file system to store temporary run files. The path
to the run folder is returned as the second value in the result tuple.
The first value in the result tuple is the state of the workflow after
the process is stated. If the workflow is executed asynchronously the
state will be RUNNING. Otherwise, the run state should be an inactive
state.
The set of arguments is not further validated. It is assumed that the
validation has been performed by the calling code (e.g., the run
service manager).
The optional configuration object can be used to override the worker
configuration that was provided at object instantiation. Expects a
dictionary with an element `workers` that contains a mapping of container
identifier to a container worker configuration object.
If the state of the run handle is not pending, an error is raised.
Parameters
----------
run: flowserv.model.base.RunObject
Handle for the run that is being executed.
template: flowserv.model.template.base.WorkflowTemplate
Workflow template containing the parameterized specification and
the parameter declarations.
arguments: dict
Dictionary of argument values for parameters in the template.
staticfs: flowserv.volume.base.StorageVolume
Storage volume that contains the static files from the workflow
template.
config: dict, default=None
Optional object to overwrite the worker configuration settings.
Returns
-------
flowserv.model.workflow.state.WorkflowState, flowserv.volume.base.StorageVolume
"""
# Get the run state. Raise an error if the run is not in pending state.
if not run.is_pending():
raise RuntimeError("invalid run state '{}'".format(run.state))
state = run.state()
# Create configuration dictionary that merges the engine global
# configuration with the workflow-specific one.
run_config = self.config if self.config is not None else dict()
if config:
run_config.update(config)
# Get the list of workflow steps, run arguments, and the list of output
# files that the workflow is expected to generate.
steps, run_args, outputs = parser.parse_template(template=template,
arguments=arguments)
# Create and prepare storage volume for run files.
runstore = self.fs.get_store_for_folder(key=util.join(
self.runsdir, run.run_id),
identifier=DEFAULT_STORE)
try:
# Copy template files to the run folder.
files = staticfs.copy(src=None, store=runstore)
# Store any given file arguments and additional input files
# that are required by actor parameters into the run folder.
for key, para in template.parameters.items():
if para.is_file() and key in arguments:
for key in arguments[key].copy(target=runstore):
files.append(key)
elif para.is_actor() and key in arguments:
input_files = arguments[key].files
for f in input_files if input_files else []:
for key in f.copy(target=runstore):
files.append(key)
# Create factory objects for storage volumes.
volumes = volume_manager(specs=run_config.get('volumes', []),
runstore=runstore,
runfiles=files)
# Create factory for workers. Include mapping of workflow steps to
# the worker that are responsible for their execution.
workers = WorkerPool(workers=run_config.get('workers', []),
managers={
doc['step']: doc['worker']
for doc in run_config.get('workflow', [])
})
# Start a new process to run the workflow. Make sure to catch all
# exceptions to set the run state properly.
state = state.start()
if self.is_async:
# Run steps asynchronously in a separate process
pool = Pool(processes=1)
task_callback_function = partial(callback_function,
lock=self.lock,
tasks=self.tasks,
service=self.service)
with self.lock:
self.tasks[run.run_id] = (pool, state)
pool.apply_async(run_workflow,
args=(run.run_id, state, outputs, steps,
run_args, volumes, workers),
callback=task_callback_function)
return state, runstore
else:
# Run steps synchronously and block the controller until done
_, _, state_dict = run_workflow(run_id=run.run_id,
state=state,
output_files=outputs,
steps=steps,
arguments=run_args,
volumes=volumes,
workers=workers)
return serialize.deserialize_state(state_dict), runstore
except Exception as ex:
# Set the workflow run into an ERROR state
logging.error(ex, exc_info=True)
return state.error(messages=util.stacktrace(ex)), runstore
