def test_resume_failed_task_and_successful_task(self, workflow_context,
thread_executor):
node = workflow_context.model.node.get_by_name(
tests_mock.models.DEPENDENCY_NODE_NAME)
node.attributes['invocations'] = models.Attribute.wrap(
'invocations', 0)
self._create_interface(workflow_context, node,
mock_pass_first_task_only)
ctx = self._prepare_execution_and_get_workflow_ctx(
workflow_context.model,
workflow_context.resource,
workflow_context.model.service.list()[0],
mock_parallel_tasks_workflow,
thread_executor,
inputs={
'retry_interval': 1,
'max_attempts': 2,
'number_of_tasks': 2
})
eng = engine.Engine(thread_executor)
wf_thread = Thread(target=eng.execute, kwargs=dict(ctx=ctx))
wf_thread.setDaemon(True)
wf_thread.start()
if custom_events['execution_failed'].wait(60) is False:
raise TimeoutError("Execution did not end")
tasks = workflow_context.model.task.list(filters={'_stub_type': None})
node = workflow_context.model.node.refresh(node)
assert node.attributes['invocations'].value == 3
failed_task = [t for t in tasks if t.status == t.FAILED][0]
# First task passes
assert any(task.status == task.FAILED for task in tasks)
assert failed_task.attempts_count == 2
# Second task fails
assert any(task.status == task.SUCCESS for task in tasks)
assert ctx.execution.status in ctx.execution.FAILED
custom_events['is_resumed'].set()
new_thread_executor = thread.ThreadExecutor()
try:
new_engine = engine.Engine(new_thread_executor)
new_engine.execute(ctx, resuming=True, retry_failed=True)
finally:
new_thread_executor.close()
# Wait for it to finish and assert changes.
node = workflow_context.model.node.refresh(node)
assert failed_task.attempts_count == 1
assert node.attributes['invocations'].value == 4
assert all(task.status == task.SUCCESS for task in tasks)
assert ctx.execution.status == ctx.execution.SUCCEEDED
def post(self, execution_id, **kwargs):
"""
Apply execution action (cancel, force-cancel) by id
"""
request_dict = get_json_and_verify_params({'action'})
action = request_dict['action']
valid_actions = ['cancel', 'force-cancel']
if action not in valid_actions:
raise manager_exceptions.BadParametersError(
'Invalid action: {0}, Valid action values are: {1}'.format(
action, valid_actions))
if action in ('cancel', 'force-cancel'):
service = self.model.execution.get(execution_id)
executor = process.ProcessExecutor(self.plugin_manager)
compiler = execution_preparer.ExecutionPreparer(
self.model,
self.resource,
self.plugin_manager,
service,
request_dict['workflow_name']
)
workflow_ctx = compiler.prepare(execution_id=execution_id)
engine_ = engine.Engine(executor)
engine_.cancel_execution(workflow_ctx)
def test_decorate_extension(context, executor):
arguments = {'arg1': 1, 'arg2': 2}
def get_node(ctx):
return ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME)
@workflow
def mock_workflow(ctx, graph):
node = get_node(ctx)
interface_name = 'test_interface'
operation_name = 'operation'
interface = mock.models.create_interface(
ctx.service,
interface_name,
operation_name,
operation_kwargs=dict(function='{0}.{1}'.format(__name__, _mock_operation.__name__),
arguments=arguments)
)
node.interfaces[interface.name] = interface
task = api.task.OperationTask(
node,
interface_name=interface_name,
operation_name=operation_name,
arguments=arguments)
graph.add_tasks(task)
return graph
graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter
eng = engine.Engine(executor=executor, workflow_context=context, tasks_graph=graph)
eng.execute()
out = get_node(context).attributes.get('out').value
assert out['wrapper_arguments'] == arguments
assert out['function_arguments'] == arguments
def post(self, **kwargs):
"""
Start an execution
"""
request_dict = rest_utils.get_json_and_verify_params(
dict(
service_id={'type': int},
workflow_name={'type': basestring},
)
)
service = self.model.service.get(request_dict['service_id'])
executor = process.ProcessExecutor(plugin_manager=self.plugin_manager)
compiler = execution_preparer.ExecutionPreparer(
self.model,
self.resource,
self.plugin_manager,
service,
request_dict['workflow_name']
)
workflow_ctx = compiler.prepare(executor=executor)
engine_ = engine.Engine(executor)
engine_.execute(workflow_ctx)
return workflow_ctx.execution.to_dict(
workflow_ctx.execution.fields() -
{'created_at', 'started_at', 'ended_at'}), \
201
def _run_workflow(context, executor, op_func, arguments=None):
node = context.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME)
interface_name = 'test_interface'
operation_name = 'operation'
wf_arguments = arguments or {}
interface = mock.models.create_interface(
context.service,
interface_name,
operation_name,
operation_kwargs=dict(function=_operation_mapping(op_func),
arguments=wf_arguments))
node.interfaces[interface.name] = interface
context.model.node.update(node)
@workflow
def mock_workflow(ctx, graph):
task = api.task.OperationTask(node,
interface_name=interface_name,
operation_name=operation_name,
arguments=wf_arguments)
graph.add_tasks(task)
return graph
graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter
graph_compiler.GraphCompiler(context, executor.__class__).compile(graph)
eng = engine.Engine({executor.__class__: executor})
eng.execute(context)
out = context.model.node.get_by_name(
mock.models.DEPENDENCY_NODE_NAME).attributes.get('out')
return out.value if out else None
def execute(env, workflow_name):
ctx = execution_preparer.ExecutionPreparer(env.model_storage,
env.resource_storage,
env.plugin_manager, env.service,
workflow_name).prepare()
eng = engine.Engine(
process.ProcessExecutor(env.plugin_manager, strict_loading=False))
# Since we want a live log feed, we need to execute the workflow
# while simultaneously printing the logs into the CFY logger. This Thread
# executes the workflow, while the main process thread writes the logs.
thread = Thread(target=eng.execute, kwargs=dict(ctx=ctx))
thread.start()
log_iterator = logger.ModelLogIterator(env.model_storage, ctx.execution.id)
while thread.is_alive():
for log in log_iterator:
leveled_log = getattr(env.ctx_logger, log.level.lower())
leveled_log(log)
if log.traceback:
leveled_log(log.traceback)
thread.join(0.1)
aria_execution = ctx.execution
if aria_execution.status != aria_execution.SUCCEEDED:
raise AriaWorkflowError(
'ARIA workflow {aria_execution.workflow_name} was not successful\n'
'status: {aria_execution.status}\n'
'error message: {aria_execution.error}'.format(
aria_execution=aria_execution))
def test_decorate_extension(context, executor):
inputs = {'input1': 1, 'input2': 2}
def get_node_instance(ctx):
return ctx.model.node_instance.get_by_name(
mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
@workflow
def mock_workflow(ctx, graph):
node_instance = get_node_instance(ctx)
op = 'test.op'
op_dict = {
'operation': '{0}.{1}'.format(__name__, _mock_operation.__name__)
}
node_instance.node.operations['test.op'] = op_dict
task = api.task.OperationTask.node_instance(instance=node_instance,
name=op,
inputs=inputs)
graph.add_tasks(task)
return graph
graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter
eng = engine.Engine(executor=executor,
workflow_context=context,
tasks_graph=graph)
eng.execute()
out = get_node_instance(context).runtime_properties['out']
assert out['wrapper_inputs'] == inputs
assert out['function_inputs'] == inputs
def _engine(workflow_func, workflow_context, executor):
graph = workflow_func(ctx=workflow_context)
execution = workflow_context.execution
graph_compiler.GraphCompiler(workflow_context, executor.__class__).compile(graph)
workflow_context.execution = execution
return engine.Engine(executors={executor.__class__: executor})
def execute(workflow_func, workflow_context, executor):
graph = workflow_func(ctx=workflow_context)
graph_compiler.GraphCompiler(workflow_context,
executor.__class__).compile(graph)
eng = engine.Engine(executor)
eng.execute(workflow_context)
def test_resume_failed_task(self, workflow_context, thread_executor):
node = workflow_context.model.node.get_by_name(
tests_mock.models.DEPENDENCY_NODE_NAME)
node.attributes['invocations'] = models.Attribute.wrap(
'invocations', 0)
self._create_interface(workflow_context, node,
mock_failed_before_resuming)
ctx = self._prepare_execution_and_get_workflow_ctx(
workflow_context.model, workflow_context.resource,
workflow_context.model.service.list()[0],
mock_parallel_tasks_workflow, thread_executor)
eng = engine.Engine(thread_executor)
wf_thread = Thread(target=eng.execute, kwargs=dict(ctx=ctx))
wf_thread.setDaemon(True)
wf_thread.start()
self._cancel_active_execution(eng, ctx)
node = workflow_context.model.node.refresh(node)
task = workflow_context.model.task.list(
filters={'_stub_type': None})[0]
assert node.attributes['invocations'].value == 2
assert task.status == task.STARTED
assert ctx.execution.status in (ctx.execution.CANCELLED,
ctx.execution.CANCELLING)
custom_events['is_resumed'].set()
assert node.attributes['invocations'].value == 2
# Create a new workflow runner, with an existing execution id. This would cause
# the old execution to restart.
new_thread_executor = thread.ThreadExecutor()
try:
new_engine = engine.Engine(new_thread_executor)
new_engine.execute(ctx, resuming=True)
finally:
new_thread_executor.close()
# Wait for it to finish and assert changes.
node = workflow_context.model.node.refresh(node)
assert node.attributes['invocations'].value == task.max_attempts - 1
assert task.status == task.SUCCESS
assert ctx.execution.status == ctx.execution.SUCCEEDED
def _execute(self,
env=None,
use_sudo=False,
hide_output=None,
process=None,
custom_input='',
test_operations=None,
commands=None):
process = process or {}
if env:
process.setdefault('env', {}).update(env)
test_operations = test_operations or [self.test_name]
local_script_path = os.path.join(resources.DIR, 'scripts',
'test_ssh.sh')
script_path = os.path.basename(local_script_path)
self._upload(local_script_path, script_path)
if commands:
operation = operations.run_commands_with_ssh
else:
operation = operations.run_script_with_ssh
@workflow
def mock_workflow(ctx, graph):
op = 'test.op'
node_instance = ctx.model.node_instance.get_by_name(
mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
node_instance.node.operations[op] = {
'operation':
'{0}.{1}'.format(operations.__name__, operation.__name__)
}
graph.sequence(*[
api.task.OperationTask.node_instance(
instance=node_instance,
name=op,
inputs={
'script_path': script_path,
'fabric_env': _FABRIC_ENV,
'process': process,
'use_sudo': use_sudo,
'hide_output': hide_output,
'custom_env_var': custom_input,
'test_operation': test_operation,
'commands': commands
}) for test_operation in test_operations
])
return graph
tasks_graph = mock_workflow(ctx=self._workflow_context) # pylint: disable=no-value-for-parameter
eng = engine.Engine(executor=self._executor,
workflow_context=self._workflow_context,
tasks_graph=tasks_graph)
eng.execute()
return self._workflow_context.model.node_instance.get_by_name(
mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties
def test_serialize_operation_context(context, executor, tmpdir):
test_file = tmpdir.join(TEST_FILE_NAME)
test_file.write(TEST_FILE_CONTENT)
resource = context.resource
resource.blueprint.upload(TEST_FILE_ENTRY_ID, str(test_file))
graph = _mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter
eng = engine.Engine(executor=executor,
workflow_context=context,
tasks_graph=graph)
eng.execute()
def test_resume_workflow(self, workflow_context, thread_executor):
node = workflow_context.model.node.get_by_name(
tests_mock.models.DEPENDENCY_NODE_NAME)
node.attributes['invocations'] = models.Attribute.wrap(
'invocations', 0)
self._create_interface(workflow_context, node,
mock_pass_first_task_only)
ctx = self._prepare_execution_and_get_workflow_ctx(
workflow_context.model,
workflow_context.resource,
workflow_context.model.service.list()[0],
mock_parallel_tasks_workflow,
thread_executor,
inputs={'number_of_tasks': 2})
eng = engine.Engine(thread_executor)
wf_thread = Thread(target=eng.execute, kwargs=dict(ctx=ctx))
wf_thread.daemon = True
wf_thread.start()
# Wait for the execution to start
self._cancel_active_execution(eng, ctx)
node = ctx.model.node.refresh(node)
tasks = ctx.model.task.list(filters={'_stub_type': None})
assert any(task.status == task.SUCCESS for task in tasks)
assert any(task.status == task.RETRYING for task in tasks)
custom_events['is_resumed'].set()
assert any(task.status == task.RETRYING for task in tasks)
# Create a new workflow engine, with an existing execution id. This would cause
# the old execution to restart.
new_engine = engine.Engine(thread_executor)
new_engine.execute(ctx, resuming=True)
# Wait for it to finish and assert changes.
node = workflow_context.model.node.refresh(node)
assert all(task.status == task.SUCCESS for task in tasks)
assert node.attributes['invocations'].value == 3
assert ctx.execution.status == ctx.execution.SUCCEEDED
def _run(self,
executor,
workflow_context,
script_path,
process=None,
env_var='value',
arguments=None):
local_script_path = script_path
script_path = os.path.basename(
local_script_path) if local_script_path else ''
arguments = arguments or {}
process = process or {}
if script_path:
workflow_context.resource.service.upload(entry_id=str(
workflow_context.service.id),
source=local_script_path,
path=script_path)
arguments.update({
'script_path': script_path,
'process': process,
'input_as_env_var': env_var
})
node = workflow_context.model.node.get_by_name(
mock.models.DEPENDENCY_NODE_NAME)
interface = mock.models.create_interface(
node.service,
'test',
'op',
operation_kwargs=dict(function='{0}.{1}'.format(
operations.__name__, operations.run_script_locally.__name__),
arguments=arguments))
node.interfaces[interface.name] = interface
workflow_context.model.node.update(node)
@workflow
def mock_workflow(ctx, graph):
graph.add_tasks(
api.task.OperationTask(node,
interface_name='test',
operation_name='op',
arguments=arguments))
return graph
tasks_graph = mock_workflow(ctx=workflow_context) # pylint: disable=no-value-for-parameter
graph_compiler.GraphCompiler(workflow_context,
executor.__class__).compile(tasks_graph)
eng = engine.Engine({executor.__class__: executor})
eng.execute(workflow_context)
return workflow_context.model.node.get_by_name(
mock.models.DEPENDENCY_NODE_NAME).attributes
def run_operation_on_node(ctx, op_name, interface_name):
node = ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME)
interface = mock.models.create_interface(
service=node.service,
interface_name=interface_name,
operation_name=op_name,
operation_kwargs=dict(function='{name}.{func.__name__}'.format(name=__name__, func=func)))
node.interfaces[interface.name] = interface
eng = engine.Engine(executor=ThreadExecutor(),
workflow_context=ctx,
tasks_graph=single_operation_workflow(ctx=ctx, # pylint: disable=no-value-for-parameter
node=node,
interface_name=interface_name,
op_name=op_name))
eng.execute()
return node
def run_operation_on_node(ctx, op_name, interface_name, executor):
node = ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME)
interface = mock.models.create_interface(
service=node.service,
interface_name=interface_name,
operation_name=op_name,
operation_kwargs=dict(function='{name}.{func.__name__}'.format(
name=__name__, func=func)))
node.interfaces[interface.name] = interface
graph_compiler.GraphCompiler(ctx, ThreadExecutor).compile(
single_operation_workflow(ctx,
node=node,
interface_name=interface_name,
op_name=op_name))
eng = engine.Engine(executor)
eng.execute(ctx)
return node
def _run(self,
executor,
workflow_context,
script_path,
process=None,
env_var='value',
inputs=None):
local_script_path = script_path
script_path = os.path.basename(local_script_path) if local_script_path else None
if script_path:
workflow_context.resource.deployment.upload(
entry_id=str(workflow_context.deployment.id),
source=local_script_path,
path=script_path)
inputs = inputs or {}
inputs.update({
'script_path': script_path,
'process': process,
'input_as_env_var': env_var
})
@workflow
def mock_workflow(ctx, graph):
op = 'test.op'
node_instance = ctx.model.node_instance.get_by_name(
mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
node_instance.node.operations[op] = {
'operation': '{0}.{1}'.format(operations.__name__,
operations.run_script_locally.__name__)}
graph.add_tasks(api.task.OperationTask.node_instance(
instance=node_instance,
name=op,
inputs=inputs))
return graph
tasks_graph = mock_workflow(ctx=workflow_context) # pylint: disable=no-value-for-parameter
eng = engine.Engine(
executor=executor,
workflow_context=workflow_context,
tasks_graph=tasks_graph)
eng.execute()
return workflow_context.model.node_instance.get_by_name(
mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties
def _run_workflow(context, executor, op_func, inputs=None):
@workflow
def mock_workflow(ctx, graph):
node_instance = ctx.model.node_instance.get_by_name(
mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
node_instance.node.operations['test.op'] = {
'operation': _operation_mapping(op_func)
}
task = api.task.OperationTask.node_instance(instance=node_instance,
name='test.op',
inputs=inputs or {})
graph.add_tasks(task)
return graph
graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter
eng = engine.Engine(executor=executor,
workflow_context=context,
tasks_graph=graph)
eng.execute()
return context.model.node_instance.get_by_name(
mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties.get(
'out')
def test_serialize_operation_context(context, executor, tmpdir):
test_file = tmpdir.join(TEST_FILE_NAME)
test_file.write(TEST_FILE_CONTENT)
resource = context.resource
resource.service_template.upload(TEST_FILE_ENTRY_ID, str(test_file))
node = context.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME)
plugin = mock.models.create_plugin()
context.model.plugin.put(plugin)
interface = mock.models.create_interface(node.service,
'test',
'op',
operation_kwargs=dict(
function=_operation_mapping(),
plugin=plugin))
node.interfaces[interface.name] = interface
context.model.node.update(node)
graph = _mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter
graph_compiler.GraphCompiler(context, executor.__class__).compile(graph)
eng = engine.Engine(executor)
eng.execute(context)
def _execute(self,
env=None,
use_sudo=False,
hide_output=None,
process=None,
custom_input='',
test_operations=None,
commands=None):
process = process or {}
if env:
process.setdefault('env', {}).update(env)
test_operations = test_operations or [self.test_name]
local_script_path = os.path.join(resources.DIR, 'scripts',
'test_ssh.sh')
script_path = os.path.basename(local_script_path)
self._upload(local_script_path, script_path)
if commands:
operation = operations.run_commands_with_ssh
else:
operation = operations.run_script_with_ssh
node = self._workflow_context.model.node.get_by_name(
mock.models.DEPENDENCY_NODE_NAME)
arguments = {
'script_path': script_path,
'fabric_env': _FABRIC_ENV,
'process': process,
'use_sudo': use_sudo,
'custom_env_var': custom_input,
'test_operation': '',
}
if hide_output:
arguments['hide_output'] = hide_output
if commands:
arguments['commands'] = commands
interface = mock.models.create_interface(
node.service,
'test',
'op',
operation_kwargs=dict(function='{0}.{1}'.format(
operations.__name__, operation.__name__),
arguments=arguments))
node.interfaces[interface.name] = interface
@workflow
def mock_workflow(ctx, graph):
ops = []
for test_operation in test_operations:
op_arguments = arguments.copy()
op_arguments['test_operation'] = test_operation
ops.append(
api.task.OperationTask(node,
interface_name='test',
operation_name='op',
arguments=op_arguments))
graph.sequence(*ops)
return graph
tasks_graph = mock_workflow(ctx=self._workflow_context) # pylint: disable=no-value-for-parameter
graph_compiler.GraphCompiler(
self._workflow_context,
self._executor.__class__).compile(tasks_graph)
eng = engine.Engine({self._executor.__class__: self._executor})
eng.execute(self._workflow_context)
return self._workflow_context.model.node.get_by_name(
mock.models.DEPENDENCY_NODE_NAME).attributes
def _engine(workflow_func, workflow_context, executor):
graph = workflow_func(ctx=workflow_context)
return engine.Engine(executor=executor,
workflow_context=workflow_context,
tasks_graph=graph)
def execute(workflow_func, workflow_context, executor):
graph = workflow_func(ctx=workflow_context)
eng = engine.Engine(executor=executor,
workflow_context=workflow_context,
tasks_graph=graph)
eng.execute()
def _run(self,
executor,
workflow_context,
func,
inputs=None,
max_attempts=None,
skip_common_assert=False,
operation_end=None,
plugin=None):
interface_name = 'test'
operation_name = 'op'
op_dict = {
'function': '{0}.{1}'.format(__name__, func.__name__),
'plugin': plugin,
'arguments': inputs or {}
}
node = self._get_node(workflow_context)
if operation_end:
actor = relationship = node.outbound_relationships[0]
relationship.interfaces[
interface_name] = mock.models.create_interface(
relationship.source_node.service,
interface_name,
operation_name,
operation_kwargs=op_dict)
workflow_context.model.relationship.update(relationship)
else:
actor = node
node.interfaces[interface_name] = mock.models.create_interface(
node.service,
interface_name,
operation_name,
operation_kwargs=op_dict)
workflow_context.model.node.update(node)
if inputs:
operation_inputs = \
actor.interfaces[interface_name].operations[operation_name].inputs
for input_name, input in inputs.iteritems():
operation_inputs[input_name] = models.Input(
name=input_name, type_name=type_.full_type_name(input))
@workflow
def mock_workflow(graph, **kwargs):
task = api.task.OperationTask(actor,
interface_name,
operation_name,
arguments=inputs or {},
max_attempts=max_attempts)
graph.add_tasks(task)
tasks_graph = mock_workflow(ctx=workflow_context)
graph_compiler.GraphCompiler(workflow_context,
executor.__class__).compile(tasks_graph)
eng = engine.Engine(executor)
eng.execute(workflow_context)
out = self._get_node(workflow_context).attributes['out'].value
if not skip_common_assert:
self._test_common(out, workflow_context)
return out
