def test_resume_failed_task_and_successful_task(self, workflow_context, thread_executor): node = workflow_context.model.node.get_by_name( tests_mock.models.DEPENDENCY_NODE_NAME) node.attributes['invocations'] = models.Attribute.wrap( 'invocations', 0) self._create_interface(workflow_context, node, mock_pass_first_task_only) ctx = self._prepare_execution_and_get_workflow_ctx( workflow_context.model, workflow_context.resource, workflow_context.model.service.list()[0], mock_parallel_tasks_workflow, thread_executor, inputs={ 'retry_interval': 1, 'max_attempts': 2, 'number_of_tasks': 2 }) eng = engine.Engine(thread_executor) wf_thread = Thread(target=eng.execute, kwargs=dict(ctx=ctx)) wf_thread.setDaemon(True) wf_thread.start() if custom_events['execution_failed'].wait(60) is False: raise TimeoutError("Execution did not end") tasks = workflow_context.model.task.list(filters={'_stub_type': None}) node = workflow_context.model.node.refresh(node) assert node.attributes['invocations'].value == 3 failed_task = [t for t in tasks if t.status == t.FAILED][0] # First task passes assert any(task.status == task.FAILED for task in tasks) assert failed_task.attempts_count == 2 # Second task fails assert any(task.status == task.SUCCESS for task in tasks) assert ctx.execution.status in ctx.execution.FAILED custom_events['is_resumed'].set() new_thread_executor = thread.ThreadExecutor() try: new_engine = engine.Engine(new_thread_executor) new_engine.execute(ctx, resuming=True, retry_failed=True) finally: new_thread_executor.close() # Wait for it to finish and assert changes. node = workflow_context.model.node.refresh(node) assert failed_task.attempts_count == 1 assert node.attributes['invocations'].value == 4 assert all(task.status == task.SUCCESS for task in tasks) assert ctx.execution.status == ctx.execution.SUCCEEDED
def post(self, execution_id, **kwargs): """ Apply execution action (cancel, force-cancel) by id """ request_dict = get_json_and_verify_params({'action'}) action = request_dict['action'] valid_actions = ['cancel', 'force-cancel'] if action not in valid_actions: raise manager_exceptions.BadParametersError( 'Invalid action: {0}, Valid action values are: {1}'.format( action, valid_actions)) if action in ('cancel', 'force-cancel'): service = self.model.execution.get(execution_id) executor = process.ProcessExecutor(self.plugin_manager) compiler = execution_preparer.ExecutionPreparer( self.model, self.resource, self.plugin_manager, service, request_dict['workflow_name'] ) workflow_ctx = compiler.prepare(execution_id=execution_id) engine_ = engine.Engine(executor) engine_.cancel_execution(workflow_ctx)
def test_decorate_extension(context, executor): arguments = {'arg1': 1, 'arg2': 2} def get_node(ctx): return ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) @workflow def mock_workflow(ctx, graph): node = get_node(ctx) interface_name = 'test_interface' operation_name = 'operation' interface = mock.models.create_interface( ctx.service, interface_name, operation_name, operation_kwargs=dict(function='{0}.{1}'.format(__name__, _mock_operation.__name__), arguments=arguments) ) node.interfaces[interface.name] = interface task = api.task.OperationTask( node, interface_name=interface_name, operation_name=operation_name, arguments=arguments) graph.add_tasks(task) return graph graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter eng = engine.Engine(executor=executor, workflow_context=context, tasks_graph=graph) eng.execute() out = get_node(context).attributes.get('out').value assert out['wrapper_arguments'] == arguments assert out['function_arguments'] == arguments
def post(self, **kwargs): """ Start an execution """ request_dict = rest_utils.get_json_and_verify_params( dict( service_id={'type': int}, workflow_name={'type': basestring}, ) ) service = self.model.service.get(request_dict['service_id']) executor = process.ProcessExecutor(plugin_manager=self.plugin_manager) compiler = execution_preparer.ExecutionPreparer( self.model, self.resource, self.plugin_manager, service, request_dict['workflow_name'] ) workflow_ctx = compiler.prepare(executor=executor) engine_ = engine.Engine(executor) engine_.execute(workflow_ctx) return workflow_ctx.execution.to_dict( workflow_ctx.execution.fields() - {'created_at', 'started_at', 'ended_at'}), \ 201
def _run_workflow(context, executor, op_func, arguments=None): node = context.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) interface_name = 'test_interface' operation_name = 'operation' wf_arguments = arguments or {} interface = mock.models.create_interface( context.service, interface_name, operation_name, operation_kwargs=dict(function=_operation_mapping(op_func), arguments=wf_arguments)) node.interfaces[interface.name] = interface context.model.node.update(node) @workflow def mock_workflow(ctx, graph): task = api.task.OperationTask(node, interface_name=interface_name, operation_name=operation_name, arguments=wf_arguments) graph.add_tasks(task) return graph graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter graph_compiler.GraphCompiler(context, executor.__class__).compile(graph) eng = engine.Engine({executor.__class__: executor}) eng.execute(context) out = context.model.node.get_by_name( mock.models.DEPENDENCY_NODE_NAME).attributes.get('out') return out.value if out else None
def execute(env, workflow_name): ctx = execution_preparer.ExecutionPreparer(env.model_storage, env.resource_storage, env.plugin_manager, env.service, workflow_name).prepare() eng = engine.Engine( process.ProcessExecutor(env.plugin_manager, strict_loading=False)) # Since we want a live log feed, we need to execute the workflow # while simultaneously printing the logs into the CFY logger. This Thread # executes the workflow, while the main process thread writes the logs. thread = Thread(target=eng.execute, kwargs=dict(ctx=ctx)) thread.start() log_iterator = logger.ModelLogIterator(env.model_storage, ctx.execution.id) while thread.is_alive(): for log in log_iterator: leveled_log = getattr(env.ctx_logger, log.level.lower()) leveled_log(log) if log.traceback: leveled_log(log.traceback) thread.join(0.1) aria_execution = ctx.execution if aria_execution.status != aria_execution.SUCCEEDED: raise AriaWorkflowError( 'ARIA workflow {aria_execution.workflow_name} was not successful\n' 'status: {aria_execution.status}\n' 'error message: {aria_execution.error}'.format( aria_execution=aria_execution))
def test_decorate_extension(context, executor): inputs = {'input1': 1, 'input2': 2} def get_node_instance(ctx): return ctx.model.node_instance.get_by_name( mock.models.DEPENDENCY_NODE_INSTANCE_NAME) @workflow def mock_workflow(ctx, graph): node_instance = get_node_instance(ctx) op = 'test.op' op_dict = { 'operation': '{0}.{1}'.format(__name__, _mock_operation.__name__) } node_instance.node.operations['test.op'] = op_dict task = api.task.OperationTask.node_instance(instance=node_instance, name=op, inputs=inputs) graph.add_tasks(task) return graph graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter eng = engine.Engine(executor=executor, workflow_context=context, tasks_graph=graph) eng.execute() out = get_node_instance(context).runtime_properties['out'] assert out['wrapper_inputs'] == inputs assert out['function_inputs'] == inputs
def _engine(workflow_func, workflow_context, executor): graph = workflow_func(ctx=workflow_context) execution = workflow_context.execution graph_compiler.GraphCompiler(workflow_context, executor.__class__).compile(graph) workflow_context.execution = execution return engine.Engine(executors={executor.__class__: executor})
def execute(workflow_func, workflow_context, executor): graph = workflow_func(ctx=workflow_context) graph_compiler.GraphCompiler(workflow_context, executor.__class__).compile(graph) eng = engine.Engine(executor) eng.execute(workflow_context)
def test_resume_failed_task(self, workflow_context, thread_executor): node = workflow_context.model.node.get_by_name( tests_mock.models.DEPENDENCY_NODE_NAME) node.attributes['invocations'] = models.Attribute.wrap( 'invocations', 0) self._create_interface(workflow_context, node, mock_failed_before_resuming) ctx = self._prepare_execution_and_get_workflow_ctx( workflow_context.model, workflow_context.resource, workflow_context.model.service.list()[0], mock_parallel_tasks_workflow, thread_executor) eng = engine.Engine(thread_executor) wf_thread = Thread(target=eng.execute, kwargs=dict(ctx=ctx)) wf_thread.setDaemon(True) wf_thread.start() self._cancel_active_execution(eng, ctx) node = workflow_context.model.node.refresh(node) task = workflow_context.model.task.list( filters={'_stub_type': None})[0] assert node.attributes['invocations'].value == 2 assert task.status == task.STARTED assert ctx.execution.status in (ctx.execution.CANCELLED, ctx.execution.CANCELLING) custom_events['is_resumed'].set() assert node.attributes['invocations'].value == 2 # Create a new workflow runner, with an existing execution id. This would cause # the old execution to restart. new_thread_executor = thread.ThreadExecutor() try: new_engine = engine.Engine(new_thread_executor) new_engine.execute(ctx, resuming=True) finally: new_thread_executor.close() # Wait for it to finish and assert changes. node = workflow_context.model.node.refresh(node) assert node.attributes['invocations'].value == task.max_attempts - 1 assert task.status == task.SUCCESS assert ctx.execution.status == ctx.execution.SUCCEEDED
def _execute(self, env=None, use_sudo=False, hide_output=None, process=None, custom_input='', test_operations=None, commands=None): process = process or {} if env: process.setdefault('env', {}).update(env) test_operations = test_operations or [self.test_name] local_script_path = os.path.join(resources.DIR, 'scripts', 'test_ssh.sh') script_path = os.path.basename(local_script_path) self._upload(local_script_path, script_path) if commands: operation = operations.run_commands_with_ssh else: operation = operations.run_script_with_ssh @workflow def mock_workflow(ctx, graph): op = 'test.op' node_instance = ctx.model.node_instance.get_by_name( mock.models.DEPENDENCY_NODE_INSTANCE_NAME) node_instance.node.operations[op] = { 'operation': '{0}.{1}'.format(operations.__name__, operation.__name__) } graph.sequence(*[ api.task.OperationTask.node_instance( instance=node_instance, name=op, inputs={ 'script_path': script_path, 'fabric_env': _FABRIC_ENV, 'process': process, 'use_sudo': use_sudo, 'hide_output': hide_output, 'custom_env_var': custom_input, 'test_operation': test_operation, 'commands': commands }) for test_operation in test_operations ]) return graph tasks_graph = mock_workflow(ctx=self._workflow_context) # pylint: disable=no-value-for-parameter eng = engine.Engine(executor=self._executor, workflow_context=self._workflow_context, tasks_graph=tasks_graph) eng.execute() return self._workflow_context.model.node_instance.get_by_name( mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties
def test_serialize_operation_context(context, executor, tmpdir): test_file = tmpdir.join(TEST_FILE_NAME) test_file.write(TEST_FILE_CONTENT) resource = context.resource resource.blueprint.upload(TEST_FILE_ENTRY_ID, str(test_file)) graph = _mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter eng = engine.Engine(executor=executor, workflow_context=context, tasks_graph=graph) eng.execute()
def test_resume_workflow(self, workflow_context, thread_executor): node = workflow_context.model.node.get_by_name( tests_mock.models.DEPENDENCY_NODE_NAME) node.attributes['invocations'] = models.Attribute.wrap( 'invocations', 0) self._create_interface(workflow_context, node, mock_pass_first_task_only) ctx = self._prepare_execution_and_get_workflow_ctx( workflow_context.model, workflow_context.resource, workflow_context.model.service.list()[0], mock_parallel_tasks_workflow, thread_executor, inputs={'number_of_tasks': 2}) eng = engine.Engine(thread_executor) wf_thread = Thread(target=eng.execute, kwargs=dict(ctx=ctx)) wf_thread.daemon = True wf_thread.start() # Wait for the execution to start self._cancel_active_execution(eng, ctx) node = ctx.model.node.refresh(node) tasks = ctx.model.task.list(filters={'_stub_type': None}) assert any(task.status == task.SUCCESS for task in tasks) assert any(task.status == task.RETRYING for task in tasks) custom_events['is_resumed'].set() assert any(task.status == task.RETRYING for task in tasks) # Create a new workflow engine, with an existing execution id. This would cause # the old execution to restart. new_engine = engine.Engine(thread_executor) new_engine.execute(ctx, resuming=True) # Wait for it to finish and assert changes. node = workflow_context.model.node.refresh(node) assert all(task.status == task.SUCCESS for task in tasks) assert node.attributes['invocations'].value == 3 assert ctx.execution.status == ctx.execution.SUCCEEDED
def _run(self, executor, workflow_context, script_path, process=None, env_var='value', arguments=None): local_script_path = script_path script_path = os.path.basename( local_script_path) if local_script_path else '' arguments = arguments or {} process = process or {} if script_path: workflow_context.resource.service.upload(entry_id=str( workflow_context.service.id), source=local_script_path, path=script_path) arguments.update({ 'script_path': script_path, 'process': process, 'input_as_env_var': env_var }) node = workflow_context.model.node.get_by_name( mock.models.DEPENDENCY_NODE_NAME) interface = mock.models.create_interface( node.service, 'test', 'op', operation_kwargs=dict(function='{0}.{1}'.format( operations.__name__, operations.run_script_locally.__name__), arguments=arguments)) node.interfaces[interface.name] = interface workflow_context.model.node.update(node) @workflow def mock_workflow(ctx, graph): graph.add_tasks( api.task.OperationTask(node, interface_name='test', operation_name='op', arguments=arguments)) return graph tasks_graph = mock_workflow(ctx=workflow_context) # pylint: disable=no-value-for-parameter graph_compiler.GraphCompiler(workflow_context, executor.__class__).compile(tasks_graph) eng = engine.Engine({executor.__class__: executor}) eng.execute(workflow_context) return workflow_context.model.node.get_by_name( mock.models.DEPENDENCY_NODE_NAME).attributes
def run_operation_on_node(ctx, op_name, interface_name): node = ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) interface = mock.models.create_interface( service=node.service, interface_name=interface_name, operation_name=op_name, operation_kwargs=dict(function='{name}.{func.__name__}'.format(name=__name__, func=func))) node.interfaces[interface.name] = interface eng = engine.Engine(executor=ThreadExecutor(), workflow_context=ctx, tasks_graph=single_operation_workflow(ctx=ctx, # pylint: disable=no-value-for-parameter node=node, interface_name=interface_name, op_name=op_name)) eng.execute() return node
def run_operation_on_node(ctx, op_name, interface_name, executor): node = ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) interface = mock.models.create_interface( service=node.service, interface_name=interface_name, operation_name=op_name, operation_kwargs=dict(function='{name}.{func.__name__}'.format( name=__name__, func=func))) node.interfaces[interface.name] = interface graph_compiler.GraphCompiler(ctx, ThreadExecutor).compile( single_operation_workflow(ctx, node=node, interface_name=interface_name, op_name=op_name)) eng = engine.Engine(executor) eng.execute(ctx) return node
def _run(self, executor, workflow_context, script_path, process=None, env_var='value', inputs=None): local_script_path = script_path script_path = os.path.basename(local_script_path) if local_script_path else None if script_path: workflow_context.resource.deployment.upload( entry_id=str(workflow_context.deployment.id), source=local_script_path, path=script_path) inputs = inputs or {} inputs.update({ 'script_path': script_path, 'process': process, 'input_as_env_var': env_var }) @workflow def mock_workflow(ctx, graph): op = 'test.op' node_instance = ctx.model.node_instance.get_by_name( mock.models.DEPENDENCY_NODE_INSTANCE_NAME) node_instance.node.operations[op] = { 'operation': '{0}.{1}'.format(operations.__name__, operations.run_script_locally.__name__)} graph.add_tasks(api.task.OperationTask.node_instance( instance=node_instance, name=op, inputs=inputs)) return graph tasks_graph = mock_workflow(ctx=workflow_context) # pylint: disable=no-value-for-parameter eng = engine.Engine( executor=executor, workflow_context=workflow_context, tasks_graph=tasks_graph) eng.execute() return workflow_context.model.node_instance.get_by_name( mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties
def _run_workflow(context, executor, op_func, inputs=None): @workflow def mock_workflow(ctx, graph): node_instance = ctx.model.node_instance.get_by_name( mock.models.DEPENDENCY_NODE_INSTANCE_NAME) node_instance.node.operations['test.op'] = { 'operation': _operation_mapping(op_func) } task = api.task.OperationTask.node_instance(instance=node_instance, name='test.op', inputs=inputs or {}) graph.add_tasks(task) return graph graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter eng = engine.Engine(executor=executor, workflow_context=context, tasks_graph=graph) eng.execute() return context.model.node_instance.get_by_name( mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties.get( 'out')
def test_serialize_operation_context(context, executor, tmpdir): test_file = tmpdir.join(TEST_FILE_NAME) test_file.write(TEST_FILE_CONTENT) resource = context.resource resource.service_template.upload(TEST_FILE_ENTRY_ID, str(test_file)) node = context.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) plugin = mock.models.create_plugin() context.model.plugin.put(plugin) interface = mock.models.create_interface(node.service, 'test', 'op', operation_kwargs=dict( function=_operation_mapping(), plugin=plugin)) node.interfaces[interface.name] = interface context.model.node.update(node) graph = _mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter graph_compiler.GraphCompiler(context, executor.__class__).compile(graph) eng = engine.Engine(executor) eng.execute(context)
def _execute(self, env=None, use_sudo=False, hide_output=None, process=None, custom_input='', test_operations=None, commands=None): process = process or {} if env: process.setdefault('env', {}).update(env) test_operations = test_operations or [self.test_name] local_script_path = os.path.join(resources.DIR, 'scripts', 'test_ssh.sh') script_path = os.path.basename(local_script_path) self._upload(local_script_path, script_path) if commands: operation = operations.run_commands_with_ssh else: operation = operations.run_script_with_ssh node = self._workflow_context.model.node.get_by_name( mock.models.DEPENDENCY_NODE_NAME) arguments = { 'script_path': script_path, 'fabric_env': _FABRIC_ENV, 'process': process, 'use_sudo': use_sudo, 'custom_env_var': custom_input, 'test_operation': '', } if hide_output: arguments['hide_output'] = hide_output if commands: arguments['commands'] = commands interface = mock.models.create_interface( node.service, 'test', 'op', operation_kwargs=dict(function='{0}.{1}'.format( operations.__name__, operation.__name__), arguments=arguments)) node.interfaces[interface.name] = interface @workflow def mock_workflow(ctx, graph): ops = [] for test_operation in test_operations: op_arguments = arguments.copy() op_arguments['test_operation'] = test_operation ops.append( api.task.OperationTask(node, interface_name='test', operation_name='op', arguments=op_arguments)) graph.sequence(*ops) return graph tasks_graph = mock_workflow(ctx=self._workflow_context) # pylint: disable=no-value-for-parameter graph_compiler.GraphCompiler( self._workflow_context, self._executor.__class__).compile(tasks_graph) eng = engine.Engine({self._executor.__class__: self._executor}) eng.execute(self._workflow_context) return self._workflow_context.model.node.get_by_name( mock.models.DEPENDENCY_NODE_NAME).attributes
def _engine(workflow_func, workflow_context, executor): graph = workflow_func(ctx=workflow_context) return engine.Engine(executor=executor, workflow_context=workflow_context, tasks_graph=graph)
def execute(workflow_func, workflow_context, executor): graph = workflow_func(ctx=workflow_context) eng = engine.Engine(executor=executor, workflow_context=workflow_context, tasks_graph=graph) eng.execute()
def _run(self, executor, workflow_context, func, inputs=None, max_attempts=None, skip_common_assert=False, operation_end=None, plugin=None): interface_name = 'test' operation_name = 'op' op_dict = { 'function': '{0}.{1}'.format(__name__, func.__name__), 'plugin': plugin, 'arguments': inputs or {} } node = self._get_node(workflow_context) if operation_end: actor = relationship = node.outbound_relationships[0] relationship.interfaces[ interface_name] = mock.models.create_interface( relationship.source_node.service, interface_name, operation_name, operation_kwargs=op_dict) workflow_context.model.relationship.update(relationship) else: actor = node node.interfaces[interface_name] = mock.models.create_interface( node.service, interface_name, operation_name, operation_kwargs=op_dict) workflow_context.model.node.update(node) if inputs: operation_inputs = \ actor.interfaces[interface_name].operations[operation_name].inputs for input_name, input in inputs.iteritems(): operation_inputs[input_name] = models.Input( name=input_name, type_name=type_.full_type_name(input)) @workflow def mock_workflow(graph, **kwargs): task = api.task.OperationTask(actor, interface_name, operation_name, arguments=inputs or {}, max_attempts=max_attempts) graph.add_tasks(task) tasks_graph = mock_workflow(ctx=workflow_context) graph_compiler.GraphCompiler(workflow_context, executor.__class__).compile(tasks_graph) eng = engine.Engine(executor) eng.execute(workflow_context) out = self._get_node(workflow_context).attributes['out'].value if not skip_common_assert: self._test_common(out, workflow_context) return out