def _engine(workflow_func, workflow_context, executor): graph = workflow_func(ctx=workflow_context) execution = workflow_context.execution graph_compiler.GraphCompiler(workflow_context, executor.__class__).compile(graph) workflow_context.execution = execution return engine.Engine(executors={executor.__class__: executor})
def _run_workflow(context, executor, op_func, arguments=None): node = context.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) interface_name = 'test_interface' operation_name = 'operation' wf_arguments = arguments or {} interface = mock.models.create_interface( context.service, interface_name, operation_name, operation_kwargs=dict(function=_operation_mapping(op_func), arguments=wf_arguments)) node.interfaces[interface.name] = interface context.model.node.update(node) @workflow def mock_workflow(ctx, graph): task = api.task.OperationTask(node, interface_name=interface_name, operation_name=operation_name, arguments=wf_arguments) graph.add_tasks(task) return graph graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter graph_compiler.GraphCompiler(context, executor.__class__).compile(graph) eng = engine.Engine({executor.__class__: executor}) eng.execute(context) out = context.model.node.get_by_name( mock.models.DEPENDENCY_NODE_NAME).attributes.get('out') return out.value if out else None
def execute(workflow_func, workflow_context, executor): graph = workflow_func(ctx=workflow_context) graph_compiler.GraphCompiler(workflow_context, executor.__class__).compile(graph) eng = engine.Engine(executor) eng.execute(workflow_context)
def _run(self, executor, workflow_context, script_path, process=None, env_var='value', arguments=None): local_script_path = script_path script_path = os.path.basename( local_script_path) if local_script_path else '' arguments = arguments or {} process = process or {} if script_path: workflow_context.resource.service.upload(entry_id=str( workflow_context.service.id), source=local_script_path, path=script_path) arguments.update({ 'script_path': script_path, 'process': process, 'input_as_env_var': env_var }) node = workflow_context.model.node.get_by_name( mock.models.DEPENDENCY_NODE_NAME) interface = mock.models.create_interface( node.service, 'test', 'op', operation_kwargs=dict(function='{0}.{1}'.format( operations.__name__, operations.run_script_locally.__name__), arguments=arguments)) node.interfaces[interface.name] = interface workflow_context.model.node.update(node) @workflow def mock_workflow(ctx, graph): graph.add_tasks( api.task.OperationTask(node, interface_name='test', operation_name='op', arguments=arguments)) return graph tasks_graph = mock_workflow(ctx=workflow_context) # pylint: disable=no-value-for-parameter graph_compiler.GraphCompiler(workflow_context, executor.__class__).compile(tasks_graph) eng = engine.Engine({executor.__class__: executor}) eng.execute(workflow_context) return workflow_context.model.node.get_by_name( mock.models.DEPENDENCY_NODE_NAME).attributes
def run_operation_on_node(ctx, op_name, interface_name, executor): node = ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) interface = mock.models.create_interface( service=node.service, interface_name=interface_name, operation_name=op_name, operation_kwargs=dict(function='{name}.{func.__name__}'.format( name=__name__, func=func))) node.interfaces[interface.name] = interface graph_compiler.GraphCompiler(ctx, ThreadExecutor).compile( single_operation_workflow(ctx, node=node, interface_name=interface_name, op_name=op_name)) eng = engine.Engine(executor) eng.execute(ctx) return node
def test_serialize_operation_context(context, executor, tmpdir): test_file = tmpdir.join(TEST_FILE_NAME) test_file.write(TEST_FILE_CONTENT) resource = context.resource resource.service_template.upload(TEST_FILE_ENTRY_ID, str(test_file)) node = context.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) plugin = mock.models.create_plugin() context.model.plugin.put(plugin) interface = mock.models.create_interface(node.service, 'test', 'op', operation_kwargs=dict( function=_operation_mapping(), plugin=plugin)) node.interfaces[interface.name] = interface context.model.node.update(node) graph = _mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter graph_compiler.GraphCompiler(context, executor.__class__).compile(graph) eng = engine.Engine(executor) eng.execute(context)
def test_decorate_extension(context, executor): arguments = {'arg1': 1, 'arg2': 2} def get_node(ctx): return ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) node = get_node(context) interface_name = 'test_interface' operation_name = 'operation' interface = mock.models.create_interface( context.service, interface_name, operation_name, operation_kwargs=dict(function='{0}.{1}'.format(__name__, _mock_operation.__name__), arguments=arguments) ) node.interfaces[interface.name] = interface context.model.node.update(node) @workflow def mock_workflow(ctx, graph): node = get_node(ctx) task = api.task.OperationTask( node, interface_name=interface_name, operation_name=operation_name, arguments=arguments) graph.add_tasks(task) return graph graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter graph_compiler.GraphCompiler(context, executor.__class__).compile(graph) eng = engine.Engine(executor) eng.execute(context) out = get_node(context).attributes.get('out').value assert out['wrapper_arguments'] == arguments assert out['function_arguments'] == arguments
def _execute(self, env=None, use_sudo=False, hide_output=None, process=None, custom_input='', test_operations=None, commands=None): process = process or {} if env: process.setdefault('env', {}).update(env) test_operations = test_operations or [self.test_name] local_script_path = os.path.join(resources.DIR, 'scripts', 'test_ssh.sh') script_path = os.path.basename(local_script_path) self._upload(local_script_path, script_path) if commands: operation = operations.run_commands_with_ssh else: operation = operations.run_script_with_ssh node = self._workflow_context.model.node.get_by_name( mock.models.DEPENDENCY_NODE_NAME) arguments = { 'script_path': script_path, 'fabric_env': _FABRIC_ENV, 'process': process, 'use_sudo': use_sudo, 'custom_env_var': custom_input, 'test_operation': '', } if hide_output: arguments['hide_output'] = hide_output if commands: arguments['commands'] = commands interface = mock.models.create_interface( node.service, 'test', 'op', operation_kwargs=dict(function='{0}.{1}'.format( operations.__name__, operation.__name__), arguments=arguments)) node.interfaces[interface.name] = interface @workflow def mock_workflow(ctx, graph): ops = [] for test_operation in test_operations: op_arguments = arguments.copy() op_arguments['test_operation'] = test_operation ops.append( api.task.OperationTask(node, interface_name='test', operation_name='op', arguments=op_arguments)) graph.sequence(*ops) return graph tasks_graph = mock_workflow(ctx=self._workflow_context) # pylint: disable=no-value-for-parameter graph_compiler.GraphCompiler( self._workflow_context, self._executor.__class__).compile(tasks_graph) eng = engine.Engine({self._executor.__class__: self._executor}) eng.execute(self._workflow_context) return self._workflow_context.model.node.get_by_name( mock.models.DEPENDENCY_NODE_NAME).attributes
def _run(self, executor, workflow_context, func, inputs=None, max_attempts=None, skip_common_assert=False, operation_end=None, plugin=None): interface_name = 'test' operation_name = 'op' op_dict = { 'function': '{0}.{1}'.format(__name__, func.__name__), 'plugin': plugin, 'arguments': inputs or {} } node = self._get_node(workflow_context) if operation_end: actor = relationship = node.outbound_relationships[0] relationship.interfaces[ interface_name] = mock.models.create_interface( relationship.source_node.service, interface_name, operation_name, operation_kwargs=op_dict) workflow_context.model.relationship.update(relationship) else: actor = node node.interfaces[interface_name] = mock.models.create_interface( node.service, interface_name, operation_name, operation_kwargs=op_dict) workflow_context.model.node.update(node) if inputs: operation_inputs = \ actor.interfaces[interface_name].operations[operation_name].inputs for input_name, input in inputs.iteritems(): operation_inputs[input_name] = models.Input( name=input_name, type_name=type_.full_type_name(input)) @workflow def mock_workflow(graph, **kwargs): task = api.task.OperationTask(actor, interface_name, operation_name, arguments=inputs or {}, max_attempts=max_attempts) graph.add_tasks(task) tasks_graph = mock_workflow(ctx=workflow_context) graph_compiler.GraphCompiler(workflow_context, executor.__class__).compile(tasks_graph) eng = engine.Engine(executor) eng.execute(workflow_context) out = self._get_node(workflow_context).attributes['out'].value if not skip_common_assert: self._test_common(out, workflow_context) return out
def test_task_graph_into_execution_graph(tmpdir): interface_name = 'Standard' op1_name, op2_name, op3_name = 'create', 'configure', 'start' workflow_context = mock.context.simple(str(tmpdir)) node = workflow_context.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME) interface = mock.models.create_interface( node.service, interface_name, op1_name, operation_kwargs=dict(function='test') ) interface.operations[op2_name] = mock.models.create_operation(op2_name) # pylint: disable=unsubscriptable-object interface.operations[op3_name] = mock.models.create_operation(op3_name) # pylint: disable=unsubscriptable-object node.interfaces[interface.name] = interface workflow_context.model.node.update(node) def sub_workflow(name, **_): return api.task_graph.TaskGraph(name) with context.workflow.current.push(workflow_context): test_task_graph = api.task.WorkflowTask(sub_workflow, name='test_task_graph') simple_before_task = api.task.OperationTask( node, interface_name=interface_name, operation_name=op1_name) simple_after_task = api.task.OperationTask( node, interface_name=interface_name, operation_name=op1_name) inner_task_graph = api.task.WorkflowTask(sub_workflow, name='test_inner_task_graph') inner_task_1 = api.task.OperationTask( node, interface_name=interface_name, operation_name=op1_name) inner_task_2 = api.task.OperationTask( node, interface_name=interface_name, operation_name=op2_name) inner_task_3 = api.task.OperationTask( node, interface_name=interface_name, operation_name=op3_name) inner_task_graph.add_tasks(inner_task_1) inner_task_graph.add_tasks(inner_task_2) inner_task_graph.add_tasks(inner_task_3) inner_task_graph.add_dependency(inner_task_2, inner_task_1) inner_task_graph.add_dependency(inner_task_3, inner_task_1) inner_task_graph.add_dependency(inner_task_3, inner_task_2) test_task_graph.add_tasks(simple_before_task) test_task_graph.add_tasks(simple_after_task) test_task_graph.add_tasks(inner_task_graph) test_task_graph.add_dependency(inner_task_graph, simple_before_task) test_task_graph.add_dependency(simple_after_task, inner_task_graph) compiler = graph_compiler.GraphCompiler(workflow_context, base.StubTaskExecutor) compiler.compile(test_task_graph) execution_tasks = topological_sort(_graph(workflow_context.execution.tasks)) assert len(execution_tasks) == 9 expected_tasks_names = [ '{0}-Start'.format(test_task_graph.id), simple_before_task.id, '{0}-Start'.format(inner_task_graph.id), inner_task_1.id, inner_task_2.id, inner_task_3.id, '{0}-End'.format(inner_task_graph.id), simple_after_task.id, '{0}-End'.format(test_task_graph.id) ] assert expected_tasks_names == [compiler._model_to_api_id[t.id] for t in execution_tasks] assert all(isinstance(task, models.Task) for task in execution_tasks) execution_tasks = iter(execution_tasks) _assert_tasks( iter(execution_tasks), iter([simple_after_task, inner_task_1, inner_task_2, inner_task_3, simple_after_task]) ) storage.release_sqlite_storage(workflow_context.model)