def test_with_items_action_context(self): wb_service.create_workbook_v2(WB_ACTION_CONTEXT) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf', WF_INPUT_URLS) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] act_exs = task_ex.executions self.engine.on_action_complete(act_exs[0].id, wf_utils.Result("Ivan")) self.engine.on_action_complete(act_exs[1].id, wf_utils.Result("John")) self.engine.on_action_complete( act_exs[2].id, wf_utils.Result("Mistral") ) self.await_workflow_success(wf_ex.id) with db_api.transaction(): task_ex = db_api.get_task_execution(task_ex.id) result = data_flow.get_task_execution_result(task_ex) self.assertIsInstance(result, list) self.assertIn('John', result) self.assertIn('Ivan', result) self.assertIn('Mistral', result) self.assertEqual(states.SUCCESS, task_ex.state)
def test_retry_policy_from_var_zero_iterations(self): wb_service.create_workbook_v2(RETRY_WB_FROM_VAR) # Start workflow. wf_ex = self.engine.start_workflow( 'wb.wf1', wf_input={'count': 0, 'delay': 1} ) with db_api.transaction(): # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self.assertEqual(states.RUNNING, task_ex.state) self.assertDictEqual({}, task_ex.runtime_context) try: self.await_task_delayed(task_ex.id, delay=0.5) except AssertionError: # There were no scheduled tasks as expected. pass else: self.fail("Shouldn't happen") self.await_task_error(task_ex.id) self.await_workflow_error(wf_ex.id) self.assertNotIn("retry_task_policy", task_ex.runtime_context)
def test_resume_direct(self): wb_service.create_workbook_v2(RESUME_WORKBOOK) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1') self.await_workflow_paused(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.PAUSED, wf_ex.state) self.assertEqual(2, len(task_execs)) self.engine.resume_workflow(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(2, len(wf_ex.task_executions)) self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.SUCCESS, wf_ex.state) self.assertEqual(2, len(task_execs))
def test_wait_after_policy_from_var_zero_seconds(self): wb_service.create_workbook_v2(WAIT_AFTER_FROM_VAR) # Start workflow. wf_ex = self.engine.start_workflow( 'wb.wf1', wf_input={'wait_after': 0} ) with db_api.transaction(): # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self.assertEqual(states.RUNNING, task_ex.state) self.assertDictEqual({}, task_ex.runtime_context) try: self.await_task_delayed(task_ex.id, delay=0.5) except AssertionError: # There was no delay as expected. pass else: self.fail("Shouldn't happen") self.await_task_success(task_ex.id)
def test_retry_policy_from_var(self): wb_service.create_workbook_v2(RETRY_WB_FROM_VAR) # Start workflow. wf_ex = self.engine.start_workflow( 'wb.wf1', wf_input={'count': 3, 'delay': 1} ) with db_api.transaction(): # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self.assertEqual(states.RUNNING, task_ex.state) self.assertDictEqual({}, task_ex.runtime_context) self.await_task_delayed(task_ex.id, delay=0.5) self.await_task_error(task_ex.id) self.await_workflow_error(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self.assertEqual( 3, task_ex.runtime_context["retry_task_policy"]["retry_no"] )
def test_wrong_policy_prop_type(self): wb = """--- version: "2.0" name: wb workflows: wf1: type: direct input: - wait_before tasks: task1: action: std.echo output="Hi!" wait-before: <% $.wait_before %> """ wb_service.create_workbook_v2(wb) # Start workflow. wf_ex = self.engine.start_workflow( 'wb.wf1', wf_input={'wait_before': '1'} ) self.assertIn( 'Invalid data type in WaitBeforePolicy', wf_ex.state_info ) self.assertEqual(states.ERROR, wf_ex.state)
def test_timeout_policy_from_var_zero_seconds(self): wb = """--- version: '2.0' name: wb workflows: wf1: type: direct input: - timeout tasks: task1: action: std.echo output="Hi!" timeout: <% $.timeout %> """ wb_service.create_workbook_v2(wb) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1', wf_input={'timeout': 0}) with db_api.transaction(): # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self.assertEqual(states.RUNNING, task_ex.state) self.await_task_success(task_ex.id) self.await_workflow_success(wf_ex.id)
def test_timeout_policy_success_after_timeout(self): wb_service.create_workbook_v2(TIMEOUT_WB2) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1') with db_api.transaction(): # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self.assertEqual(states.RUNNING, task_ex.state) self.await_task_error(task_ex.id) self.await_workflow_error(wf_ex.id) # Wait until timeout exceeds. self._sleep(1) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions # Make sure that engine did not create extra tasks. self.assertEqual(1, len(task_execs))
def test_with_items_results_one_item_as_list(self): wb_service.create_workbook_v2(WB) # Start workflow. wf_ex = self.engine.start_workflow('wb1.with_items', WF_INPUT_ONE_ITEM) self.await_execution_success(wf_ex.id) # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(1, len(task_execs)) task1_ex = self._assert_single_item( task_execs, name='task1', state=states.SUCCESS ) result = data_flow.get_task_execution_result(task1_ex) self.assertIsInstance(result, list) self.assertIn('Guy', result) self.assertIn(task1_ex.published['result'], ['Guy'])
def test_rerun_from_prev_step(self): wb_service.create_workbook_v2(SIMPLE_WORKBOOK) # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb1.wf1', {}, task_name='t3') self.await_workflow_error(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(2, len(wf_ex.task_executions)) task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1') task_2_ex = self._assert_single_item(wf_ex.task_executions, name='t2') self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertEqual(states.ERROR, task_2_ex.state) self.assertIsNotNone(task_2_ex.state_info) # Resume workflow and re-run failed task. e = self.assertRaises( exc.MistralError, self.engine.rerun_workflow, task_1_ex.id ) self.assertIn('not supported', str(e))
def test_rerun_with_items_concurrency(self): wb_service.create_workbook_v2(WITH_ITEMS_WORKBOOK_CONCURRENCY) # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb3.wf1', {}) self._await(lambda: self.is_execution_error(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(1, len(wf_ex.task_executions)) task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1') self.assertEqual(states.ERROR, task_1_ex.state) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) self.assertEqual(4, len(task_1_action_exs)) # Resume workflow and re-run failed task. self.engine.rerun_workflow(wf_ex.id, task_1_ex.id, reset=False) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) self._await(lambda: self.is_execution_success(wf_ex.id), delay=10) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.SUCCESS, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertEqual(2, len(wf_ex.task_executions)) task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1') task_2_ex = self._assert_single_item(wf_ex.task_executions, name='t2') # Check action executions of task 1. self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertIsNone(task_1_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) # The action executions that succeeded should not re-run. self.assertEqual(6, len(task_1_action_exs)) self.assertListEqual(['Task 1.0', 'Task 1.1', 'Task 1.2', 'Task 1.3'], task_1_ex.published.get('v1')) # Check action executions of task 2. self.assertEqual(states.SUCCESS, task_2_ex.state) task_2_action_exs = db_api.get_action_executions( task_execution_id=task_2_ex.id ) self.assertEqual(1, len(task_2_action_exs))
def test_resume_fails(self): # Start and pause workflow. wb_service.create_workbook_v2(WORKBOOK_DIFFERENT_TASK_STATES) wf_ex = self.engine.start_workflow('wb.wf1', {}) self.await_workflow_paused(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.PAUSED, wf_ex.state) # Simulate failure and check if it is handled. err = exc.MistralError('foo') with mock.patch.object( db_api, 'get_workflow_execution', side_effect=err): self.assertRaises( exc.MistralError, self.engine.resume_workflow, wf_ex.id )
def test_resume_direct(self): wb_service.create_workbook_v2(RESUME_WORKBOOK) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1', {}) self._await(lambda: self.is_execution_paused(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.PAUSED, wf_ex.state) self.assertEqual(2, len(wf_ex.task_executions)) self.engine.resume_workflow(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(2, len(wf_ex.task_executions)) self._await(lambda: self.is_execution_success(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.SUCCESS, wf_ex.state) self.assertEqual(2, len(wf_ex.task_executions))
def test_resume_two_start_tasks(self): wb_service.create_workbook_v2(WORKBOOK_TWO_START_TASKS) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1', {}) self._await(lambda: self.is_execution_paused(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.PAUSED, wf_ex.state) task_execs = wf_ex.task_executions # The exact number of tasks depends on which of two tasks # 'task1' and 'task2' completed earlier. self.assertTrue(len(task_execs) >= 2) task1_ex = self._assert_single_item(task_execs, name='task1') task2_ex = self._assert_single_item(task_execs, name='task2') self._await(lambda: self.is_task_success(task1_ex.id)) self._await(lambda: self.is_task_success(task2_ex.id)) self.engine.resume_workflow(wf_ex.id) self._await(lambda: self.is_execution_success(wf_ex.id), 1, 5) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.SUCCESS, wf_ex.state) self.assertEqual(3, len(wf_ex.task_executions))
def test_resume_fails(self, mock_fw): # Start and pause workflow. wb_service.create_workbook_v2(WORKBOOK_DIFFERENT_TASK_STATES) wf_ex = self.engine.start_workflow('wb.wf1', {}) self._await(lambda: self.is_execution_paused(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.PAUSED, wf_ex.state) # Simulate failure and check if it is handled. err = exc.MistralException('foo') with mock.patch.object( db_api, 'acquire_lock', side_effect=err): self.assertRaises( exc.MistralException, self.engine.resume_workflow, wf_ex.id ) mock_fw.assert_called_once_with(wf_ex.id, err)
def test_with_items_multi_array(self): wb_service.create_workbook_v2(WORKBOOK_MULTI_ARRAY) wf_input = {'arrayI': ['a', 'b', 'c'], 'arrayJ': [1, 2, 3]} # Start workflow. wf_ex = self.engine.start_workflow('wb1.with_items', wf_input) self._await( lambda: self.is_execution_success(wf_ex.id), ) # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) tasks = wf_ex.task_executions task1 = self._assert_single_item(tasks, name='task1') # Since we know that we can receive results in random order, # check is not depend on order of items. result = data_flow.get_task_execution_result(task1) self.assertTrue(isinstance(result, list)) self.assertIn('a 1', result) self.assertIn('b 2', result) self.assertIn('c 3', result) self.assertEqual(1, len(tasks)) self.assertEqual(states.SUCCESS, task1.state)
def test_with_items_results_one_item_as_list(self): wb_service.create_workbook_v2(WORKBOOK) # Start workflow. wf_ex = self.engine.start_workflow('wb1.with_items', WORKFLOW_INPUT_ONE_ITEM) self._await( lambda: self.is_execution_success(wf_ex.id), ) # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) tasks = wf_ex.task_executions task1 = self._assert_single_item(tasks, name='task1') result = data_flow.get_task_execution_result(task1) self.assertTrue(isinstance(result, list)) self.assertIn('Guy', result) published = task1.published self.assertIn(published['result'], ['Guy']) self.assertEqual(1, len(tasks)) self.assertEqual(states.SUCCESS, task1.state)
def test_resume_two_branches(self): wb_service.create_workbook_v2(WORKBOOK_TWO_BRANCHES) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1', {}) self.await_workflow_paused(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.PAUSED, wf_ex.state) self.assertEqual(3, len(task_execs)) wf_ex = self.engine.resume_workflow(wf_ex.id) self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.SUCCESS, wf_ex.state) # We can see 3 tasks in execution. self.assertEqual(3, len(task_execs))
def test_with_items_action_context(self): wb_service.create_workbook_v2(WORKBOOK_ACTION_CONTEXT) # Start workflow. wf_ex = self.engine.start_workflow( 'wb1.wf1_with_items', WF_INPUT_URLS ) wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] act_exs = task_ex.executions self.engine.on_action_complete(act_exs[0].id, wf_utils.Result("Ivan")) self.engine.on_action_complete(act_exs[1].id, wf_utils.Result("John")) self.engine.on_action_complete( act_exs[2].id, wf_utils.Result("Mistral") ) self._await( lambda: self.is_execution_success(wf_ex.id), ) # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = db_api.get_task_execution(task_ex.id) result = data_flow.get_task_execution_result(task_ex) self.assertTrue(isinstance(result, list)) self.assertIn('John', result) self.assertIn('Ivan', result) self.assertIn('Mistral', result) self.assertEqual(states.SUCCESS, task_ex.state)
def test_retry_policy_one_line(self): retry_wb = """--- version: '2.0' name: wb workflows: wf1: type: direct tasks: task1: action: std.fail retry: count=3 delay=1 """ wb_service.create_workbook_v2(retry_wb) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1', {}) # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self._await(lambda: self.is_task_error(task_ex.id)) self._await(lambda: self.is_execution_error(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self.assertEqual( 2, task_ex.runtime_context['retry_task_policy']['retry_no'] )
def test_with_items_static_var(self): wb_service.create_workbook_v2(WORKBOOK_WITH_STATIC_VAR) wf_input = copy.copy(WORKFLOW_INPUT) wf_input.update({'greeting': 'Hello'}) # Start workflow. wf_ex = self.engine.start_workflow('wb1.with_items', wf_input) self._await( lambda: self.is_execution_success(wf_ex.id), ) # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) tasks = wf_ex.task_executions task1 = self._assert_single_item(tasks, name='task1') result = data_flow.get_task_execution_result(task1) self.assertTrue(isinstance(result, list)) self.assertIn('Hello, John!', result) self.assertIn('Hello, Ivan!', result) self.assertIn('Hello, Mistral!', result) self.assertEqual(1, len(tasks)) self.assertEqual(states.SUCCESS, task1.state)
def test_with_items_simple(self): wb_service.create_workbook_v2(WORKBOOK) # Start workflow. wf_ex = self.engine.start_workflow('wb1.with_items', WORKFLOW_INPUT) self._await( lambda: self.is_execution_success(wf_ex.id), ) # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) tasks = wf_ex.task_executions task1 = self._assert_single_item(tasks, name='task1') with_items_context = task1.runtime_context['with_items'] self.assertEqual(3, with_items_context['count']) # Since we know that we can receive results in random order, # check is not depend on order of items. result = data_flow.get_task_execution_result(task1) self.assertTrue(isinstance(result, list)) self.assertIn('John', result) self.assertIn('Ivan', result) self.assertIn('Mistral', result) published = task1.published self.assertIn(published['result'], ['John', 'Ivan', 'Mistral']) self.assertEqual(1, len(tasks)) self.assertEqual(states.SUCCESS, task1.state)
def test_pause_before_policy(self): wb_service.create_workbook_v2(PAUSE_BEFORE_WB) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1', {}) wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = self._assert_single_item( wf_ex.task_executions, name='task1' ) self.assertEqual(states.IDLE, task_ex.state) self._await(lambda: self.is_execution_paused(wf_ex.id)) self._sleep(1) self.engine.resume_workflow(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self._assert_single_item(wf_ex.task_executions, name='task1') self._await(lambda: self.is_execution_success(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = self._assert_single_item( wf_ex.task_executions, name='task1' ) next_task_ex = self._assert_single_item( wf_ex.task_executions, name='task2' ) self.assertEqual(states.SUCCESS, task_ex.state) self.assertEqual(states.SUCCESS, next_task_ex.state)
def test_retry_policy(self): wb_service.create_workbook_v2(RETRY_WB) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1', {}) # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self.assertEqual(states.RUNNING, task_ex.state) self.assertDictEqual({}, task_ex.runtime_context) self._await( lambda: self.is_task_delayed(task_ex.id), delay=0.5 ) self._await(lambda: self.is_task_error(task_ex.id)) self._await(lambda: self.is_execution_error(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self.assertEqual( 2, task_ex.runtime_context["retry_task_policy"]["retry_no"] )
def test_action_context(self): wb_service.create_workbook_v2(WORKBOOK) wf_ex = self.engine.start_workflow('wb.wf1', {}) self._await(lambda: self.is_execution_success(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.SUCCESS, wf_ex.state) task_ex = self._assert_single_item(wf_ex.task_executions, name='task1') action_ex = self._assert_single_item(task_ex.executions) headers = { 'Mistral-Workflow-Name': wf_ex.workflow_name, 'Mistral-Workflow-Execution-Id': wf_ex.id, 'Mistral-Task-Id': task_ex.id, 'Mistral-Action-Execution-Id': action_ex.id } requests.request.assert_called_with( 'GET', 'https://wiki.openstack.org/wiki/mistral', params=None, data=None, headers=headers, cookies=None, auth=None, timeout=None, allow_redirects=None, proxies=None, verify=None )
def test_publish_failure(self): wb_service.create_workbook_v2(SIMPLE_WORKBOOK) # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb1.wf1') self.await_workflow_error(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.ERROR, wf_ex.state) self.assertEqual(1, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') # Task 1 should have failed. self.assertEqual(states.ERROR, task_1_ex.state) self.assertIn('Can not evaluate YAQL expression', task_1_ex.state_info) # Action execution of task 1 should have succeeded. task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) self.assertEqual(1, len(task_1_action_exs)) self.assertEqual(states.SUCCESS, task_1_action_exs[0].state)
def test_retry_policy_never_happen(self): retry_wb = """--- version: '2.0' name: wb workflows: wf1: tasks: task1: action: std.echo output="hello" retry: count: 3 delay: 1 """ wb_service.create_workbook_v2(retry_wb) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1', {}) # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self._await(lambda: self.is_task_success(task_ex.id)) self._await(lambda: self.is_execution_success(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) task_ex = wf_ex.task_executions[0] self.assertEqual( {}, task_ex.runtime_context["retry_task_policy"] )
def test_retry_join_task_after_idle_task(self): retry_wb = """--- version: '2.0' name: wb workflows: wf1: task-defaults: retry: count: 1 delay: 0 tasks: task1: on-success: join_task task2: action: std.fail on-success: task3 task3: on-success: join_task join_task: join: all """ wb_service.create_workbook_v2(retry_wb) # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1') self.await_workflow_error(wf_ex.id) with db_api.transaction(): # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) tasks = wf_ex.task_executions self._assert_single_item(tasks, name="task2", state=states.ERROR) self._assert_single_item(tasks, name="join_task", state=states.ERROR)
def test_resume_reverse(self): wb_service.create_workbook_v2(RESUME_WORKBOOK_REVERSE) # Start workflow. wf_ex = self.engine.start_workflow( 'resume_reverse.wf', {}, task_name='task2' ) # Note: We need to reread execution to access related tasks. self.engine.pause_workflow(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.PAUSED, wf_ex.state) self.assertEqual(1, len(wf_ex.task_executions)) self.engine.resume_workflow(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self._await(lambda: self.is_execution_success(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.SUCCESS, wf_ex.state) self.assertEqual(2, len(wf_ex.task_executions))
def test_cancel_with_items_concurrency(self): wb_def = """ version: '2.0' name: wb1 workflows: wf1: tasks: t1: with-items: i in <% list(range(0, 4)) %> action: std.async_noop concurrency: 2 on-success: - t2 t2: action: std.echo output="Task 2" """ wb_service.create_workbook_v2(wb_def) wf1_ex = self.engine.start_workflow('wb1.wf1') self.await_workflow_state(wf1_ex.id, states.RUNNING) with db_api.transaction(): wf1_execs = db_api.get_workflow_executions() wf1_ex = self._assert_single_item(wf1_execs, name='wb1.wf1') wf1_t1_ex = self._assert_single_item( wf1_ex.task_executions, name='t1' ) wf1_t1_action_exs = db_api.get_action_executions( task_execution_id=wf1_t1_ex.id ) self.assertEqual(2, len(wf1_t1_action_exs)) self.assertEqual(states.RUNNING, wf1_t1_action_exs[0].state) self.assertEqual(states.RUNNING, wf1_t1_action_exs[1].state) # Cancel action execution for task. for wf1_t1_action_ex in wf1_t1_action_exs: self.engine.on_action_complete( wf1_t1_action_ex.id, ml_actions.Result(cancel=True) ) self.await_task_cancelled(wf1_t1_ex.id) self.await_workflow_cancelled(wf1_ex.id) wf1_t1_action_exs = db_api.get_action_executions( task_execution_id=wf1_t1_ex.id ) self.assertEqual(2, len(wf1_t1_action_exs)) self.assertEqual(states.CANCELLED, wf1_t1_action_exs[0].state) self.assertEqual(states.CANCELLED, wf1_t1_action_exs[1].state)
def test_workbook_notify(self): wb_text = """ version: '2.0' name: wb workflows: wf1: tasks: t1: workflow: wf2 on-success: - t2 t2: action: std.noop wf2: tasks: t1: action: std.noop """ wb_svc.create_workbook_v2(wb_text) notify_options = [{'type': 'webhook'}] params = {'notify': notify_options} wf1_ex = self.engine.start_workflow('wb.wf1', '', **params) self.await_workflow_success(wf1_ex.id) with db_api.transaction(): wf1_ex = db_api.get_workflow_execution(wf1_ex.id) wf1_task_exs = wf1_ex.task_executions wf1_t1_ex = self._assert_single_item(wf1_task_exs, name='t1') wf1_t2_ex = self._assert_single_item(wf1_task_exs, name='t2') wf1_t1_act_exs = db_api.get_workflow_executions( task_execution_id=wf1_t1_ex.id) wf2_ex = wf1_t1_act_exs[0] wf2_task_exs = wf2_ex.task_executions wf2_t1_ex = self._assert_single_item(wf2_task_exs, name='t1') self.assertEqual(states.SUCCESS, wf1_ex.state) self.assertIsNone(wf1_ex.state_info) self.assertEqual(2, len(wf1_task_exs)) self.assertEqual(states.SUCCESS, wf1_t1_ex.state) self.assertIsNone(wf1_t1_ex.state_info) self.assertEqual(states.SUCCESS, wf1_t2_ex.state) self.assertIsNone(wf1_t2_ex.state_info) self.assertEqual(1, len(wf1_t1_act_exs)) self.assertEqual(states.SUCCESS, wf2_ex.state) self.assertIsNone(wf2_ex.state_info) self.assertEqual(1, len(wf2_task_exs)) self.assertEqual(states.SUCCESS, wf2_t1_ex.state) self.assertIsNone(wf2_t1_ex.state_info) expected_order = [(wf1_ex.id, events.WORKFLOW_LAUNCHED), (wf1_t1_ex.id, events.TASK_LAUNCHED), (wf2_ex.id, events.WORKFLOW_LAUNCHED), (wf2_t1_ex.id, events.TASK_LAUNCHED), (wf2_t1_ex.id, events.TASK_SUCCEEDED), (wf2_ex.id, events.WORKFLOW_SUCCEEDED), (wf1_t1_ex.id, events.TASK_SUCCEEDED), (wf1_t2_ex.id, events.TASK_LAUNCHED), (wf1_t2_ex.id, events.TASK_SUCCEEDED), (wf1_ex.id, events.WORKFLOW_SUCCEEDED)] self.assertTrue(self.publishers['wbhk'].publish.called) self.assertListEqual(expected_order, EVENT_LOGS)
def test_rerun_join_with_branch_errors(self): wb_service.create_workbook_v2(JOIN_WORKBOOK) # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb1.wf1') with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') self.await_task_error(task_1_ex.id) self.await_task_error(task_2_ex.id) self.await_workflow_error(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(2, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') self.assertEqual(states.ERROR, task_1_ex.state) self.assertIsNotNone(task_1_ex.state_info) task_2_ex = self._assert_single_item(task_execs, name='t2') self.assertEqual(states.ERROR, task_2_ex.state) self.assertIsNotNone(task_2_ex.state_info) # Resume workflow and re-run failed task. wf_ex = self.engine.rerun_workflow(task_1_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions # Wait for the task to succeed. task_1_ex = self._assert_single_item(task_execs, name='t1') self.await_task_success(task_1_ex.id) self.await_workflow_error(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(3, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') task_3_ex = self._assert_single_item(task_execs, name='t3') self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertEqual(states.ERROR, task_2_ex.state) self.assertEqual(states.ERROR, task_3_ex.state) # Resume workflow and re-run failed task. wf_ex = self.engine.rerun_workflow(task_2_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) # Join now should finally complete. self.await_task_success(task_3_ex.id) # Wait for the workflow to succeed. self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.SUCCESS, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertEqual(3, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') task_3_ex = self._assert_single_item(task_execs, name='t3') # Check action executions of task 1. self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertIsNone(task_1_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) self.assertEqual(2, len(task_1_action_exs)) # Check there is exactly 1 action in Success and 1 in error state. # Order doesn't matter. self._assert_single_item(task_1_action_exs, state=states.SUCCESS) self._assert_single_item(task_1_action_exs, state=states.ERROR) # Check action executions of task 2. self.assertEqual(states.SUCCESS, task_2_ex.state) self.assertIsNone(task_2_ex.state_info) task_2_action_exs = db_api.get_action_executions( task_execution_id=task_2_ex.id ) self.assertEqual(2, len(task_2_action_exs)) # Check there is exactly 1 action in Success and 1 in error state. # Order doesn't matter. self._assert_single_item(task_2_action_exs, state=states.SUCCESS) self._assert_single_item(task_2_action_exs, state=states.ERROR) # Check action executions of task 3. self.assertEqual(states.SUCCESS, task_3_ex.state) task_3_action_exs = db_api.get_action_executions( task_execution_id=task_execs[2].id ) self.assertEqual(1, len(task_3_action_exs)) self.assertEqual(states.SUCCESS, task_3_action_exs[0].state)
def test_rerun_with_items_diff_env_vars(self): wb_service.create_workbook_v2(WITH_ITEMS_WORKBOOK_DIFF_ENV_VAR) # Initial environment variables for the workflow execution. env = {'var1': 'fee fi fo fum'} # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb3.wf1', env=env) self.await_workflow_error(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(1, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') self.assertEqual(states.ERROR, task_1_ex.state) self.assertIsNotNone(task_1_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) self.assertEqual(3, len(task_1_action_exs)) # Update env in workflow execution with the following. updated_env = {'var1': 'foobar'} # Resume workflow and re-run failed task. self.engine.rerun_workflow( task_1_ex.id, reset=False, env=updated_env ) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.SUCCESS, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertEqual(2, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') # Check action executions of task 1. self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertIsNone(task_1_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) expected_inputs = [ 'Task 1.0 [%s]' % env['var1'], # Task 1 item 0 (error). 'Task 1.1 [%s]' % env['var1'], # Task 1 item 1. 'Task 1.2 [%s]' % env['var1'], # Task 1 item 2 (error). 'Task 1.0 [%s]' % updated_env['var1'], # Task 1 item 0 (rerun). 'Task 1.2 [%s]' % updated_env['var1'] # Task 1 item 2 (rerun). ] # Assert that every expected input is in actual task input. for action_ex in task_1_action_exs: self.assertIn(action_ex.input['output'], expected_inputs) # Assert that there was same number of unique inputs as action execs. self.assertEqual( len(task_1_action_exs), len(set( [action_ex.input['output'] for action_ex in task_1_action_exs] )) ) # Check action executions of task 2. self.assertEqual(states.SUCCESS, task_2_ex.state) task_2_action_exs = db_api.get_action_executions( task_execution_id=task_2_ex.id ) self.assertEqual(1, len(task_2_action_exs))
def test_rerun_with_items_concurrency(self): wb_service.create_workbook_v2(WITH_ITEMS_WORKBOOK_CONCURRENCY) # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb3.wf1') self.await_workflow_error(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(1, len(wf_ex.task_executions)) task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1') self.assertEqual(states.ERROR, task_1_ex.state) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) self.assertEqual(4, len(task_1_action_exs)) # Resume workflow and re-run failed task. self.engine.rerun_workflow(task_1_ex.id, reset=False) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.await_workflow_success(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.SUCCESS, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertEqual(2, len(wf_ex.task_executions)) task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1') task_2_ex = self._assert_single_item(wf_ex.task_executions, name='t2') # Check action executions of task 1. self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertIsNone(task_1_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) # The action executions that succeeded should not re-run. self.assertEqual(6, len(task_1_action_exs)) self.assertListEqual(['Task 1.0', 'Task 1.1', 'Task 1.2', 'Task 1.3'], task_1_ex.published.get('v1')) # Check action executions of task 2. self.assertEqual(states.SUCCESS, task_2_ex.state) task_2_action_exs = db_api.get_action_executions( task_execution_id=task_2_ex.id ) self.assertEqual(1, len(task_2_action_exs))
def test_rerun_diff_env_vars(self): wb_service.create_workbook_v2(SIMPLE_WORKBOOK_DIFF_ENV_VAR) # Initial environment variables for the workflow execution. env = { 'var1': 'fee fi fo fum', 'var2': 'mirror mirror', 'var3': 'heigh-ho heigh-ho' } # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb1.wf1', env=env) self.await_workflow_error(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(3, len(task_execs)) self.assertDictEqual(env, wf_ex.params['env']) self.assertDictEqual(env, wf_ex.context['__env']) task_10_ex = self._assert_single_item(task_execs, name='t10') task_21_ex = self._assert_single_item(task_execs, name='t21') task_30_ex = self._assert_single_item(task_execs, name='t30') self.assertEqual(states.SUCCESS, task_10_ex.state) self.assertEqual(states.ERROR, task_21_ex.state) self.assertIsNotNone(task_21_ex.state_info) self.assertEqual(states.ERROR, task_30_ex.state) # Update env in workflow execution with the following. updated_env = { 'var1': 'Task 21', 'var2': 'Task 22', 'var3': 'Task 30' } # Resume workflow and re-run failed task. wf_ex = self.engine.rerun_workflow(task_21_ex.id, env=updated_env) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertDictEqual(updated_env, wf_ex.params['env']) self.assertDictEqual(updated_env, wf_ex.context['__env']) # Await t30 success. self.await_task_success(task_30_ex.id) # Wait for the workflow to succeed. self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.SUCCESS, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertEqual(4, len(task_execs)) task_10_ex = self._assert_single_item(task_execs, name='t10') task_21_ex = self._assert_single_item(task_execs, name='t21') task_22_ex = self._assert_single_item(task_execs, name='t22') task_30_ex = self._assert_single_item(task_execs, name='t30') # Check action executions of task 10. self.assertEqual(states.SUCCESS, task_10_ex.state) task_10_action_exs = db_api.get_action_executions( task_execution_id=task_10_ex.id ) self.assertEqual(1, len(task_10_action_exs)) self.assertEqual(states.SUCCESS, task_10_action_exs[0].state) self.assertDictEqual( {'output': 'Task 10'}, task_10_action_exs[0].input ) # Check action executions of task 21. self.assertEqual(states.SUCCESS, task_21_ex.state) self.assertIsNone(task_21_ex.state_info) task_21_action_exs = db_api.get_action_executions( task_execution_id=task_21_ex.id ) self.assertEqual(2, len(task_21_action_exs)) # Check there is exactly 1 action in Success and 1 in error state. # Order doesn't matter. task_21_action_exs_1 = self._assert_single_item( task_21_action_exs, state=states.ERROR ) task_21_action_exs_2 = self._assert_single_item( task_21_action_exs, state=states.SUCCESS ) self.assertDictEqual( {'output': env['var1']}, task_21_action_exs_1.input ) self.assertDictEqual( {'output': updated_env['var1']}, task_21_action_exs_2.input ) # Check action executions of task 22. self.assertEqual(states.SUCCESS, task_22_ex.state) task_22_action_exs = db_api.get_action_executions( task_execution_id=task_22_ex.id ) self.assertEqual(1, len(task_22_action_exs)) self.assertEqual(states.SUCCESS, task_22_action_exs[0].state) self.assertDictEqual( {'output': updated_env['var2']}, task_22_action_exs[0].input ) # Check action executions of task 30. self.assertEqual(states.SUCCESS, task_30_ex.state) task_30_action_exs = db_api.get_action_executions( task_execution_id=task_30_ex.id ) self.assertEqual(1, len(task_30_action_exs)) self.assertEqual(states.SUCCESS, task_30_action_exs[0].state) self.assertDictEqual( {'output': updated_env['var3']}, task_30_action_exs[0].input )
def test_resume_diff_env_vars(self): wb_service.create_workbook_v2(RESUME_WORKBOOK_DIFF_ENV_VAR) # Initial environment variables for the workflow execution. env = {'var1': 'fee fi fo fum', 'var2': 'foobar'} # Start workflow. wf_ex = self.engine.start_workflow('wb.wf1', env=env) self.await_workflow_paused(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions task_1_ex = self._assert_single_item(task_execs, name='task1') task_2_ex = self._assert_single_item(task_execs, name='task2') self.assertEqual(states.PAUSED, wf_ex.state) self.assertEqual(2, len(task_execs)) self.assertDictEqual(env, wf_ex.params['env']) self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertEqual(states.IDLE, task_2_ex.state) # Update env in workflow execution with the following. updated_env = {'var1': 'Task 2', 'var2': 'Task 3'} # Update the env variables and resume workflow. self.engine.resume_workflow(wf_ex.id, env=updated_env) self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertDictEqual(updated_env, wf_ex.params['env']) self.assertEqual(3, len(task_execs)) # Check result of task2. task_2_ex = self._assert_single_item(task_execs, name='task2') self.assertEqual(states.SUCCESS, task_2_ex.state) # Re-read task execution, otherwise lazy loading of action executions # may not work. with db_api.transaction(): task_2_ex = db_api.get_task_execution(task_2_ex.id) task_2_result = data_flow.get_task_execution_result(task_2_ex) self.assertEqual(updated_env['var1'], task_2_result) # Check result of task3. task_3_ex = self._assert_single_item(task_execs, name='task3') self.assertEqual(states.SUCCESS, task_3_ex.state) # Re-read task execution, otherwise lazy loading of action executions # may not work. with db_api.transaction(): task_3_ex = db_api.get_task_execution(task_3_ex.id) task_3_result = data_flow.get_task_execution_result(task_3_ex) self.assertEqual(updated_env['var2'], task_3_result)
def setUp(self): super(EnvironmentTest, self).setUp() wb_service.create_workbook_v2(WORKBOOK)
def test_with_items_and_adhoc_action(self, mock_http_action): mock_http_action.return_value = '' wb_text = """--- version: "2.0" name: test actions: http: input: - url: http://www.example.com - method: GET - timeout: 10 output: <% $.content %> base: std.http base-input: url: <% $.url %> method: <% $.method %> timeout: <% $.timeout %> workflows: with_items_default_bug: description: Re-create the with-items bug with default values type: direct tasks: get_pages: with-items: page in <% range(0, 1) %> action: test.http input: url: http://www.example.com method: GET on-success: - well_done well_done: action: std.echo output="Well done" """ wb_service.create_workbook_v2(wb_text) # Start workflow. wf_ex = self.engine.start_workflow('test.with_items_default_bug', {}) self.await_workflow_success(wf_ex.id) with db_api.transaction(): # Note: We need to reread execution to access related tasks. wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions task1_ex = self._assert_single_item(task_execs, name='get_pages') task2_ex = self._assert_single_item(task_execs, name='well_done') self.assertEqual(2, len(task_execs)) self.assertEqual(states.SUCCESS, task1_ex.state) self.assertEqual(states.SUCCESS, task2_ex.state) self.assertEqual(1, mock_http_action.call_count)
def test_nested_wf_max_depth(self): wb_text = """--- version: '2.0' name: wb workflows: parent_wf: tasks: task1: action: std.noop on-success: task2 task2: workflow: sub_wf on-success: task3 task3: action: std.fail sub_wf: tasks: task1: action: std.noop on-success: task2 task2: action: std.fail """ wb_service.create_workbook_v2(wb_text) wf_ex = self.engine.start_workflow('wb.parent_wf') self.await_workflow_error(wf_ex.id) resp = self.app.get('/v2/executions/%s/report?max_depth=0' % wf_ex.id) self.assertEqual(200, resp.status_int) # Now let's verify the response structure self.assertIn('root_workflow_execution', resp.json) root_wf_ex = resp.json['root_workflow_execution'] self.assertIsInstance(root_wf_ex, dict) self.assertEqual('wb.parent_wf', root_wf_ex['name']) self.assertEqual(states.ERROR, root_wf_ex['state']) self.assertGreater(len(root_wf_ex['state_info']), 0) tasks = root_wf_ex['task_executions'] self.assertIsInstance(tasks, list) self.assertEqual(2, len(tasks)) # Verify task1 info. task1 = self._assert_single_item(tasks, name='task1') self.assertEqual(states.SUCCESS, task1['state']) self.assertEqual(0, len(task1['workflow_executions'])) self.assertEqual(1, len(task1['action_executions'])) task1_action = task1['action_executions'][0] self.assertEqual(states.SUCCESS, task1_action['state']) self.assertEqual('std.noop', task1_action['name']) # Verify task2 info. task2 = self._assert_single_item(tasks, name='task2') self.assertEqual(states.ERROR, task2['state']) self.assertEqual(0, len(task2['action_executions'])) self.assertEqual(1, len(task2['workflow_executions'])) sub_wf_entry = task2['workflow_executions'][0] self.assertEqual(states.ERROR, sub_wf_entry['state']) # We still must have an entry for the subworkflow itself # but it must not have info about task executions because # we've now limited max depth. self.assertNotIn('task_executions', sub_wf_entry) # Verify statistics. stat = resp.json['statistics'] self.assertEqual(1, stat['error_tasks_count']) self.assertEqual(0, stat['idle_tasks_count']) self.assertEqual(0, stat['paused_tasks_count']) self.assertEqual(0, stat['running_tasks_count']) self.assertEqual(1, stat['success_tasks_count']) self.assertEqual(2, stat['total_tasks_count'])
def test_tasks_function(self): wb_service.create_workbook_v2(WORKBOOK_WITH_EXPRESSIONS) # Start helping workflow executions. wf1_ex = self.engine.start_workflow('wb.wf1_top_lvl', '', {}) wf2_ex = self.engine.start_workflow('wb.wf2_top_lvl', '', {}) wf3_ex = self.engine.start_workflow('wb.wf3_top_lvl', '', {}) wf4_ex = self.engine.start_workflow('wb.wf4_top_lvl', '', {}) wf5_ex = self.engine.start_workflow('wb.wf5_top_lvl', '', {}) self.await_workflow_success(wf1_ex.id) self.await_workflow_success(wf2_ex.id) self.await_workflow_error(wf3_ex.id) self.await_workflow_error(wf4_ex.id) self.await_workflow_error(wf5_ex.id) # Start test workflow execution execution = self.engine.start_workflow( 'wb.test_tasks_function', '', { "wf1_wx_id": wf1_ex.id, "wf2_wx_id": wf2_ex.id, "wf3_wx_id": wf3_ex.id, "wf4_wx_id": wf4_ex.id, "wf5_wx_id": wf5_ex.id }) self.await_workflow_success(execution.id) with db_api.transaction(): execution = db_api.get_workflow_execution(execution.id) task_executions = execution.task_executions self.assertEqual(states.SUCCESS, execution.state) self.assertEqual(1, len(task_executions)) main_task = task_executions[0] self._assert_published_tasks(main_task, "all_tasks_yaql", 22) self._assert_published_tasks(main_task, "all_tasks_jinja", 22) self._assert_published_tasks( main_task, "wf1_tasks_yaql", 2, ["top_lvl_wf1_task_1", "top_lvl_wf1_task_2"]) self._assert_published_tasks( main_task, "wf1_tasks_jinja", 2, ["top_lvl_wf1_task_1", "top_lvl_wf1_task_2"]) self._assert_published_tasks( main_task, "wf1_recursive_tasks_yaql", 8, [ 'top_lvl_wf1_task_1', 'top_lvl_wf1_task_2', 'second_lvl_wf1_task_3', 'second_lvl_wf1_task_1', 'second_lvl_wf1_task_2', 'third_lvl_wf1_task_3', 'third_lvl_wf1_task_1', 'third_lvl_wf1_task_2_fail' ]) self._assert_published_tasks( main_task, "wf1_recursive_tasks_jinja", 8, [ 'top_lvl_wf1_task_1', 'top_lvl_wf1_task_2', 'second_lvl_wf1_task_3', 'second_lvl_wf1_task_1', 'second_lvl_wf1_task_2', 'third_lvl_wf1_task_3', 'third_lvl_wf1_task_1', 'third_lvl_wf1_task_2_fail' ]) self._assert_published_tasks( main_task, "wf1_recursive_error_tasks_yaql", 2, ['second_lvl_wf1_task_1', 'third_lvl_wf1_task_2_fail']) self._assert_published_tasks( main_task, "wf1_recursive_error_tasks_jinja", 2, ['second_lvl_wf1_task_1', 'third_lvl_wf1_task_2_fail']) self._assert_published_tasks(main_task, "wf1_not_recursive_error_tasks_yaql", 0) self._assert_published_tasks(main_task, "wf1_not_recursive_error_tasks_jinja", 0) self._assert_published_tasks( main_task, "wf1_recursive_success_flat_tasks_yaql", 5, [ 'top_lvl_wf1_task_2', 'second_lvl_wf1_task_3', 'second_lvl_wf1_task_2', 'third_lvl_wf1_task_3', 'third_lvl_wf1_task_1' ]) self._assert_published_tasks( main_task, "wf1_recursive_success_flat_tasks_jinja", 5, [ 'top_lvl_wf1_task_2', 'second_lvl_wf1_task_3', 'second_lvl_wf1_task_2', 'third_lvl_wf1_task_3', 'third_lvl_wf1_task_1' ]) self._assert_published_tasks( main_task, "wf2_recursive_tasks_yaql", 2, ['top_lvl_wf2_task_2', 'top_lvl_wf2_task_1']) self._assert_published_tasks( main_task, "wf2_recursive_tasks_jinja", 2, ['top_lvl_wf2_task_2', 'top_lvl_wf2_task_1']) self._assert_published_tasks( main_task, "wf3_recursive_error_tasks_yaql", 4, [ 'top_lvl_wf3_task_1_fail', 'top_lvl_wf3_task_2_fail', 'second_lvl_wf3_task_1_fail', 'third_lvl_wf3_task_3_fail' ]) self._assert_published_tasks( main_task, "wf3_recursive_error_tasks_jinja", 4, [ 'top_lvl_wf3_task_1_fail', 'top_lvl_wf3_task_2_fail', 'second_lvl_wf3_task_1_fail', 'third_lvl_wf3_task_3_fail' ]) self._assert_published_tasks( main_task, "wf3_recursive_error_flat_tasks_yaql", 2, ['top_lvl_wf3_task_2_fail', 'third_lvl_wf3_task_3_fail']) self._assert_published_tasks( main_task, "wf3_recursive_error_flat_tasks_jinja", 2, ['top_lvl_wf3_task_2_fail', 'third_lvl_wf3_task_3_fail']) self._assert_published_tasks(main_task, "wf4_recursive_error_flat_tasks_yaql", 1, ['top_lvl_wf4_task_1']) self._assert_published_tasks(main_task, "wf4_recursive_error_flat_tasks_jinja", 1, ['top_lvl_wf4_task_1']) self._assert_published_tasks(main_task, "wf5_recursive_error_flat_tasks_yaql", 1, ['top_lvl_wf5_task_1']) self._assert_published_tasks(main_task, "wf5_recursive_error_flat_tasks_jinja", 1, ['top_lvl_wf5_task_1'])
def setUp(self): super(AdhocActionsTest, self).setUp() wb_service.create_workbook_v2(WORKBOOK)
def setUp(self): super(SimpleEngineCommandsTest, self).setUp() wb_service.create_workbook_v2(WORKBOOK1)
def setUp(self): super(ReverseWorkflowEngineTest, self).setUp() wb_service.create_workbook_v2(WORKBOOK)
def test_rerun_cancelled_task(self): wb_def = """ version: '2.0' name: wb1 workflows: wf1: type: reverse tasks: t1: action: std.async_noop t2: action: std.echo output="Task 2" requires: - t1 t3: action: std.echo output="Task 3" requires: - t2 """ wb_service.create_workbook_v2(wb_def) wf1_ex = self.engine.start_workflow('wb1.wf1', {}, task_name='t3') self.await_workflow_state(wf1_ex.id, states.RUNNING) with db_api.transaction(): wf1_execs = db_api.get_workflow_executions() wf1_ex = self._assert_single_item(wf1_execs, name='wb1.wf1') wf1_t1_ex = self._assert_single_item(wf1_ex.task_executions, name='t1') wf1_t1_action_exs = db_api.get_action_executions( task_execution_id=wf1_t1_ex.id) self.assertEqual(1, len(wf1_t1_action_exs)) self.assertEqual(states.RUNNING, wf1_t1_action_exs[0].state) # Cancel action execution for task. self.engine.on_action_complete(wf1_t1_action_exs[0].id, wf_utils.Result(cancel=True)) self.await_workflow_cancelled(wf1_ex.id) with db_api.transaction(): wf1_ex = db_api.get_workflow_execution(wf1_ex.id) wf1_t1_ex = self._assert_single_item(wf1_ex.task_executions, name='t1') self.await_task_cancelled(wf1_t1_ex.id) with db_api.transaction(): wf1_ex = db_api.get_workflow_execution(wf1_ex.id) wf1_t1_ex = self._assert_single_item(wf1_ex.task_executions, name='t1') self.assertEqual(states.CANCELLED, wf1_ex.state) self.assertEqual("Cancelled tasks: t1", wf1_ex.state_info) self.assertEqual(1, len(wf1_ex.task_executions)) self.assertEqual(states.CANCELLED, wf1_t1_ex.state) self.assertIsNone(wf1_t1_ex.state_info) # Resume workflow and re-run cancelled task. self.engine.rerun_workflow(wf1_t1_ex.id) with db_api.transaction(): wf1_ex = db_api.get_workflow_execution(wf1_ex.id) wf1_task_execs = wf1_ex.task_executions self.assertEqual(states.RUNNING, wf1_ex.state) self.assertIsNone(wf1_ex.state_info) # Mark async action execution complete. wf1_t1_ex = self._assert_single_item(wf1_task_execs, name='t1') wf1_t1_action_exs = db_api.get_action_executions( task_execution_id=wf1_t1_ex.id) self.assertEqual(states.RUNNING, wf1_t1_ex.state) self.assertEqual(2, len(wf1_t1_action_exs)) # Check there is exactly 1 action in Running and 1 in Cancelled state. # Order doesn't matter. self._assert_single_item(wf1_t1_action_exs, state=states.RUNNING) self._assert_single_item(wf1_t1_action_exs, state=states.CANCELLED) self.engine.on_action_complete(wf1_t1_action_exs[1].id, wf_utils.Result(data={'foo': 'bar'})) # Wait for the workflow to succeed. self.await_workflow_success(wf1_ex.id) with db_api.transaction(): wf1_ex = db_api.get_workflow_execution(wf1_ex.id) wf1_task_execs = wf1_ex.task_executions self.assertEqual(states.SUCCESS, wf1_ex.state) self.assertIsNone(wf1_ex.state_info) self.assertEqual(3, len(wf1_task_execs)) wf1_t1_ex = self._assert_single_item(wf1_task_execs, name='t1') wf1_t2_ex = self._assert_single_item(wf1_task_execs, name='t2') wf1_t3_ex = self._assert_single_item(wf1_task_execs, name='t3') # Check action executions of task 1. self.assertEqual(states.SUCCESS, wf1_t1_ex.state) self.assertIsNone(wf1_t2_ex.state_info) wf1_t1_action_exs = db_api.get_action_executions( task_execution_id=wf1_t1_ex.id) self.assertEqual(2, len(wf1_t1_action_exs)) # Check there is exactly 1 action in Success and 1 in Cancelled state. # Order doesn't matter. self._assert_single_item(wf1_t1_action_exs, state=states.SUCCESS) self._assert_single_item(wf1_t1_action_exs, state=states.CANCELLED) # Check action executions of task 2. self.assertEqual(states.SUCCESS, wf1_t2_ex.state) wf1_t2_action_exs = db_api.get_action_executions( task_execution_id=wf1_t2_ex.id) self.assertEqual(1, len(wf1_t2_action_exs)) self.assertEqual(states.SUCCESS, wf1_t2_action_exs[0].state) # Check action executions of task 3. self.assertEqual(states.SUCCESS, wf1_t3_ex.state) wf1_t3_action_exs = db_api.get_action_executions( task_execution_id=wf1_t3_ex.id) self.assertEqual(1, len(wf1_t3_action_exs)) self.assertEqual(states.SUCCESS, wf1_t3_action_exs[0].state)
def setUp(self): super(SubworkflowsTest, self).setUp() wb_service.create_workbook_v2(WB1) wb_service.create_workbook_v2(WB2) wb_service.create_workbook_v2(WB3) wb_service.create_workbook_v2(WB4) wb_service.create_workbook_v2(WB5) wb_service.create_workbook_v2(WB6)
def setUp(self): super(SimpleEngineWorkflowLevelCmdsWithMsgTest, self).setUp() wb_service.create_workbook_v2(WORKBOOK5)
def test_fail_then_cancel_with_items_child_workflow(self): workbook = """ version: '2.0' name: wb workflows: wf: type: direct tasks: taskx: with-items: i in [1, 2] workflow: subwf subwf: type: direct tasks: task1: action: std.echo output="Echo" on-complete: - task2 task2: action: std.echo output="foo" wait-before: 1 """ wb_service.create_workbook_v2(workbook) self.engine.start_workflow('wb.wf', {}) with db_api.transaction(): wf_execs = db_api.get_workflow_executions() wf_ex = self._assert_single_item(wf_execs, name='wb.wf') task_ex = self._assert_single_item(wf_ex.task_executions, name='taskx') subwf_exs = self._assert_multiple_items(wf_execs, 2, name='wb.subwf') self.engine.stop_workflow(subwf_exs[1].id, states.ERROR, "Failed by user.") self.engine.stop_workflow(subwf_exs[0].id, states.CANCELLED, "Cancelled by user.") self.await_workflow_cancelled(subwf_exs[0].id) self.await_workflow_error(subwf_exs[1].id) self.await_task_cancelled(task_ex.id) self.await_workflow_cancelled(wf_ex.id) with db_api.transaction(): wf_execs = db_api.get_workflow_executions() wf_ex = self._assert_single_item(wf_execs, name='wb.wf') task_ex = self._assert_single_item(wf_ex.task_executions, name='taskx') subwf_exs = self._assert_multiple_items(wf_execs, 2, name='wb.subwf') self.assertEqual(states.CANCELLED, subwf_exs[0].state) self.assertEqual("Cancelled by user.", subwf_exs[0].state_info) self.assertEqual(states.ERROR, subwf_exs[1].state) self.assertEqual("Failed by user.", subwf_exs[1].state_info) self.assertEqual(states.CANCELLED, task_ex.state) self.assertIn("cancelled", task_ex.state_info) self.assertEqual(states.CANCELLED, wf_ex.state) self.assertEqual("Cancelled tasks: taskx", wf_ex.state_info)
def test_with_items_subflow_concurrency_gt_list_length(self): wb_text = """--- version: "2.0" name: wb1 workflows: main: type: direct input: - names tasks: task1: with-items: name in <% $.names %> workflow: subflow1 name=<% $.name %> concurrency: 3 subflow1: type: direct input: - name output: result: <% task(task1).result %> tasks: task1: action: std.echo output=<% $.name %> """ wb_service.create_workbook_v2(wb_text) # Start workflow. names = ["Peter", "Susan", "Edmund", "Lucy", "Aslan", "Caspian"] wf_ex = self.engine.start_workflow( 'wb1.main', wf_input={'names': names} ) self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions task_ex = self._assert_single_item( task_execs, name='task1', state=states.SUCCESS ) with db_api.transaction(): task_ex = db_api.get_task_execution(task_ex.id) task_result = data_flow.get_task_execution_result(task_ex) result = [item['result'] for item in task_result] self.assertListEqual(sorted(result), sorted(names))
def test_cancel_parent_workflow(self): workbook = """ version: '2.0' name: wb workflows: wf: type: direct tasks: taskx: workflow: subwf subwf: type: direct tasks: task1: action: std.echo output="Echo" on-complete: - task2 task2: action: std.echo output="foo" wait-before: 2 """ wb_service.create_workbook_v2(workbook) wf_ex = self.engine.start_workflow('wb.wf', {}) self.engine.stop_workflow(wf_ex.id, states.CANCELLED, "Cancelled by user.") self.await_workflow_cancelled(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions task_ex = self._assert_single_item(task_execs, name='taskx') self.await_task_cancelled(task_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions task_ex = self._assert_single_item(task_execs, name='taskx') subwf_execs = db_api.get_workflow_executions( task_execution_id=task_ex.id) self.assertEqual(states.CANCELLED, wf_ex.state) self.assertEqual("Cancelled by user.", wf_ex.state_info) self.assertEqual(states.CANCELLED, task_ex.state) self.assertEqual("Cancelled by user.", task_ex.state_info) self.assertEqual(1, len(subwf_execs)) self.assertEqual(states.CANCELLED, subwf_execs[0].state) self.assertEqual("Cancelled by user.", subwf_execs[0].state_info)
def test_cancel_child_workflow_action_execution(self): workbook = """ version: '2.0' name: wb workflows: wf: tasks: taskx: workflow: subwf subwf: tasks: task1: action: std.async_noop on-success: - task2 on-error: - task3 on-complete: - task4 task2: action: std.noop task3: action: std.noop task4: action: std.noop """ wb_service.create_workbook_v2(workbook) wf_ex = self.engine.start_workflow('wb.wf', '', {}) self.await_workflow_state(wf_ex.id, states.RUNNING) with db_api.transaction(): wf_execs = db_api.get_workflow_executions() wf_ex = self._assert_single_item(wf_execs, name='wb.wf') task_ex = self._assert_single_item(wf_ex.task_executions, name='taskx') subwf_ex = self._assert_single_item(wf_execs, name='wb.subwf') task_1_ex = self._assert_single_item(subwf_ex.task_executions, name='task1') task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id) self.assertEqual(1, len(task_1_action_exs)) self.assertEqual(states.RUNNING, task_1_action_exs[0].state) self.engine.on_action_complete(task_1_action_exs[0].id, ml_actions.Result(cancel=True)) self.await_workflow_cancelled(subwf_ex.id) self.await_task_cancelled(task_ex.id) self.await_workflow_cancelled(wf_ex.id) with db_api.transaction(): wf_execs = db_api.get_workflow_executions() wf_ex = self._assert_single_item(wf_execs, name='wb.wf') task_ex = self._assert_single_item(wf_ex.task_executions, name='taskx') subwf_ex = self._assert_single_item(wf_execs, name='wb.subwf') subwf_task_execs = subwf_ex.task_executions self.assertEqual(states.CANCELLED, subwf_ex.state) self.assertEqual("Cancelled tasks: task1", subwf_ex.state_info) self.assertEqual(1, len(subwf_task_execs)) self.assertEqual(states.CANCELLED, task_ex.state) self.assertEqual("Cancelled tasks: task1", task_ex.state_info) self.assertEqual(states.CANCELLED, wf_ex.state) self.assertEqual("Cancelled tasks: taskx", wf_ex.state_info)
def test_rerun_cancelled_subflow_task(self): wb_def = """ version: '2.0' name: wb1 workflows: wf1: type: direct tasks: t1: action: std.echo output="Task 1" on-success: - t2 t2: workflow: wf2 on-success: - t3 t3: action: std.echo output="Task 3" wf2: type: direct output: result: <% task(wf2_t1).result %> tasks: wf2_t1: action: std.async_noop """ wb_service.create_workbook_v2(wb_def) wf1_ex = self.engine.start_workflow('wb1.wf1', {}) self.await_workflow_state(wf1_ex.id, states.RUNNING) with db_api.transaction(): # Wait for task 1 to complete. wf1_execs = db_api.get_workflow_executions() wf1_ex = self._assert_single_item(wf1_execs, name='wb1.wf1') wf1_t1_ex = self._assert_single_item( wf1_ex.task_executions, name='t1' ) self.await_task_success(wf1_t1_ex.id) with db_api.transaction(): # Wait for the async task to run. wf1_execs = db_api.get_workflow_executions() wf1_ex = self._assert_single_item(wf1_execs, name='wb1.wf1') wf1_t2_ex = self._assert_single_item( wf1_ex.task_executions, name='t2' ) self.await_task_state(wf1_t2_ex.id, states.RUNNING) with db_api.transaction(): sub_wf_exs = db_api.get_workflow_executions( task_execution_id=wf1_t2_ex.id ) self.assertEqual(1, len(sub_wf_exs)) self.assertEqual(states.RUNNING, sub_wf_exs[0].state) wf2_ex = sub_wf_exs[0] wf2_t1_ex = self._assert_single_item( wf2_ex.task_executions, name='wf2_t1' ) self.await_task_state(wf2_t1_ex.id, states.RUNNING) wf2_t1_action_exs = db_api.get_action_executions( task_execution_id=wf2_t1_ex.id ) self.assertEqual(1, len(wf2_t1_action_exs)) self.assertEqual(states.RUNNING, wf2_t1_action_exs[0].state) # Cancel action execution for task. self.engine.on_action_complete( wf2_t1_action_exs[0].id, ml_actions.Result(cancel=True) ) self.await_workflow_cancelled(wf2_ex.id) self.await_workflow_cancelled(wf1_ex.id) # Resume workflow and re-run failed subworkflow task. self.engine.rerun_workflow(wf2_t1_ex.id) with db_api.transaction(): wf1_execs = db_api.get_workflow_executions() wf1_ex = self._assert_single_item(wf1_execs, name='wb1.wf1') wf1_t2_ex = self._assert_single_item( wf1_ex.task_executions, name='t2' ) self.await_task_state(wf1_t2_ex.id, states.RUNNING) with db_api.transaction(): sub_wf_exs = db_api.get_workflow_executions( task_execution_id=wf1_t2_ex.id ) self.assertEqual(1, len(sub_wf_exs)) self.assertEqual(states.RUNNING, sub_wf_exs[0].state) wf2_ex = sub_wf_exs[0] wf2_t1_ex = self._assert_single_item( wf2_ex.task_executions, name='wf2_t1' ) self.await_task_state(wf2_t1_ex.id, states.RUNNING) wf2_t1_action_exs = db_api.get_action_executions( task_execution_id=wf2_t1_ex.id ) self.assertEqual(2, len(wf2_t1_action_exs)) # Check there is exactly 1 action in Running and 1 in Cancelled state. # Order doesn't matter. self._assert_single_item(wf2_t1_action_exs, state=states.RUNNING) self._assert_single_item(wf2_t1_action_exs, state=states.CANCELLED) # Mark async action execution complete. self.engine.on_action_complete( wf2_t1_action_exs[1].id, ml_actions.Result(data={'foo': 'bar'}) ) # Wait for the workflows to succeed. self.await_workflow_success(wf1_ex.id) self.await_workflow_success(wf2_ex.id) sub_wf_exs = db_api.get_workflow_executions( task_execution_id=wf1_t2_ex.id ) self.assertEqual(1, len(sub_wf_exs)) self.assertEqual(states.SUCCESS, sub_wf_exs[0].state) wf2_t1_action_exs = db_api.get_action_executions( task_execution_id=wf2_t1_ex.id ) self.assertEqual(2, len(wf2_t1_action_exs)) # Check there is exactly 1 action in Success and 1 in Cancelled state. # Order doesn't matter. self._assert_single_item(wf2_t1_action_exs, state=states.SUCCESS) self._assert_single_item(wf2_t1_action_exs, state=states.CANCELLED)
def test_rerun_subflow_task(self): wb_service.create_workbook_v2(SUBFLOW_WORKBOOK) # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb1.wf1') self.await_workflow_error(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(2, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertEqual(states.ERROR, task_2_ex.state) self.assertIsNotNone(task_2_ex.state_info) with db_api.transaction(): # Get subworkflow and related task sub_wf_exs = db_api.get_workflow_executions( task_execution_id=task_2_ex.id ) sub_wf_ex = sub_wf_exs[0] sub_wf_task_execs = sub_wf_ex.task_executions self.assertEqual(states.ERROR, sub_wf_ex.state) self.assertIsNotNone(sub_wf_ex.state_info) self.assertEqual(1, len(sub_wf_task_execs)) sub_wf_task_ex = self._assert_single_item( sub_wf_task_execs, name='wf2_t1' ) self.assertEqual(states.ERROR, sub_wf_task_ex.state) self.assertIsNotNone(sub_wf_task_ex.state_info) # Resume workflow and re-run failed subworkflow task. self.engine.rerun_workflow(sub_wf_task_ex.id) sub_wf_ex = db_api.get_workflow_execution(sub_wf_ex.id) self.assertEqual(states.RUNNING, sub_wf_ex.state) self.assertIsNone(sub_wf_ex.state_info) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) # Wait for the subworkflow to succeed. self.await_workflow_success(sub_wf_ex.id) with db_api.transaction(): sub_wf_ex = db_api.get_workflow_execution(sub_wf_ex.id) sub_wf_task_execs = sub_wf_ex.task_executions self.assertEqual(states.SUCCESS, sub_wf_ex.state) self.assertIsNone(sub_wf_ex.state_info) self.assertEqual(1, len(sub_wf_task_execs)) sub_wf_task_ex = self._assert_single_item( sub_wf_task_execs, name='wf2_t1' ) # Check action executions of subworkflow task. self.assertEqual(states.SUCCESS, sub_wf_task_ex.state) self.assertIsNone(sub_wf_task_ex.state_info) sub_wf_task_ex_action_exs = db_api.get_action_executions( task_execution_id=sub_wf_task_ex.id ) self.assertEqual(2, len(sub_wf_task_ex_action_exs)) # Check there is exactly 1 action in Success and 1 in error state. # Order doesn't matter. self._assert_single_item( sub_wf_task_ex_action_exs, state=states.SUCCESS ) self._assert_single_item(sub_wf_task_ex_action_exs, state=states.ERROR) # Wait for the main workflow to succeed. self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.SUCCESS, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertEqual(3, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') task_3_ex = self._assert_single_item(task_execs, name='t3') # Check action executions of task 1. self.assertEqual(states.SUCCESS, task_1_ex.state) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) self.assertEqual(1, len(task_1_action_exs)) self.assertEqual(states.SUCCESS, task_1_action_exs[0].state) # Check action executions of task 2. self.assertEqual(states.SUCCESS, task_2_ex.state) self.assertIsNone(task_2_ex.state_info) task_2_action_exs = db_api.get_workflow_executions( task_execution_id=task_2_ex.id ) self.assertEqual(1, len(task_2_action_exs)) self.assertEqual(states.SUCCESS, task_1_action_exs[0].state) # Check action executions of task 3. self.assertEqual(states.SUCCESS, task_3_ex.state) task_3_action_exs = db_api.get_action_executions( task_execution_id=task_3_ex.id) self.assertEqual(1, len(task_3_action_exs)) self.assertEqual(states.SUCCESS, task_3_action_exs[0].state)
def test_rerun_with_items_diff_env_vars(self): wb_service.create_workbook_v2(WITH_ITEMS_WORKBOOK_DIFF_ENV_VAR) # Initial environment variables for the workflow execution. env = {'var1': 'fee fi fo fum'} # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb3.wf1', {}, env=env) self.await_execution_error(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(1, len(wf_ex.task_executions)) task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1') self.assertEqual(states.ERROR, task_1_ex.state) self.assertIsNotNone(task_1_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id) self.assertEqual(3, len(task_1_action_exs)) # Update env in workflow execution with the following. updated_env = {'var1': 'foobar'} # Resume workflow and re-run failed task. self.engine.rerun_workflow(wf_ex.id, task_1_ex.id, reset=False, env=updated_env) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.await_execution_success(wf_ex.id, delay=10) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.SUCCESS, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertEqual(2, len(wf_ex.task_executions)) task_1_ex = self._assert_single_item(wf_ex.task_executions, name='t1') task_2_ex = self._assert_single_item(wf_ex.task_executions, name='t2') # Check action executions of task 1. self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertIsNone(task_1_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id) expected_inputs = [ 'Task 1.0 [%s]' % env['var1'], # Task 1 item 0 (error). 'Task 1.1 [%s]' % env['var1'], # Task 1 item 1. 'Task 1.2 [%s]' % env['var1'], # Task 1 item 2 (error). 'Task 1.0 [%s]' % updated_env['var1'], # Task 1 item 0 (rerun). 'Task 1.2 [%s]' % updated_env['var1'] # Task 1 item 2 (rerun). ] result = zip(task_1_action_exs, expected_inputs) for (action_ex, expected_input) in result: self.assertDictEqual({'output': expected_input}, action_ex.input) # Check action executions of task 2. self.assertEqual(states.SUCCESS, task_2_ex.state) task_2_action_exs = db_api.get_action_executions( task_execution_id=task_2_ex.id) self.assertEqual(1, len(task_2_action_exs))
def test_rerun_cancelled_with_items(self): wb_def = """ version: '2.0' name: wb1 workflows: wf1: type: direct tasks: t1: with-items: i in <% list(range(0, 3)) %> action: std.async_noop on-success: - t2 t2: action: std.echo output="Task 2" """ wb_service.create_workbook_v2(wb_def) wf1_ex = self.engine.start_workflow('wb1.wf1', {}) self.await_workflow_state(wf1_ex.id, states.RUNNING) with db_api.transaction(): wf1_execs = db_api.get_workflow_executions() wf1_ex = self._assert_single_item(wf1_execs, name='wb1.wf1') wf1_t1_ex = self._assert_single_item( wf1_ex.task_executions, name='t1' ) wf1_t1_action_exs = db_api.get_action_executions( task_execution_id=wf1_t1_ex.id ) self.assertEqual(3, len(wf1_t1_action_exs)) self.assertEqual(states.RUNNING, wf1_t1_action_exs[0].state) self.assertEqual(states.RUNNING, wf1_t1_action_exs[1].state) self.assertEqual(states.RUNNING, wf1_t1_action_exs[2].state) # Cancel action execution for task. for wf1_t1_action_ex in wf1_t1_action_exs: self.engine.on_action_complete( wf1_t1_action_ex.id, ml_actions.Result(cancel=True) ) self.await_workflow_cancelled(wf1_ex.id) wf1_t1_action_exs = db_api.get_action_executions( task_execution_id=wf1_t1_ex.id ) self.assertEqual(3, len(wf1_t1_action_exs)) self.assertEqual(states.CANCELLED, wf1_t1_action_exs[0].state) self.assertEqual(states.CANCELLED, wf1_t1_action_exs[1].state) self.assertEqual(states.CANCELLED, wf1_t1_action_exs[2].state) # Resume workflow and re-run failed with items task. self.engine.rerun_workflow(wf1_t1_ex.id, reset=False) with db_api.transaction(): wf1_execs = db_api.get_workflow_executions() wf1_ex = self._assert_single_item(wf1_execs, name='wb1.wf1') wf1_t1_ex = self._assert_single_item( wf1_ex.task_executions, name='t1' ) self.await_workflow_state(wf1_ex.id, states.RUNNING) wf1_t1_action_exs = db_api.get_action_executions( task_execution_id=wf1_t1_ex.id ) self.assertEqual(6, len(wf1_t1_action_exs)) # Check there is exactly 3 action in Running and 3 in Cancelled state. # Order doesn't matter. self._assert_multiple_items(wf1_t1_action_exs, 3, state=states.RUNNING) self._assert_multiple_items( wf1_t1_action_exs, 3, state=states.CANCELLED ) # Mark async action execution complete. for i in range(3, 6): self.engine.on_action_complete( wf1_t1_action_exs[i].id, ml_actions.Result(data={'foo': 'bar'}) ) # Wait for the workflows to succeed. self.await_workflow_success(wf1_ex.id) with db_api.transaction(): wf1_ex = db_api.get_workflow_execution(wf1_ex.id) wf1_t1_ex = self._assert_single_item( wf1_ex.task_executions, name='t1' ) wf1_t1_action_exs = db_api.get_action_executions( task_execution_id=wf1_t1_ex.id ) self.assertEqual(6, len(wf1_t1_action_exs)) # Check there is exactly 3 action in Success and 3 in Cancelled state. # Order doesn't matter. self._assert_multiple_items(wf1_t1_action_exs, 3, state=states.SUCCESS) self._assert_multiple_items( wf1_t1_action_exs, 3, state=states.CANCELLED )
def test_rerun_diff_env_vars(self): wb_service.create_workbook_v2(SIMPLE_WORKBOOK_DIFF_ENV_VAR) # Initial environment variables for the workflow execution. env = {'var1': 'fee fi fo fum', 'var2': 'foobar'} # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb1.wf1', {}, task_name='t3', env=env) self.await_workflow_error(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(2, len(task_execs)) self.assertDictEqual(env, wf_ex.params['env']) self.assertDictEqual(env, wf_ex.context['__env']) task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertEqual(states.ERROR, task_2_ex.state) self.assertIsNotNone(task_2_ex.state_info) # Update env in workflow execution with the following. updated_env = {'var1': 'Task 2', 'var2': 'Task 3'} # Resume workflow and re-run failed task. self.engine.rerun_workflow(task_2_ex.id, env=updated_env) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertDictEqual(updated_env, wf_ex.params['env']) self.assertDictEqual(updated_env, wf_ex.context['__env']) # Wait for the workflow to succeed. self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.SUCCESS, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertEqual(3, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') task_3_ex = self._assert_single_item(task_execs, name='t3') # Check action executions of task 1. self.assertEqual(states.SUCCESS, task_1_ex.state) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id) self.assertEqual(1, len(task_1_action_exs)) self.assertEqual(states.SUCCESS, task_1_action_exs[0].state) self.assertDictEqual({'output': 'Task 1'}, task_1_action_exs[0].input) # Check action executions of task 2. self.assertEqual(states.SUCCESS, task_2_ex.state) self.assertIsNone(task_2_ex.state_info) task_2_action_exs = db_api.get_action_executions( task_execution_id=task_2_ex.id) self.assertEqual(2, len(task_2_action_exs)) self.assertEqual(states.ERROR, task_2_action_exs[0].state) self.assertEqual(states.SUCCESS, task_2_action_exs[1].state) self.assertDictEqual({'output': env['var1']}, task_2_action_exs[0].input) self.assertDictEqual({'output': updated_env['var1']}, task_2_action_exs[1].input) # Check action executions of task 3. self.assertEqual(states.SUCCESS, task_3_ex.state) task_3_action_exs = db_api.get_action_executions( task_execution_id=task_3_ex.id) self.assertEqual(1, len(task_3_action_exs)) self.assertEqual(states.SUCCESS, task_3_action_exs[0].state) self.assertDictEqual({'output': updated_env['var2']}, task_3_action_exs[0].input)
def test_workflow_spec_cache_update_via_workbook_service(self): wb_text = """ version: '2.0' name: wb workflows: wf: tasks: task1: action: std.echo output="Echo" """ wb_service.create_workbook_v2(wb_text) self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size()) self.assertEqual(0, spec_parser.get_wf_definition_spec_cache_size()) wf = db_api.get_workflow_definition('wb.wf') wf_spec = spec_parser.get_workflow_spec_by_definition_id( wf.id, wf.updated_at ) self.assertEqual(1, len(wf_spec.get_tasks())) self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size()) self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size()) # Now update workflow definition and check that cache is updated too. wb_text = """ version: '2.0' name: wb workflows: wf: tasks: task1: action: std.echo output="1" task2: action: std.echo output="2" """ wb_service.update_workbook_v2(wb_text) self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size()) self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size()) wf = db_api.get_workflow_definition(wf.id) wf_spec = spec_parser.get_workflow_spec_by_definition_id( wf.id, wf.updated_at ) self.assertEqual(2, len(wf_spec.get_tasks())) self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size()) self.assertEqual(2, spec_parser.get_wf_definition_spec_cache_size())
def test_rerun(self): wb_service.create_workbook_v2(SIMPLE_WORKBOOK) # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb1.wf1', {}, task_name='t3') self.await_workflow_error(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(2, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertEqual(states.ERROR, task_2_ex.state) self.assertIsNotNone(task_2_ex.state_info) # Resume workflow and re-run failed task. self.engine.rerun_workflow(task_2_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) # Wait for the workflow to succeed. self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.SUCCESS, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertEqual(3, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') task_3_ex = self._assert_single_item(task_execs, name='t3') # Check action executions of task 1. self.assertEqual(states.SUCCESS, task_1_ex.state) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id) self.assertEqual(1, len(task_1_action_exs)) self.assertEqual(states.SUCCESS, task_1_action_exs[0].state) # Check action executions of task 2. self.assertEqual(states.SUCCESS, task_2_ex.state) self.assertIsNone(task_2_ex.state_info) task_2_action_exs = db_api.get_action_executions( task_execution_id=task_2_ex.id) self.assertEqual(2, len(task_2_action_exs)) self.assertEqual(states.ERROR, task_2_action_exs[0].state) self.assertEqual(states.SUCCESS, task_2_action_exs[1].state) # Check action executions of task 3. self.assertEqual(states.SUCCESS, task_3_ex.state) task_3_action_exs = db_api.get_action_executions( task_execution_id=task_3_ex.id) self.assertEqual(1, len(task_3_action_exs)) self.assertEqual(states.SUCCESS, task_3_action_exs[0].state)
def test_multiple_reruns_with_items(self): wb_service.create_workbook_v2(WITH_ITEMS_WORKBOOK) # Run workflow and fail task. wf_ex = self.engine.start_workflow('wb3.wf1') self.await_workflow_error(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) self.assertEqual(1, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') self.await_task_error(task_1_ex.id) self.assertIsNotNone(task_1_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) self.assertEqual(3, len(task_1_action_exs)) # Resume workflow and re-run failed task. Re-run #1 with no reset. wf_ex = self.engine.rerun_workflow(task_1_ex.id, reset=False) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.await_workflow_error(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) self.assertEqual(5, len(task_1_action_exs)) # Resume workflow and re-run failed task. Re-run #2 with reset. self.engine.rerun_workflow(task_1_ex.id, reset=True) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.await_workflow_error(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) self.assertEqual(8, len(task_1_action_exs)) # Resume workflow and re-run failed task. Re-run #3 with no reset. self.engine.rerun_workflow(task_1_ex.id, reset=False) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.await_workflow_error(wf_ex.id) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.ERROR, wf_ex.state) self.assertIsNotNone(wf_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) self.assertEqual(10, len(task_1_action_exs)) # Resume workflow and re-run failed task. Re-run #4 with no reset. self.engine.rerun_workflow(task_1_ex.id, reset=False) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(states.RUNNING, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(states.SUCCESS, wf_ex.state) self.assertIsNone(wf_ex.state_info) self.assertEqual(2, len(task_execs)) task_1_ex = self._assert_single_item(task_execs, name='t1') task_2_ex = self._assert_single_item(task_execs, name='t2') # Check action executions of task 1. self.assertEqual(states.SUCCESS, task_1_ex.state) self.assertIsNone(task_1_ex.state_info) task_1_action_exs = db_api.get_action_executions( task_execution_id=task_1_ex.id ) # The single action execution that succeeded should not re-run. self.assertEqual(12, len(task_1_action_exs)) self.assertListEqual( ['Task 1.0', 'Task 1.1', 'Task 1.2'], task_1_ex.published.get('v1') ) # Check action executions of task 2. self.assertEqual(states.SUCCESS, task_2_ex.state) task_2_action_exs = db_api.get_action_executions( task_execution_id=task_2_ex.id ) self.assertEqual(1, len(task_2_action_exs))
def test_nested_wf_errors_only(self): wb_text = """--- version: '2.0' name: wb workflows: parent_wf: tasks: task1: action: std.noop on-success: task2 task2: workflow: sub_wf on-success: task3 task3: action: std.fail sub_wf: tasks: task1: action: std.noop on-success: task2 task2: action: std.fail """ wb_service.create_workbook_v2(wb_text) wf_ex = self.engine.start_workflow('wb.parent_wf') self.await_workflow_error(wf_ex.id) resp = self.app.get( '/v2/executions/%s/report?errors_only=true' % wf_ex.id ) self.assertEqual(200, resp.status_int) # Now let's verify the response structure self.assertIn('root_workflow_execution', resp.json) root_wf_ex = resp.json['root_workflow_execution'] self.assertIsInstance(root_wf_ex, dict) self.assertEqual('wb.parent_wf', root_wf_ex['name']) self.assertEqual(states.ERROR, root_wf_ex['state']) self.assertGreater(len(root_wf_ex['state_info']), 0) tasks = root_wf_ex['task_executions'] self.assertIsInstance(tasks, list) self.assertEqual(1, len(tasks)) # There must be only task2 in the response. # Verify task2 info. task2 = self._assert_single_item(tasks, name='task2') self.assertEqual(states.ERROR, task2['state']) self.assertEqual(0, len(task2['action_executions'])) self.assertEqual(1, len(task2['workflow_executions'])) sub_wf_entry = task2['workflow_executions'][0] self.assertEqual(states.ERROR, sub_wf_entry['state']) sub_wf_tasks = sub_wf_entry['task_executions'] self.assertEqual(1, len(sub_wf_tasks)) sub_wf_task2 = self._assert_single_item( sub_wf_tasks, name='task2', state=states.ERROR ) self.assertEqual(1, len(sub_wf_task2['action_executions'])) self.assertEqual( states.ERROR, sub_wf_task2['action_executions'][0]['state'] ) # Verify statistics. stat = resp.json['statistics'] self.assertEqual(2, stat['error_tasks_count']) self.assertEqual(0, stat['idle_tasks_count']) self.assertEqual(0, stat['paused_tasks_count']) self.assertEqual(0, stat['running_tasks_count']) self.assertEqual(0, stat['success_tasks_count']) self.assertEqual(2, stat['total_tasks_count'])
def setUp(self): super(SubworkflowsTest, self).setUp() wb_service.create_workbook_v2(WORKBOOK)