def test_chain_pause_resume_with_init_vars(self): # A temp file is created during test setup. Ensure the temp file exists. # The test action chain will stall until this file is deleted. This gives # the unit test a moment to run any test related logic. path = self.temp_file_path self.assertTrue(os.path.exists(path)) action = TEST_PACK + '.' + 'test_pause_resume_with_init_vars' params = {'tempfile': path, 'message': 'foobar'} liveaction = LiveActionDB(action=action, parameters=params) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # Wait until the liveaction is running. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING) # Request action chain to pause. liveaction, execution = action_service.request_pause( liveaction, USERNAME) # Wait until the liveaction is pausing. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSING) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSING) # Delete the temporary file that the action chain is waiting on. os.remove(path) self.assertFalse(os.path.exists(path)) # Wait until the liveaction is paused. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSED) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED) # Wait for non-blocking threads to complete. Ensure runner is not running. MockLiveActionPublisherNonBlocking.wait_all() # Request action chain to resume. liveaction, execution = action_service.request_resume( liveaction, USERNAME) # Wait until the liveaction is completed. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) # Wait for non-blocking threads to complete. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 2) self.assertEqual(liveaction.result['tasks'][1]['result']['stdout'], 'FOOBAR')
def test_chain_pause_resume_status_change(self): """Tests context_result is updated when last task's status changes between pause and resume """ # A temp file is created during test setup. Ensure the temp file exists. # The test action chain will stall until this file is deleted. This gives # the unit test a moment to run any test related logic. path = self.temp_file_path self.assertTrue(os.path.exists(path)) action = TEST_PACK + '.' + 'test_pause_resume_context_result' params = {'tempfile': path} liveaction = LiveActionDB(action=action, parameters=params) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # This workflow runs 'core.ask' so will pause on its own. We just need to # wait until the liveaction is pausing. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSING) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSING) # Delete the temporary file that the action chain is waiting on. os.remove(path) self.assertFalse(os.path.exists(path)) # Wait until the liveaction is paused. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSED) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED) # Wait for non-blocking threads to complete. Ensure runner is not running. MockLiveActionPublisherNonBlocking.wait_all() last_task_liveaction_id = liveaction.result['tasks'][-1][ 'liveaction_id'] action_utils.update_liveaction_status( status=action_constants.LIVEACTION_STATUS_SUCCEEDED, result={'foo': 'bar'}, liveaction_id=last_task_liveaction_id) # Request action chain to resume. liveaction, execution = action_service.request_resume( liveaction, USERNAME) # Wait until the liveaction is completed. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) # Wait for non-blocking threads to complete. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 2) self.assertEqual(liveaction.result['tasks'][0]['result']['foo'], 'bar')
def test_skip_notify_default_for_task_with_notify(self): action = TEST_PACK + "." + "test_subworkflow_default_with_notify_task" liveaction = LiveActionDB(action=action) liveaction.notify = notify_api_models.NotificationsHelper.to_model( MOCK_NOTIFY) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # Wait until the liveaction is running. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING) execution = self._wait_for_children(execution) self.assertEqual(len(execution.children), 1) # Assert task1 notify is set. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED) notify = notify_api_models.NotificationsHelper.from_model( notify_model=task1_live.notify) self.assertEqual(notify, MOCK_NOTIFY) execution = self._wait_for_children(execution, expected_children=2, retries=300) self.assertEqual(len(execution.children), 2) # Assert task2 notify is not skipped by default. task2_exec = ActionExecution.get_by_id(execution.children[1]) task2_live = LiveAction.get_by_id(task2_exec.liveaction["id"]) self.assertIsNone(task2_live.notify) MockLiveActionPublisherNonBlocking.wait_all()
def test_chain_cancel_cascade_to_subworkflow(self): # A temp file is created during test setup. Ensure the temp file exists. # The test action chain will stall until this file is deleted. This gives # the unit test a moment to run any test related logic. path = self.temp_file_path self.assertTrue(os.path.exists(path)) action = TEST_PACK + '.' + 'test_cancel_with_subworkflow' params = {'tempfile': path, 'message': 'foobar'} liveaction = LiveActionDB(action=action, parameters=params) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # Wait until the liveaction is running. liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING) # Wait for subworkflow to register. execution = self._wait_for_children(execution) self.assertEqual(len(execution.children), 1) # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_RUNNING) # Request action chain to cancel. liveaction, execution = action_service.request_cancellation(liveaction, USERNAME) # Wait until the liveaction is canceling. liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELING) self.assertEqual(len(execution.children), 1) # Wait until the subworkflow is canceling. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_CANCELING) # Delete the temporary file that the action chain is waiting on. os.remove(path) self.assertFalse(os.path.exists(path)) # Wait until the liveaction is canceled. liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual(len(execution.children), 1) # Wait until the subworkflow is canceled. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_CANCELED) # Wait for non-blocking threads to complete. Ensure runner is not running. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 1) subworkflow = liveaction.result['tasks'][0] self.assertEqual(len(subworkflow['result']['tasks']), 1) self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_CANCELED)
def test_cancel_retry_exhausted(self): liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS) liveaction, execution = action_service.request(liveaction) MockLiveActionPublisherNonBlocking.wait_all() eventlet.sleep(4) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING) mistral_context = liveaction.context.get('mistral', None) self.assertIsNotNone(mistral_context) self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id')) self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name')) requester = cfg.CONF.system_user.user liveaction, execution = action_service.request_cancellation( liveaction, requester) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_CANCELING) expected_call_count = 2 self._wait_on_call_count(executions.ExecutionManager.update, expected_call_count) calls = [ call(WF1_EXEC.get('id'), 'CANCELLED') for i in range(0, expected_call_count) ] executions.ExecutionManager.update.assert_has_calls(calls)
def test_chain_pause_resume_last_task_failed_with_no_next_task(self): # A temp file is created during test setup. Ensure the temp file exists. # The test action chain will stall until this file is deleted. This gives # the unit test a moment to run any test related logic. path = self.temp_file_path self.assertTrue(os.path.exists(path)) action = TEST_PACK + '.' + 'test_pause_resume_last_task_failed_with_no_next_task' params = {'tempfile': path, 'message': 'foobar'} liveaction = LiveActionDB(action=action, parameters=params) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # Wait until the liveaction is running. liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING) # Request action chain to pause. liveaction, execution = action_service.request_pause(liveaction, USERNAME) # Wait until the liveaction is pausing. liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSING) extra_info = str(liveaction) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSING, extra_info) # Delete the temporary file that the action chain is waiting on. os.remove(path) self.assertFalse(os.path.exists(path)) # Wait until the liveaction is paused. liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED) extra_info = str(liveaction) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info) # Wait for non-blocking threads to complete. Ensure runner is not running. MockLiveActionPublisherNonBlocking.wait_all() # Request action chain to resume. liveaction, execution = action_service.request_resume(liveaction, USERNAME) # Wait until the liveaction is completed. liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_FAILED) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED) # Wait for non-blocking threads to complete. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 1) self.assertEqual( liveaction.result['tasks'][0]['state'], action_constants.LIVEACTION_STATUS_FAILED )
def test_chain_pause_resume_status_change(self): # Tests context_result is updated when last task's status changes between pause and resume action = TEST_PACK + "." + "test_pause_resume_context_result" liveaction = LiveActionDB(action=action) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # Wait until the liveaction is paused. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSED) extra_info = str(liveaction) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info) # Wait for non-blocking threads to complete. Ensure runner is not running. MockLiveActionPublisherNonBlocking.wait_all() last_task_liveaction_id = liveaction.result["tasks"][-1][ "liveaction_id"] action_utils.update_liveaction_status( status=action_constants.LIVEACTION_STATUS_SUCCEEDED, end_timestamp=date_utils.get_datetime_utc_now(), result={"foo": "bar"}, liveaction_id=last_task_liveaction_id, ) # Request action chain to resume. liveaction, execution = action_service.request_resume( liveaction, USERNAME) # Wait until the liveaction is completed. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual( liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED, str(liveaction), ) # Wait for non-blocking threads to complete. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn("tasks", liveaction.result) self.assertEqual(len(liveaction.result["tasks"]), 2) self.assertEqual(liveaction.result["tasks"][0]["result"]["foo"], "bar")
def test_chain_pause_resume_status_change(self): # Tests context_result is updated when last task's status changes between pause and resume action = TEST_PACK + '.' + 'test_pause_resume_context_result' liveaction = LiveActionDB(action=action) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # Wait until the liveaction is paused. liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED) extra_info = str(liveaction) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info) # Wait for non-blocking threads to complete. Ensure runner is not running. MockLiveActionPublisherNonBlocking.wait_all() last_task_liveaction_id = liveaction.result['tasks'][-1]['liveaction_id'] action_utils.update_liveaction_status( status=action_constants.LIVEACTION_STATUS_SUCCEEDED, end_timestamp=date_utils.get_datetime_utc_now(), result={'foo': 'bar'}, liveaction_id=last_task_liveaction_id ) # Request action chain to resume. liveaction, execution = action_service.request_resume(liveaction, USERNAME) # Wait until the liveaction is completed. liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual( liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED, str(liveaction) ) # Wait for non-blocking threads to complete. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 2) self.assertEqual(liveaction.result['tasks'][0]['result']['foo'], 'bar')
def test_cancel_retry_exhausted(self): liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS) liveaction, execution = action_service.request(liveaction) MockLiveActionPublisherNonBlocking.wait_all() eventlet.sleep(4) liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING) mistral_context = liveaction.context.get('mistral', None) self.assertIsNotNone(mistral_context) self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id')) self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name')) requester = cfg.CONF.system_user.user liveaction, execution = action_service.request_cancellation(liveaction, requester) liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELING) expected_call_count = 2 self._wait_on_call_count(executions.ExecutionManager.update, expected_call_count) calls = [call(WF1_EXEC.get('id'), 'CANCELLED') for i in range(0, expected_call_count)] executions.ExecutionManager.update.assert_has_calls(calls)
def test_chain_cancel(self): # A temp file is created during test setup. Ensure the temp file exists. # The test action chain will stall until this file is deleted. This gives # the unit test a moment to run any test related logic. path = self.temp_file_path self.assertTrue(os.path.exists(path)) action = TEST_PACK + "." + "test_cancel" params = {"tempfile": path, "message": "foobar"} liveaction = LiveActionDB(action=action, parameters=params) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # Wait until the liveaction is running. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING) # Request action chain to cancel. liveaction, execution = action_service.request_cancellation( liveaction, USERNAME) # Wait until the liveaction is canceling. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_CANCELING) # Delete the temporary file that the action chain is waiting on. os.remove(path) self.assertFalse(os.path.exists(path)) # Wait until the liveaction is canceled. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_CANCELED) # Wait for non-blocking threads to complete. Ensure runner is not running. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn("tasks", liveaction.result) self.assertEqual(len(liveaction.result["tasks"]), 1)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) MockLiveActionPublisherNonBlocking.wait_all() # Since states are being processed asynchronously, wait for the # liveactions to go into scheduled states. for i in range(0, 100): eventlet.sleep(1) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] if len(scheduled) == policy_db.parameters['threshold']: break MockLiveActionPublisherNonBlocking.wait_all() scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 1 # Tally requested state. # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status == action_constants.LIVEACTION_STATUS_DELAYED: break # Assert the action is delayed. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally requested state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in SCHEDULED_STATES: break # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_chain_pause_resume_cascade_to_parent_workflow(self): # A temp file is created during test setup. Ensure the temp file exists. # The test action chain will stall until this file is deleted. This gives # the unit test a moment to run any test related logic. path = self.temp_file_path self.assertTrue(os.path.exists(path)) action = TEST_PACK + '.' + 'test_pause_resume_with_subworkflow' params = {'tempfile': path, 'message': 'foobar'} liveaction = LiveActionDB(action=action, parameters=params) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # Wait until the liveaction is running. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING) # Wait for subworkflow to register. execution = self._wait_for_children(execution) self.assertEqual(len(execution.children), 1) # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING) self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_RUNNING) # Request subworkflow to pause. task1_live, task1_exec = action_service.request_pause( task1_live, USERNAME) # Wait until the subworkflow is pausing. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSING) extra_info = str(task1_live) self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_PAUSING, extra_info) # Delete the temporary file that the action chain is waiting on. os.remove(path) self.assertFalse(os.path.exists(path)) # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSED) extra_info = str(task1_live) self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info) # Wait until the parent liveaction is paused. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSED) extra_info = str(liveaction) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info) self.assertEqual(len(execution.children), 1) # Wait for non-blocking threads to complete. Ensure runner is not running. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 1) subworkflow = liveaction.result['tasks'][0] self.assertEqual(len(subworkflow['result']['tasks']), 1) self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_PAUSED) # Request subworkflow to resume. task1_live, task1_exec = action_service.request_resume( task1_live, USERNAME) # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) # The parent workflow will stay paused. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSED) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED) # Wait for non-blocking threads to complete. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result of the parent, which should stay the same # because only the subworkflow was resumed. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 1) subworkflow = liveaction.result['tasks'][0] self.assertEqual(len(subworkflow['result']['tasks']), 1) self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_PAUSED) # Request parent workflow to resume. liveaction, execution = action_service.request_resume( liveaction, USERNAME) # Wait until the liveaction is completed. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) # Wait for non-blocking threads to complete. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 2) subworkflow = liveaction.result['tasks'][0] self.assertEqual(len(subworkflow['result']['tasks']), 2) self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_SUCCEEDED)
def test_chain_pause_resume_cascade_to_parent_workflow(self): # A temp file is created during test setup. Ensure the temp file exists. # The test action chain will stall until this file is deleted. This gives # the unit test a moment to run any test related logic. path = self.temp_file_path self.assertTrue(os.path.exists(path)) action = TEST_PACK + '.' + 'test_pause_resume_with_subworkflow' params = {'tempfile': path, 'message': 'foobar'} liveaction = LiveActionDB(action=action, parameters=params) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # Wait until the liveaction is running. liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING) # Wait for subworkflow to register. execution = self._wait_for_children(execution) self.assertEqual(len(execution.children), 1) # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_RUNNING) self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_RUNNING) # Request subworkflow to pause. task1_live, task1_exec = action_service.request_pause(task1_live, USERNAME) # Wait until the subworkflow is pausing. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_PAUSING) extra_info = str(task1_live) self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_PAUSING, extra_info) # Delete the temporary file that the action chain is waiting on. os.remove(path) self.assertFalse(os.path.exists(path)) # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_PAUSED) extra_info = str(task1_live) self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info) # Wait until the parent liveaction is paused. liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED) extra_info = str(liveaction) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info) self.assertEqual(len(execution.children), 1) # Wait for non-blocking threads to complete. Ensure runner is not running. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 1) subworkflow = liveaction.result['tasks'][0] self.assertEqual(len(subworkflow['result']['tasks']), 1) self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_PAUSED) # Request subworkflow to resume. task1_live, task1_exec = action_service.request_resume(task1_live, USERNAME) # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) task1_live = LiveAction.get_by_id(task1_exec.liveaction['id']) task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) # The parent workflow will stay paused. liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED) # Wait for non-blocking threads to complete. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result of the parent, which should stay the same # because only the subworkflow was resumed. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 1) subworkflow = liveaction.result['tasks'][0] self.assertEqual(len(subworkflow['result']['tasks']), 1) self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_PAUSED) # Request parent workflow to resume. liveaction, execution = action_service.request_resume(liveaction, USERNAME) # Wait until the liveaction is completed. liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) # Wait for non-blocking threads to complete. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn('tasks', liveaction.result) self.assertEqual(len(liveaction.result['tasks']), 2) subworkflow = liveaction.result['tasks'][0] self.assertEqual(len(subworkflow['result']['tasks']), 2) self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_SUCCEEDED)
def test_chain_pause_resume_last_task_failed_with_no_next_task(self): # A temp file is created during test setup. Ensure the temp file exists. # The test action chain will stall until this file is deleted. This gives # the unit test a moment to run any test related logic. path = self.temp_file_path self.assertTrue(os.path.exists(path)) action = (TEST_PACK + "." + "test_pause_resume_last_task_failed_with_no_next_task") params = {"tempfile": path, "message": "foobar"} liveaction = LiveActionDB(action=action, parameters=params) liveaction, execution = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) # Wait until the liveaction is running. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING) # Request action chain to pause. liveaction, execution = action_service.request_pause( liveaction, USERNAME) # Wait until the liveaction is pausing. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSING) extra_info = str(liveaction) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSING, extra_info) # Delete the temporary file that the action chain is waiting on. os.remove(path) self.assertFalse(os.path.exists(path)) # Wait until the liveaction is paused. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSED) extra_info = str(liveaction) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info) # Wait for non-blocking threads to complete. Ensure runner is not running. MockLiveActionPublisherNonBlocking.wait_all() # Request action chain to resume. liveaction, execution = action_service.request_resume( liveaction, USERNAME) # Wait until the liveaction is completed. liveaction = self._wait_for_status( liveaction, action_constants.LIVEACTION_STATUS_FAILED) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED) # Wait for non-blocking threads to complete. MockLiveActionPublisherNonBlocking.wait_all() # Check liveaction result. self.assertIn("tasks", liveaction.result) self.assertEqual(len(liveaction.result["tasks"]), 1) self.assertEqual( liveaction.result["tasks"][0]["state"], action_constants.LIVEACTION_STATUS_FAILED, )
def setUp(self): super(ConcurrencyPolicyTestCase, self).setUp() # Wait for all threads to finish processing so there is no cross test polution MockLiveActionPublisherNonBlocking.wait_all()
def setUp(self): super(ConcurrencyByAttributePolicyTestCase, self).setUp() MockLiveActionPublisherNonBlocking.wait_all()
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) # Since states are being processed asynchronously, wait for the # liveactions to go into scheduled states. MockLiveActionPublisherNonBlocking.wait_all() for i in range(0, 100): eventlet.sleep(1) scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] if len(scheduled) == policy_db.parameters['threshold']: break MockLiveActionPublisherNonBlocking.wait_all() scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status == action_constants.LIVEACTION_STATUS_DELAYED: break # Assert the action is delayed. delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running states. # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in SCHEDULED_STATES: break liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') expected_num_pubs += 2 # Tally the canceling and canceled states. # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in SCHEDULED_STATES: break # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)