def test_chain_pause_resume_with_init_vars(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_with_init_vars'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to pause.
liveaction, execution = action_service.request_pause(
liveaction, USERNAME)
# Wait until the liveaction is pausing.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Request action chain to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 2)
self.assertEqual(liveaction.result['tasks'][1]['result']['stdout'],
'FOOBAR')
def test_chain_pause_resume_status_change(self):
"""Tests context_result is updated when last task's status changes between pause and resume
"""
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_context_result'
params = {'tempfile': path}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# This workflow runs 'core.ask' so will pause on its own. We just need to
# wait until the liveaction is pausing.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
last_task_liveaction_id = liveaction.result['tasks'][-1][
'liveaction_id']
action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_SUCCEEDED,
result={'foo': 'bar'},
liveaction_id=last_task_liveaction_id)
# Request action chain to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 2)
self.assertEqual(liveaction.result['tasks'][0]['result']['foo'], 'bar')
def test_skip_notify_default_for_task_with_notify(self):
action = TEST_PACK + "." + "test_subworkflow_default_with_notify_task"
liveaction = LiveActionDB(action=action)
liveaction.notify = notify_api_models.NotificationsHelper.to_model(
MOCK_NOTIFY)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Assert task1 notify is set.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"])
task1_live = self._wait_on_status(
task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED)
notify = notify_api_models.NotificationsHelper.from_model(
notify_model=task1_live.notify)
self.assertEqual(notify, MOCK_NOTIFY)
execution = self._wait_for_children(execution,
expected_children=2,
retries=300)
self.assertEqual(len(execution.children), 2)
# Assert task2 notify is not skipped by default.
task2_exec = ActionExecution.get_by_id(execution.children[1])
task2_live = LiveAction.get_by_id(task2_exec.liveaction["id"])
self.assertIsNone(task2_live.notify)
MockLiveActionPublisherNonBlocking.wait_all()
def test_chain_cancel_cascade_to_subworkflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_cancel_with_subworkflow'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to cancel.
liveaction, execution = action_service.request_cancellation(liveaction, USERNAME)
# Wait until the liveaction is canceling.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is canceling.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_CANCELING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is canceled.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is canceled.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_CANCELED)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_CANCELED)
def test_cancel_retry_exhausted(self):
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
MockLiveActionPublisherNonBlocking.wait_all()
eventlet.sleep(4)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'],
WF1_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_cancellation(
liveaction, requester)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
expected_call_count = 2
self._wait_on_call_count(executions.ExecutionManager.update,
expected_call_count)
calls = [
call(WF1_EXEC.get('id'), 'CANCELLED')
for i in range(0, expected_call_count)
]
executions.ExecutionManager.update.assert_has_calls(calls)
def test_chain_pause_resume_last_task_failed_with_no_next_task(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_last_task_failed_with_no_next_task'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to pause.
liveaction, execution = action_service.request_pause(liveaction, USERNAME)
# Wait until the liveaction is pausing.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(liveaction)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSING, extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Request action chain to resume.
liveaction, execution = action_service.request_resume(liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_FAILED)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
self.assertEqual(
liveaction.result['tasks'][0]['state'],
action_constants.LIVEACTION_STATUS_FAILED
)
def test_chain_pause_resume_status_change(self):
# Tests context_result is updated when last task's status changes between pause and resume
action = TEST_PACK + "." + "test_pause_resume_context_result"
liveaction = LiveActionDB(action=action)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
last_task_liveaction_id = liveaction.result["tasks"][-1][
"liveaction_id"]
action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_SUCCEEDED,
end_timestamp=date_utils.get_datetime_utc_now(),
result={"foo": "bar"},
liveaction_id=last_task_liveaction_id,
)
# Request action chain to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(
liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED,
str(liveaction),
)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn("tasks", liveaction.result)
self.assertEqual(len(liveaction.result["tasks"]), 2)
self.assertEqual(liveaction.result["tasks"][0]["result"]["foo"], "bar")
def test_chain_pause_resume_status_change(self):
# Tests context_result is updated when last task's status changes between pause and resume
action = TEST_PACK + '.' + 'test_pause_resume_context_result'
liveaction = LiveActionDB(action=action)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
last_task_liveaction_id = liveaction.result['tasks'][-1]['liveaction_id']
action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_SUCCEEDED,
end_timestamp=date_utils.get_datetime_utc_now(),
result={'foo': 'bar'},
liveaction_id=last_task_liveaction_id
)
# Request action chain to resume.
liveaction, execution = action_service.request_resume(liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(
liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED,
str(liveaction)
)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 2)
self.assertEqual(liveaction.result['tasks'][0]['result']['foo'], 'bar')
def test_cancel_retry_exhausted(self):
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
MockLiveActionPublisherNonBlocking.wait_all()
eventlet.sleep(4)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_cancellation(liveaction, requester)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
expected_call_count = 2
self._wait_on_call_count(executions.ExecutionManager.update, expected_call_count)
calls = [call(WF1_EXEC.get('id'), 'CANCELLED') for i in range(0, expected_call_count)]
executions.ExecutionManager.update.assert_has_calls(calls)
def test_chain_cancel(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + "." + "test_cancel"
params = {"tempfile": path, "message": "foobar"}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to cancel.
liveaction, execution = action_service.request_cancellation(
liveaction, USERNAME)
# Wait until the liveaction is canceling.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is canceled.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn("tasks", liveaction.result)
self.assertEqual(len(liveaction.result["tasks"]), 1)
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
MockLiveActionPublisherNonBlocking.wait_all()
# Since states are being processed asynchronously, wait for the
# liveactions to go into scheduled states.
for i in range(0, 100):
eventlet.sleep(1)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
if len(scheduled) == policy_db.parameters['threshold']:
break
MockLiveActionPublisherNonBlocking.wait_all()
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 1 # Tally requested state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into delayed state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status == action_constants.LIVEACTION_STATUS_DELAYED:
break
# Assert the action is delayed.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
expected_num_pubs += 1 # Tally requested state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in SCHEDULED_STATES:
break
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_chain_pause_resume_cascade_to_parent_workflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_with_subworkflow'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Request subworkflow to pause.
task1_live, task1_exec = action_service.request_pause(
task1_live, USERNAME)
# Wait until the subworkflow is pausing.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(task1_live)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_PAUSING,
extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(task1_live)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait until the parent liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
self.assertEqual(len(execution.children), 1)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'],
action_constants.LIVEACTION_STATUS_PAUSED)
# Request subworkflow to resume.
task1_live, task1_exec = action_service.request_resume(
task1_live, USERNAME)
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# The parent workflow will stay paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result of the parent, which should stay the same
# because only the subworkflow was resumed.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'],
action_constants.LIVEACTION_STATUS_PAUSED)
# Request parent workflow to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 2)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 2)
self.assertEqual(subworkflow['state'],
action_constants.LIVEACTION_STATUS_SUCCEEDED)
def test_chain_pause_resume_cascade_to_parent_workflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_with_subworkflow'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Request subworkflow to pause.
task1_live, task1_exec = action_service.request_pause(task1_live, USERNAME)
# Wait until the subworkflow is pausing.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(task1_live)
self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_PAUSING, extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(task1_live)
self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait until the parent liveaction is paused.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
self.assertEqual(len(execution.children), 1)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_PAUSED)
# Request subworkflow to resume.
task1_live, task1_exec = action_service.request_resume(task1_live, USERNAME)
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
# The parent workflow will stay paused.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result of the parent, which should stay the same
# because only the subworkflow was resumed.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_PAUSED)
# Request parent workflow to resume.
liveaction, execution = action_service.request_resume(liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 2)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 2)
self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_SUCCEEDED)
def test_chain_pause_resume_last_task_failed_with_no_next_task(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = (TEST_PACK + "." +
"test_pause_resume_last_task_failed_with_no_next_task")
params = {"tempfile": path, "message": "foobar"}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to pause.
liveaction, execution = action_service.request_pause(
liveaction, USERNAME)
# Wait until the liveaction is pausing.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSING,
extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Request action chain to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_FAILED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_FAILED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn("tasks", liveaction.result)
self.assertEqual(len(liveaction.result["tasks"]), 1)
self.assertEqual(
liveaction.result["tasks"][0]["state"],
action_constants.LIVEACTION_STATUS_FAILED,
)
def setUp(self):
super(ConcurrencyPolicyTestCase, self).setUp()
# Wait for all threads to finish processing so there is no cross test polution
MockLiveActionPublisherNonBlocking.wait_all()
def setUp(self):
super(ConcurrencyByAttributePolicyTestCase, self).setUp()
MockLiveActionPublisherNonBlocking.wait_all()
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'fu'})
action_service.request(liveaction)
# Since states are being processed asynchronously, wait for the
# liveactions to go into scheduled states.
MockLiveActionPublisherNonBlocking.wait_all()
for i in range(0, 100):
eventlet.sleep(1)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
if len(scheduled) == policy_db.parameters['threshold']:
break
MockLiveActionPublisherNonBlocking.wait_all()
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into delayed state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status == action_constants.LIVEACTION_STATUS_DELAYED:
break
# Assert the action is delayed.
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status,
action_constants.LIVEACTION_STATUS_DELAYED)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in SCHEDULED_STATES:
break
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
expected_num_pubs += 2 # Tally the canceling and canceled states.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in SCHEDULED_STATES:
break
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)