def test_do_next_clean_step_oob_reboot_last_step(self, mock_execute): # Resume where last_step is the last cleaning step tgt_prov_state = states.MANAGEABLE info = { 'clean_steps': self.clean_steps, 'cleaning_reboot': True, 'clean_step_index': len(self.clean_steps) - 1 } node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.CLEANING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info=info, clean_step=self.clean_steps[-1]) with task_manager.acquire(self.context, node.uuid, shared=False) as task: cleaning.do_next_clean_step(task, None) node.refresh() # Cleaning should be complete without calling additional steps self.assertEqual(tgt_prov_state, node.provision_state) self.assertEqual(states.NOSTATE, node.target_provision_state) self.assertEqual({}, node.clean_step) self.assertNotIn('clean_step_index', node.driver_internal_info) self.assertNotIn('cleaning_reboot', node.driver_internal_info) self.assertIsNone(node.driver_internal_info['clean_steps']) self.assertFalse(mock_execute.called)
def test_do_next_clean_step_oob_reboot_fail(self, tear_mock, mock_execute): # When a clean step fails with no reboot requested go to CLEANFAIL tgt_prov_state = states.MANAGEABLE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.CLEANING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={ 'clean_steps': self.clean_steps, 'clean_step_index': None }, clean_step={}) mock_execute.side_effect = exception.AgentConnectionFailed( reason='failed') with task_manager.acquire(self.context, node.uuid, shared=False) as task: cleaning.do_next_clean_step(task, 0) tear_mock.assert_called_once_with(task.driver.deploy, task) node.refresh() # Make sure we go to CLEANFAIL, clear clean_steps self.assertEqual(states.CLEANFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual({}, node.clean_step) self.assertNotIn('clean_step_index', node.driver_internal_info) self.assertNotIn('skip_current_clean_step', node.driver_internal_info) self.assertIsNotNone(node.last_error) self.assertTrue(node.maintenance) mock_execute.assert_called_once_with(mock.ANY, mock.ANY, self.clean_steps[0])
def _do_next_clean_step_continue_from_last_cleaning( self, return_state, mock_execute, manual=False): # Resume an in-progress cleaning after the first async step tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.CLEANING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={ 'clean_steps': self.clean_steps, 'clean_step_index': 0 }, clean_step=self.clean_steps[0]) mock_execute.return_value = return_state with task_manager.acquire(self.context, node.uuid, shared=False) as task: cleaning.do_next_clean_step(task, self.next_clean_step_index) node.refresh() self.assertEqual(states.CLEANWAIT, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual(self.clean_steps[1], node.clean_step) self.assertEqual(1, node.driver_internal_info['clean_step_index']) mock_execute.assert_called_once_with(mock.ANY, mock.ANY, self.clean_steps[1])
def test_do_next_clean_step_oob_reboot(self, mock_execute): # When a clean step fails, go to CLEANWAIT tgt_prov_state = states.MANAGEABLE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.CLEANING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={ 'clean_steps': self.clean_steps, 'clean_step_index': None, 'cleaning_reboot': True }, clean_step={}) mock_execute.side_effect = exception.AgentConnectionFailed( reason='failed') with task_manager.acquire(self.context, node.uuid, shared=False) as task: cleaning.do_next_clean_step(task, 0) node.refresh() # Make sure we go to CLEANWAIT self.assertEqual(states.CLEANWAIT, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual(self.clean_steps[0], node.clean_step) self.assertEqual(0, node.driver_internal_info['clean_step_index']) self.assertFalse(node.driver_internal_info['skip_current_clean_step']) mock_execute.assert_called_once_with(mock.ANY, mock.ANY, self.clean_steps[0])
def _do_next_clean_step_no_steps(self, mock_execute, manual=False, fast_track=False): if fast_track: self.config(fast_track=True, group='deploy') for info in ({ 'clean_steps': None, 'clean_step_index': None, 'agent_url': 'test-url', 'agent_secret_token': 'magic' }, { 'clean_steps': None, 'agent_url': 'test-url', 'agent_secret_token': 'it_is_a_kind_of_magic' }): # Resume where there are no steps, should be a noop tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE node = obj_utils.create_test_node( self.context, driver='fake-hardware', uuid=uuidutils.generate_uuid(), provision_state=states.CLEANING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info=info, clean_step={}) with task_manager.acquire(self.context, node.uuid, shared=False) as task: cleaning.do_next_clean_step(task, None) node.refresh() # Cleaning should be complete without calling additional steps self.assertEqual(tgt_prov_state, node.provision_state) self.assertEqual(states.NOSTATE, node.target_provision_state) self.assertEqual({}, node.clean_step) self.assertNotIn('clean_step_index', node.driver_internal_info) self.assertFalse(mock_execute.called) if fast_track: self.assertEqual('test-url', node.driver_internal_info.get('agent_url')) self.assertIsNotNone( node.driver_internal_info.get('agent_secret_token')) else: self.assertNotIn('agent_url', node.driver_internal_info) self.assertNotIn('agent_secret_token', node.driver_internal_info) mock_execute.reset_mock()
def _do_next_clean_step_fail_in_tear_down_cleaning(self, tear_mock, power_exec_mock, deploy_exec_mock, log_mock, manual=True): tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.CLEANING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={ 'clean_steps': self.clean_steps, 'clean_step_index': None }, clean_step={}) deploy_exec_mock.return_value = None power_exec_mock.return_value = None tear_mock.side_effect = Exception('boom') with task_manager.acquire(self.context, node.uuid, shared=False) as task: cleaning.do_next_clean_step(task, 0) node.refresh() # Make sure we go to CLEANFAIL, clear clean_steps self.assertEqual(states.CLEANFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual({}, node.clean_step) self.assertNotIn('clean_step_index', node.driver_internal_info) self.assertIsNotNone(node.last_error) self.assertEqual(1, tear_mock.call_count) self.assertTrue(node.maintenance) deploy_exec_calls = [ mock.call(mock.ANY, mock.ANY, self.clean_steps[0]), mock.call(mock.ANY, mock.ANY, self.clean_steps[2]), ] self.assertEqual(deploy_exec_calls, deploy_exec_mock.call_args_list) power_exec_calls = [ mock.call(mock.ANY, mock.ANY, self.clean_steps[1]), ] self.assertEqual(power_exec_calls, power_exec_mock.call_args_list) log_mock.exception.assert_called_once_with( 'Failed to tear down from cleaning for node {}, reason: boom'. format(node.uuid))
def _do_next_clean_step_all(self, mock_deploy_execute, mock_power_execute, mock_collect_logs, manual=False): # Run all steps from start to finish (all synchronous) tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.CLEANING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={ 'clean_steps': self.clean_steps, 'clean_step_index': None }, clean_step={}) def fake_deploy(conductor_obj, task, step): driver_internal_info = task.node.driver_internal_info driver_internal_info['goober'] = 'test' task.node.driver_internal_info = driver_internal_info task.node.save() mock_deploy_execute.side_effect = fake_deploy mock_power_execute.return_value = None with task_manager.acquire(self.context, node.uuid, shared=False) as task: cleaning.do_next_clean_step(task, 0) node.refresh() # Cleaning should be complete self.assertEqual(tgt_prov_state, node.provision_state) self.assertEqual(states.NOSTATE, node.target_provision_state) self.assertEqual({}, node.clean_step) self.assertNotIn('clean_step_index', node.driver_internal_info) self.assertEqual('test', node.driver_internal_info['goober']) self.assertIsNone(node.driver_internal_info['clean_steps']) mock_power_execute.assert_called_once_with(mock.ANY, mock.ANY, self.clean_steps[1]) mock_deploy_execute.assert_has_calls([ mock.call(mock.ANY, mock.ANY, self.clean_steps[0]), mock.call(mock.ANY, mock.ANY, self.clean_steps[2]) ]) self.assertFalse(mock_collect_logs.called)
def _do_next_clean_step_execute_fail(self, tear_mock, mock_execute, mock_collect_logs, manual=False): # When a clean step fails, go to CLEANFAIL tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.CLEANING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={ 'clean_steps': self.clean_steps, 'clean_step_index': None }, clean_step={}) mock_execute.side_effect = Exception() with task_manager.acquire(self.context, node.uuid, shared=False) as task: cleaning.do_next_clean_step(task, 0) tear_mock.assert_called_once_with(task.driver.deploy, task) node.refresh() # Make sure we go to CLEANFAIL, clear clean_steps self.assertEqual(states.CLEANFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual({}, node.clean_step) self.assertNotIn('clean_step_index', node.driver_internal_info) self.assertIsNotNone(node.last_error) self.assertTrue(node.maintenance) mock_execute.assert_called_once_with(mock.ANY, mock.ANY, self.clean_steps[0]) mock_collect_logs.assert_called_once_with(mock.ANY, label='cleaning')
def _do_next_clean_step_bad_step_return_value(self, deploy_exec_mock, power_exec_mock, manual=False): # When a clean step fails, go to CLEANFAIL tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.CLEANING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info={ 'clean_steps': self.clean_steps, 'clean_step_index': None }, clean_step={}) deploy_exec_mock.return_value = "foo" with task_manager.acquire(self.context, node.uuid, shared=False) as task: cleaning.do_next_clean_step(task, 0) node.refresh() # Make sure we go to CLEANFAIL, clear clean_steps self.assertEqual(states.CLEANFAIL, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual({}, node.clean_step) self.assertNotIn('clean_step_index', node.driver_internal_info) self.assertIsNotNone(node.last_error) self.assertTrue(node.maintenance) deploy_exec_mock.assert_called_once_with(mock.ANY, mock.ANY, self.clean_steps[0]) # Make sure we don't execute any other step and return self.assertFalse(power_exec_mock.called)
def _do_next_clean_step_first_step_async(self, return_state, mock_execute, clean_steps=None): # Execute the first async clean step on a node driver_internal_info = {'clean_step_index': None} if clean_steps: tgt_prov_state = states.MANAGEABLE driver_internal_info['clean_steps'] = clean_steps else: tgt_prov_state = states.AVAILABLE driver_internal_info['clean_steps'] = self.clean_steps node = obj_utils.create_test_node( self.context, driver='fake-hardware', provision_state=states.CLEANING, target_provision_state=tgt_prov_state, last_error=None, driver_internal_info=driver_internal_info, clean_step={}) mock_execute.return_value = return_state expected_first_step = node.driver_internal_info['clean_steps'][0] with task_manager.acquire(self.context, node.uuid, shared=False) as task: cleaning.do_next_clean_step(task, 0) node.refresh() self.assertEqual(states.CLEANWAIT, node.provision_state) self.assertEqual(tgt_prov_state, node.target_provision_state) self.assertEqual(expected_first_step, node.clean_step) self.assertEqual(0, node.driver_internal_info['clean_step_index']) mock_execute.assert_called_once_with(mock.ANY, mock.ANY, expected_first_step)