def test_do_next_clean_step_oob_reboot_last_step(self, mock_execute):
# Resume where last_step is the last cleaning step
tgt_prov_state = states.MANAGEABLE
info = {
'clean_steps': self.clean_steps,
'cleaning_reboot': True,
'clean_step_index': len(self.clean_steps) - 1
}
node = obj_utils.create_test_node(
self.context,
driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info=info,
clean_step=self.clean_steps[-1])
with task_manager.acquire(self.context, node.uuid,
shared=False) as task:
cleaning.do_next_clean_step(task, None)
node.refresh()
# Cleaning should be complete without calling additional steps
self.assertEqual(tgt_prov_state, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertNotIn('cleaning_reboot', node.driver_internal_info)
self.assertIsNone(node.driver_internal_info['clean_steps'])
self.assertFalse(mock_execute.called)
def test_do_next_clean_step_oob_reboot_fail(self, tear_mock, mock_execute):
# When a clean step fails with no reboot requested go to CLEANFAIL
tgt_prov_state = states.MANAGEABLE
node = obj_utils.create_test_node(
self.context,
driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={
'clean_steps': self.clean_steps,
'clean_step_index': None
},
clean_step={})
mock_execute.side_effect = exception.AgentConnectionFailed(
reason='failed')
with task_manager.acquire(self.context, node.uuid,
shared=False) as task:
cleaning.do_next_clean_step(task, 0)
tear_mock.assert_called_once_with(task.driver.deploy, task)
node.refresh()
# Make sure we go to CLEANFAIL, clear clean_steps
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertNotIn('skip_current_clean_step', node.driver_internal_info)
self.assertIsNotNone(node.last_error)
self.assertTrue(node.maintenance)
mock_execute.assert_called_once_with(mock.ANY, mock.ANY,
self.clean_steps[0])
def _do_next_clean_step_continue_from_last_cleaning(
self, return_state, mock_execute, manual=False):
# Resume an in-progress cleaning after the first async step
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context,
driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={
'clean_steps': self.clean_steps,
'clean_step_index': 0
},
clean_step=self.clean_steps[0])
mock_execute.return_value = return_state
with task_manager.acquire(self.context, node.uuid,
shared=False) as task:
cleaning.do_next_clean_step(task, self.next_clean_step_index)
node.refresh()
self.assertEqual(states.CLEANWAIT, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual(self.clean_steps[1], node.clean_step)
self.assertEqual(1, node.driver_internal_info['clean_step_index'])
mock_execute.assert_called_once_with(mock.ANY, mock.ANY,
self.clean_steps[1])
def test_do_next_clean_step_oob_reboot(self, mock_execute):
# When a clean step fails, go to CLEANWAIT
tgt_prov_state = states.MANAGEABLE
node = obj_utils.create_test_node(
self.context,
driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={
'clean_steps': self.clean_steps,
'clean_step_index': None,
'cleaning_reboot': True
},
clean_step={})
mock_execute.side_effect = exception.AgentConnectionFailed(
reason='failed')
with task_manager.acquire(self.context, node.uuid,
shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
# Make sure we go to CLEANWAIT
self.assertEqual(states.CLEANWAIT, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual(self.clean_steps[0], node.clean_step)
self.assertEqual(0, node.driver_internal_info['clean_step_index'])
self.assertFalse(node.driver_internal_info['skip_current_clean_step'])
mock_execute.assert_called_once_with(mock.ANY, mock.ANY,
self.clean_steps[0])
def _do_next_clean_step_no_steps(self,
mock_execute,
manual=False,
fast_track=False):
if fast_track:
self.config(fast_track=True, group='deploy')
for info in ({
'clean_steps': None,
'clean_step_index': None,
'agent_url': 'test-url',
'agent_secret_token': 'magic'
}, {
'clean_steps': None,
'agent_url': 'test-url',
'agent_secret_token': 'it_is_a_kind_of_magic'
}):
# Resume where there are no steps, should be a noop
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context,
driver='fake-hardware',
uuid=uuidutils.generate_uuid(),
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info=info,
clean_step={})
with task_manager.acquire(self.context, node.uuid,
shared=False) as task:
cleaning.do_next_clean_step(task, None)
node.refresh()
# Cleaning should be complete without calling additional steps
self.assertEqual(tgt_prov_state, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertFalse(mock_execute.called)
if fast_track:
self.assertEqual('test-url',
node.driver_internal_info.get('agent_url'))
self.assertIsNotNone(
node.driver_internal_info.get('agent_secret_token'))
else:
self.assertNotIn('agent_url', node.driver_internal_info)
self.assertNotIn('agent_secret_token',
node.driver_internal_info)
mock_execute.reset_mock()
def _do_next_clean_step_fail_in_tear_down_cleaning(self,
tear_mock,
power_exec_mock,
deploy_exec_mock,
log_mock,
manual=True):
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context,
driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={
'clean_steps': self.clean_steps,
'clean_step_index': None
},
clean_step={})
deploy_exec_mock.return_value = None
power_exec_mock.return_value = None
tear_mock.side_effect = Exception('boom')
with task_manager.acquire(self.context, node.uuid,
shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
# Make sure we go to CLEANFAIL, clear clean_steps
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertIsNotNone(node.last_error)
self.assertEqual(1, tear_mock.call_count)
self.assertTrue(node.maintenance)
deploy_exec_calls = [
mock.call(mock.ANY, mock.ANY, self.clean_steps[0]),
mock.call(mock.ANY, mock.ANY, self.clean_steps[2]),
]
self.assertEqual(deploy_exec_calls, deploy_exec_mock.call_args_list)
power_exec_calls = [
mock.call(mock.ANY, mock.ANY, self.clean_steps[1]),
]
self.assertEqual(power_exec_calls, power_exec_mock.call_args_list)
log_mock.exception.assert_called_once_with(
'Failed to tear down from cleaning for node {}, reason: boom'.
format(node.uuid))
def _do_next_clean_step_all(self,
mock_deploy_execute,
mock_power_execute,
mock_collect_logs,
manual=False):
# Run all steps from start to finish (all synchronous)
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context,
driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={
'clean_steps': self.clean_steps,
'clean_step_index': None
},
clean_step={})
def fake_deploy(conductor_obj, task, step):
driver_internal_info = task.node.driver_internal_info
driver_internal_info['goober'] = 'test'
task.node.driver_internal_info = driver_internal_info
task.node.save()
mock_deploy_execute.side_effect = fake_deploy
mock_power_execute.return_value = None
with task_manager.acquire(self.context, node.uuid,
shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
# Cleaning should be complete
self.assertEqual(tgt_prov_state, node.provision_state)
self.assertEqual(states.NOSTATE, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertEqual('test', node.driver_internal_info['goober'])
self.assertIsNone(node.driver_internal_info['clean_steps'])
mock_power_execute.assert_called_once_with(mock.ANY, mock.ANY,
self.clean_steps[1])
mock_deploy_execute.assert_has_calls([
mock.call(mock.ANY, mock.ANY, self.clean_steps[0]),
mock.call(mock.ANY, mock.ANY, self.clean_steps[2])
])
self.assertFalse(mock_collect_logs.called)
def _do_next_clean_step_execute_fail(self,
tear_mock,
mock_execute,
mock_collect_logs,
manual=False):
# When a clean step fails, go to CLEANFAIL
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context,
driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={
'clean_steps': self.clean_steps,
'clean_step_index': None
},
clean_step={})
mock_execute.side_effect = Exception()
with task_manager.acquire(self.context, node.uuid,
shared=False) as task:
cleaning.do_next_clean_step(task, 0)
tear_mock.assert_called_once_with(task.driver.deploy, task)
node.refresh()
# Make sure we go to CLEANFAIL, clear clean_steps
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertIsNotNone(node.last_error)
self.assertTrue(node.maintenance)
mock_execute.assert_called_once_with(mock.ANY, mock.ANY,
self.clean_steps[0])
mock_collect_logs.assert_called_once_with(mock.ANY, label='cleaning')
def _do_next_clean_step_bad_step_return_value(self,
deploy_exec_mock,
power_exec_mock,
manual=False):
# When a clean step fails, go to CLEANFAIL
tgt_prov_state = states.MANAGEABLE if manual else states.AVAILABLE
node = obj_utils.create_test_node(
self.context,
driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info={
'clean_steps': self.clean_steps,
'clean_step_index': None
},
clean_step={})
deploy_exec_mock.return_value = "foo"
with task_manager.acquire(self.context, node.uuid,
shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
# Make sure we go to CLEANFAIL, clear clean_steps
self.assertEqual(states.CLEANFAIL, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual({}, node.clean_step)
self.assertNotIn('clean_step_index', node.driver_internal_info)
self.assertIsNotNone(node.last_error)
self.assertTrue(node.maintenance)
deploy_exec_mock.assert_called_once_with(mock.ANY, mock.ANY,
self.clean_steps[0])
# Make sure we don't execute any other step and return
self.assertFalse(power_exec_mock.called)
def _do_next_clean_step_first_step_async(self,
return_state,
mock_execute,
clean_steps=None):
# Execute the first async clean step on a node
driver_internal_info = {'clean_step_index': None}
if clean_steps:
tgt_prov_state = states.MANAGEABLE
driver_internal_info['clean_steps'] = clean_steps
else:
tgt_prov_state = states.AVAILABLE
driver_internal_info['clean_steps'] = self.clean_steps
node = obj_utils.create_test_node(
self.context,
driver='fake-hardware',
provision_state=states.CLEANING,
target_provision_state=tgt_prov_state,
last_error=None,
driver_internal_info=driver_internal_info,
clean_step={})
mock_execute.return_value = return_state
expected_first_step = node.driver_internal_info['clean_steps'][0]
with task_manager.acquire(self.context, node.uuid,
shared=False) as task:
cleaning.do_next_clean_step(task, 0)
node.refresh()
self.assertEqual(states.CLEANWAIT, node.provision_state)
self.assertEqual(tgt_prov_state, node.target_provision_state)
self.assertEqual(expected_first_step, node.clean_step)
self.assertEqual(0, node.driver_internal_info['clean_step_index'])
mock_execute.assert_called_once_with(mock.ANY, mock.ANY,
expected_first_step)
