def test_do_scale_in_no_pd_no_inputs(self, mock_count, mock_delete,
mock_select, mock_load):
cluster = mock.Mock(id='CID', min_size=1, max_size=-1)
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id,
'CLUSTER_ACTION',
self.ctx,
data={},
inputs={})
mock_count.return_value = 10
mock_delete.return_value = (action.RES_OK, 'Life is beautiful.')
# do it
res_code, res_msg = action.do_scale_in()
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('Cluster scaling succeeded.', res_msg)
# deleting 1 nodes
mock_count.assert_called_once_with(action.context, 'CID')
mock_delete.assert_called_once_with(mock.ANY)
mock_select.assert_called_once_with(cluster.nodes, 1)
cluster.set_status.assert_called_once_with(action.context,
consts.CS_RESIZING,
'Cluster scale in started.',
desired_capacity=9)
cluster.eval_status.assert_called_once_with(action.context,
consts.CLUSTER_SCALE_IN)
def test_do_update_multi(self, mock_update, mock_load):
node1 = mock.Mock(id='fake id 1')
node2 = mock.Mock(id='fake id 2')
cluster = mock.Mock(id='FAKE_ID',
nodes=[node1, node2],
ACTIVE='ACTIVE')
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
action.inputs = {
'name': 'FAKE_NAME',
'metadata': {
'foo': 'bar'
},
'timeout': 3600,
'new_profile_id': 'FAKE_PROFILE'
}
reason = 'Cluster update completed.'
mock_update.return_value = (action.RES_OK, reason)
# do it
res_code, res_msg = action.do_update()
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual(reason, res_msg)
mock_update.assert_called_once_with('FAKE_PROFILE', [node1, node2])
def test_do_scale_in_failed_delete_nodes(self, mock_count, mock_delete,
mock_select, mock_load):
cluster = mock.Mock(id='CID', min_size=1, max_size=-1)
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id,
'CLUSTER_ACTION',
self.ctx,
data={},
inputs={'count': 2})
mock_count.return_value = 5
# Error cases
for result in (action.RES_ERROR, action.RES_CANCEL, action.RES_TIMEOUT,
action.RES_RETRY):
mock_delete.return_value = result, 'Too cold to work!'
# do it
res_code, res_msg = action.do_scale_in()
# assertions
self.assertEqual(result, res_code)
self.assertEqual('Too cold to work!', res_msg)
cluster.set_status.assert_called_once_with(
action.context,
consts.CS_RESIZING,
'Cluster scale in started.',
desired_capacity=3)
cluster.eval_status.assert_called_once_with(
action.context, consts.CLUSTER_SCALE_IN)
cluster.set_status.reset_mock()
cluster.eval_status.reset_mock()
mock_delete.assert_called_once_with(mock.ANY)
mock_delete.reset_mock()
def test_update_nodes_batch_policy(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
node1 = mock.Mock(id='node_id1')
node2 = mock.Mock(id='node_id2')
cluster = mock.Mock(id='FAKE_ID',
nodes=[node1, node2],
ACTIVE='ACTIVE')
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
action.inputs = {'new_profile_id': 'FAKE_PROFILE'}
action.id = 'CLUSTER_ACTION_ID'
action.data = {
'update': {
'pause_time': 0.1,
'min_in_service': 1,
'plan': [{node1.id}, {node2.id}],
}
}
mock_wait.return_value = (action.RES_OK, 'All dependents completed')
mock_action.side_effect = ['NODE_ACTION1', 'NODE_ACTION2']
res_code, reason = action._update_nodes('FAKE_PROFILE', [node1, node2])
self.assertEqual(res_code, action.RES_OK)
self.assertEqual(reason, 'Cluster update completed.')
self.assertEqual(2, mock_action.call_count)
self.assertEqual(2, mock_dep.call_count)
self.assertEqual(2, mock_update.call_count)
self.assertEqual(2, mock_start.call_count)
cluster.eval_status.assert_called_once_with(action.context,
consts.CLUSTER_UPDATE,
profile_id='FAKE_PROFILE',
updated_at=mock.ANY)
def test_do_del_nodes_failed_delete(self, mock_count, mock_delete,
mock_get, mock_load):
cluster = mock.Mock(id='FAKE_CLUSTER', min_size=0, max_size=5)
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id,
'CLUSTER_ACTION',
self.ctx,
inputs={'candidates': ['NODE_1']},
data={})
node1 = mock.Mock(cluster_id='FAKE_CLUSTER')
mock_get.side_effect = [node1]
mock_count.return_value = 3
mock_delete.return_value = (action.RES_ERROR, 'Things went bad.')
# do it
res_code, res_msg = action.do_del_nodes()
# assertions
self.assertEqual(action.RES_ERROR, res_code)
self.assertEqual("Things went bad.", res_msg)
mock_load.assert_called_once_with(action.context, 'FAKE_CLUSTER')
mock_get.assert_called_once_with(action.context, 'NODE_1')
mock_count.assert_called_once_with(action.context, 'FAKE_CLUSTER')
cluster.eval_status.assert_called_once_with(action.context,
consts.CLUSTER_DEL_NODES,
desired_capacity=2)
def test_do_del_nodes_node_not_member(self, mock_get, mock_load):
cluster = mock.Mock(id='FAKE_CLUSTER')
mock_load.return_value = cluster
action = ca.ClusterAction('ID',
'CLUSTER_ACTION',
self.ctx,
inputs={'candidates': ['NODE_1', 'NODE_2']})
node1 = mock.Mock(cluster_id='')
node2 = mock.Mock(cluster_id='ANOTHER_CLUSTER')
mock_get.side_effect = [node1, node2]
# do it
res_code, res_msg = action.do_del_nodes()
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual("Completed deleting nodes.", res_msg)
expected = {
'deletion': {
'destroy_after_deletion': False,
'grace_period': 0,
'reduce_desired_capacity': True,
}
}
self.assertEqual(expected, action.data)
def test_do_replace_nodes_node_not_active(self, mock_get_node, mock_load):
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=10)
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'candidates': {'ORIGIN_NODE': 'REPLACE_NODE'}}
action.outputs = {}
origin_node = mock.Mock(id='ORIGIN_NODE',
cluster_id='CLUSTER_ID',
ACTIVE='ACTIVE',
status='ACTIVE')
replace_node = mock.Mock(id='REPLACE_NODE',
cluster_id='',
ACTIVE='ACTIVE',
status='ERROR')
mock_get_node.side_effect = [origin_node, replace_node]
# do it
res_code, res_msg = action.do_replace_nodes()
# assertions
self.assertEqual(action.RES_ERROR, res_code)
self.assertEqual("Node REPLACE_NODE is not in ACTIVE status.", res_msg)
def test_do_add_nodes_failed_check(self, mock_count, mock_get, mock_load):
cluster = mock.Mock(id='CID', min_size=1, max_size=2)
mock_load.return_value = cluster
node1 = mock.Mock(id='nid1',
cluster_id='',
ACTIVE='ACTIVE',
status='ACTIVE')
node2 = mock.Mock(id='nid2',
cluster_id='',
ACTIVE='ACTIVE',
status='ACTIVE')
mock_get.side_effect = [node1, node2]
inputs = {'nodes': [node1.id, node2.id]}
action = ca.ClusterAction(cluster.id,
'CLUSTER_ACTION',
self.ctx,
data={},
inputs=inputs)
mock_count.return_value = 1
# execute
res_code, res_msg = action.do_add_nodes()
# assertions
self.assertEqual(action.RES_ERROR, res_code)
self.assertEqual(
"The target capacity (3) is greater than the "
"cluster's max_size (2).", res_msg)
self.assertEqual(2, mock_get.call_count)
mock_count.assert_called_once_with(action.context, 'CID')
def test__delete_nodes_multi(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100)
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
mock_wait.return_value = (action.RES_OK, 'All dependents completed')
mock_action.side_effect = ['NODE_ACTION_1', 'NODE_ACTION_2']
# do it
res_code, res_msg = action._delete_nodes(['NODE_1', 'NODE_2'])
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('All dependents completed', res_msg)
self.assertEqual(2, mock_action.call_count)
update_calls = [
mock.call(action.context, 'NODE_ACTION_1', {'status': 'READY'}),
mock.call(action.context, 'NODE_ACTION_2', {'status': 'READY'})
]
mock_update.assert_has_calls(update_calls)
self.assertEqual(1, mock_dep.call_count)
mock_start.assert_called_once_with()
mock_wait.assert_called_once_with()
self.assertEqual({'nodes_removed': ['NODE_1', 'NODE_2']},
action.outputs)
cluster.remove_node.assert_has_calls(
[mock.call('NODE_1'), mock.call('NODE_2')])
def test_delete_nodes_with_lifecycle_hook_unsupported_webhook(self,
mock_action,
mock_update,
mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100, config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.data = {
'hooks': {
'timeout': 10,
'type': 'webhook',
'params': {
'queue': 'myqueue'
}
}
}
mock_action.return_value = 'NODE_ACTION_ID'
# do it
res_code, res_msg = action._delete_nodes(['NODE_ID'])
# assertions (other assertions are skipped)
self.assertEqual(action.RES_ERROR, res_code)
self.assertEqual("Failed in deleting nodes: Lifecycle hook type "
"'webhook' is not implemented", res_msg)
def test_delete_nodes_failed_remove_stop_node(self, mock_remove,
mock_load):
# prepare mocks
cluster = mock.Mock(id='ID',
config={'cluster.stop_node_before_delete': True})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
action.data = {}
mock_remove.side_effect = [(action.RES_TIMEOUT, 'Timeout!'),
(action.RES_OK, 'OK')]
# do it
res_code, res_msg = action._delete_nodes(['NODE_ID'])
# assertions (other assertions are skipped)
self.assertEqual(action.RES_OK, res_code)
self.assertEqual({}, action.data)
remove_calls = [
mock.call('NODE_OPERATION', ['NODE_ID'],
{'operation': 'stop', 'update_parent_status': False}),
mock.call('NODE_DELETE', ['NODE_ID']),
]
mock_remove.assert_has_calls(remove_calls)
def test_do_resize_shrink(self, mock_delete, mock_sleep, mock_select,
mock_size, mock_count, mock_load):
cluster = mock.Mock(id='CID', nodes=[], RESIZING='RESIZING')
for n in range(10):
node = mock.Mock(id='NODE-ID-%s' % (n + 1))
cluster.nodes.append(node)
mock_load.return_value = cluster
mock_count.return_value = 10
action = ca.ClusterAction(cluster.id,
'CLUSTER_RESIZE',
self.ctx,
data={
'deletion': {
'count': 2,
'grace_period': 2,
'destroy_after_deletion': True
}
})
mock_delete.return_value = (action.RES_OK, 'All dependents completed.')
# do it
res_code, res_msg = action.do_resize()
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('Cluster resize succeeded.', res_msg)
mock_select.assert_called_once_with(cluster.nodes, 2)
mock_size.assert_called_once_with(8)
mock_sleep.assert_called_once_with(2)
mock_delete.assert_called_once_with(mock_select.return_value)
cluster.eval_status.assert_called_once_with(action.context,
consts.CLUSTER_RESIZE)
def test_delete_nodes_with_pd(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100, config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
action.data = {
'deletion': {
'destroy_after_deletion': False
}
}
mock_wait.return_value = (action.RES_OK, 'All dependents completed')
mock_action.return_value = 'NODE_ACTION_ID'
# do it
res_code, res_msg = action._delete_nodes(['NODE_ID'])
# assertions (other assertions are skipped)
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('All dependents completed', res_msg)
mock_action.assert_called_once_with(
action.context, 'NODE_ID', 'NODE_LEAVE',
name='node_delete_NODE_ID', cause='Derived Action', inputs={})
def test__delete_nodes_single(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='FAKE_CLUSTER', desired_capacity=100)
# cluster action is real
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
mock_wait.return_value = (action.RES_OK, 'All dependents completed')
mock_action.return_value = 'NODE_ACTION_ID'
# do it
res_code, res_msg = action._delete_nodes(['NODE_ID'])
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('All dependents completed', res_msg)
mock_action.assert_called_once_with(action.context,
'NODE_ID',
'NODE_DELETE',
name='node_delete_NODE_ID',
cause='Derived Action')
mock_dep.assert_called_once_with(action.context, ['NODE_ACTION_ID'],
'CLUSTER_ACTION_ID')
mock_update.assert_called_once_with(action.context, 'NODE_ACTION_ID',
{'status': 'READY'})
mock_start.assert_called_once_with()
mock_wait.assert_called_once_with()
self.assertEqual(['NODE_ID'], action.outputs['nodes_removed'])
cluster.remove_node.assert_called_once_with('NODE_ID')
def test_execute_failed_execute(self, mock_release, mock_acquire,
mock_load):
cluster = mock.Mock()
cluster.id = 'CLUSTER_ID'
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'ACTION_ID'
mock_acquire.return_value = action
self.patchobject(action,
'_execute',
return_value=(action.RES_ERROR, 'Failed execution.'))
res_code, res_msg = action.execute()
self.assertEqual(action.RES_ERROR, res_code)
self.assertEqual('Failed execution.', res_msg)
mock_load.assert_has_calls([
mock.call(action.context, cluster.id),
mock.call(action.context, cluster.id)
])
mock_acquire.assert_called_once_with(self.ctx, 'CLUSTER_ID',
'ACTION_ID', None,
senlin_lock.CLUSTER_SCOPE, True)
mock_release.assert_called_once_with('CLUSTER_ID', 'ACTION_ID',
senlin_lock.CLUSTER_SCOPE)
def test_do_resize_shrink_with_parsing(self, mock_delete, mock_parse,
mock_sleep, mock_select, mock_size,
mock_count, mock_load):
def fake_parse(*args, **kwargs):
# side effect
action.data = {'deletion': {'count': 1}}
return action.RES_OK, ''
cluster = mock.Mock(id='CID', nodes=[], RESIZING='RESIZING')
for n in range(10):
node = mock.Mock(id='NODE-ID-%s' % (n + 1))
cluster.nodes.append(node)
mock_count.return_value = 10
mock_load.return_value = cluster
mock_parse.side_effect = fake_parse
action = ca.ClusterAction(cluster.id,
'CLUSTER_RESIZE',
self.ctx,
inputs={'blah': 'blah'},
data={})
mock_delete.return_value = (action.RES_OK, 'All dependents completed.')
# deletion policy is attached to the action
res_code, res_msg = action.do_resize()
self.assertEqual({'deletion': {'count': 1}}, action.data)
mock_parse.assert_called_once_with(action, cluster, 10)
mock_select.assert_called_once_with(cluster.nodes, 1)
mock_size.assert_called_once_with(9)
mock_sleep.assert_called_once_with(0)
mock_delete.assert_called_once_with(mock_select.return_value)
cluster.eval_status.assert_called_once_with(action.context,
consts.CLUSTER_RESIZE)
def test_delete_nodes_with_error_nodes(self, mock_wait, mock_start,
mock_post, mock_dep,
mock_node_get, mock_action,
mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100)
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.data = {
'hooks': {
'timeout': 10,
'type': 'zaqar',
'params': {
'queue': 'myqueue'
}
}
}
action.owner = 'OWNER_ID'
mock_action.side_effect = ['NODE_ACTION_1', 'NODE_ACTION_2']
mock_wait.return_value = (action.RES_OK, 'All dependents completed')
node1 = mock.Mock(status=consts.NS_ACTIVE, id='NODE_1',
physical_id=None)
node2 = mock.Mock(status=consts.NS_ACTIVE, id='NODE_2',
physical_id="nova-server-1")
mock_node_get.side_effect = [node1, node2]
# do it
res_code, res_msg = action._remove_nodes_with_hook(
'NODE_DELETE', ['NODE_1', 'NODE_2'], action.data['hooks'])
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('All dependents completed', res_msg)
update_calls = [
mock.call(action.context, 'NODE_ACTION_1', {'status': 'READY',
'owner': None}),
mock.call(action.context, 'NODE_ACTION_2',
{'status': 'WAITING_LIFECYCLE_COMPLETION',
'owner': 'OWNER_ID'})
]
mock_update.assert_has_calls(update_calls)
create_actions = [
mock.call(action.context, 'NODE_1', 'NODE_DELETE',
name='node_delete_NODE_1',
cause='Derived Action with Lifecycle Hook', inputs={}),
mock.call(action.context, 'NODE_2', 'NODE_DELETE',
name='node_delete_NODE_2',
cause='Derived Action with Lifecycle Hook', inputs={})
]
mock_action.assert_has_calls(create_actions)
mock_post.assert_called_once_with('NODE_ACTION_2', 'NODE_2',
node2.physical_id,
consts.LIFECYCLE_NODE_TERMINATION)
mock_start.assert_called_once_with()
mock_wait.assert_called_once_with(action.data['hooks']['timeout'])
self.assertEqual(1, mock_dep.call_count)
def test_do_operation(self, mock_wait, mock_start, mock_dep, mock_action,
mock_update, mock_load):
cluster = mock.Mock(id='FAKE_ID')
cluster.do_operation.return_value = True
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_OPERATION', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {
'operation': 'dance',
'params': {
'style': 'tango'
},
'nodes': ['NODE_ID_1', 'NODE_ID_2'],
}
mock_action.side_effect = ['NODE_OP_ID_1', 'NODE_OP_ID_2']
mock_wait.return_value = (action.RES_OK, 'Everything is Okay')
# do it
res_code, res_msg = action.do_operation()
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual("Cluster operation 'dance' completed.", res_msg)
cluster.do_operation.assert_called_once_with(action.context,
operation='dance')
mock_action.assert_has_calls([
mock.call(action.context,
'NODE_ID_1',
'NODE_OPERATION',
name='node_dance_NODE_ID_',
cause=consts.CAUSE_DERIVED,
inputs={
'operation': 'dance',
'params': {
'style': 'tango'
}
}),
mock.call(action.context,
'NODE_ID_2',
'NODE_OPERATION',
name='node_dance_NODE_ID_',
cause=consts.CAUSE_DERIVED,
inputs={
'operation': 'dance',
'params': {
'style': 'tango'
}
}),
])
mock_dep.assert_called_once_with(action.context,
['NODE_OP_ID_1', 'NODE_OP_ID_2'],
'CLUSTER_ACTION_ID')
mock_update.assert_has_calls([
mock.call(action.context, 'NODE_OP_ID_1', {'status': 'READY'}),
mock.call(action.context, 'NODE_OP_ID_2', {'status': 'READY'}),
])
mock_start.assert_called_once_with()
mock_wait.assert_called_once_with()
cluster.eval_status.assert_called_once_with(action.context, 'dance')
def test_do_replace_nodes_in_other_cluster(self, mock_get_node, mock_load):
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=10)
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'ORIGIN_NODE': 'REPLACE_NODE'}
action.outputs = {}
origin_node = mock.Mock(id='ORIGIN_NODE',
cluster_id='CLUSTER_ID',
ACTIVE='ACTIVE',
status='ACTIVE')
replace_node = mock.Mock(id='REPLACE_NODE',
cluster_id='FAKE_CLUSTER',
ACTIVE='ACTIVE',
status='ACTIVE')
mock_get_node.side_effect = [origin_node, replace_node]
# do it
res_code, res_msg = action.do_replace_nodes()
# assertions
self.assertEqual(action.RES_ERROR, res_code)
self.assertEqual(
'Node REPLACE_NODE is already owned by cluster '
'FAKE_CLUSTER.', res_msg)
def test_execute_post_check_failed(self, mock_load):
def fake_check(cluster_id, target):
if target == 'BEFORE':
action.data = {
'status': pb.CHECK_OK,
'reason': 'Policy checking passed.'
}
else:
action.data = {
'status': pb.CHECK_ERROR,
'reason': 'Policy checking failed.'
}
cluster = mock.Mock()
cluster.id = 'FAKE_CLUSTER'
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_FLY', self.ctx)
action.do_fly = mock.Mock(return_value=(action.RES_OK, 'Cool!'))
mock_check = self.patchobject(action,
'policy_check',
side_effect=fake_check)
res_code, res_msg = action._execute()
self.assertEqual(action.RES_ERROR, res_code)
self.assertEqual('Policy check failure: Policy checking failed.',
res_msg)
mock_check.assert_has_calls([
mock.call('FAKE_CLUSTER', 'BEFORE'),
mock.call('FAKE_CLUSTER', 'AFTER')
])
def test_do_recover_failed_waiting(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
node = mock.Mock(id='NODE_1', cluster_id='CID', status='ERROR')
cluster = mock.Mock(id='CID')
cluster.do_recover.return_value = True
cluster.nodes = [node]
mock_load.return_value = cluster
mock_action.return_value = 'NODE_ACTION_ID'
action = ca.ClusterAction('FAKE_CLUSTER', 'CLUSTER_REOVER', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
mock_wait.return_value = (action.RES_TIMEOUT, 'Timeout!')
res_code, res_msg = action.do_recover()
self.assertEqual(action.RES_TIMEOUT, res_code)
self.assertEqual('Timeout!', res_msg)
mock_load.assert_called_once_with(self.ctx, 'FAKE_CLUSTER')
cluster.do_recover.assert_called_once_with(action.context)
mock_action.assert_called_once_with(action.context,
'NODE_1',
'NODE_RECOVER',
name='node_recover_NODE_1',
cause=consts.CAUSE_DERIVED,
inputs={})
mock_dep.assert_called_once_with(action.context, ['NODE_ACTION_ID'],
'CLUSTER_ACTION_ID')
mock_update.assert_called_once_with(action.context, 'NODE_ACTION_ID',
{'status': 'READY'})
mock_start.assert_called_once_with()
mock_wait.assert_called_once_with()
cluster.eval_status.assert_called_once_with(action.context,
consts.CLUSTER_RECOVER)
def test_do_scale_out_no_pd_with_inputs(self, mock_count, mock_create,
mock_load):
cluster = mock.Mock(id='CID', min_size=1, max_size=-1)
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id,
'CLUSTER_ACTION',
self.ctx,
data={},
inputs={'count': 2})
mock_count.return_value = 8
mock_create.return_value = (action.RES_OK, 'Life is beautiful.')
# do it
res_code, res_msg = action.do_scale_out()
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('Cluster scaling succeeded.', res_msg)
# creating 2 nodes, given that the cluster is empty now
mock_count.assert_called_once_with(action.context, 'CID')
mock_create.assert_called_once_with(2)
cluster.set_status.assert_called_once_with(
action.context,
consts.CS_RESIZING,
'Cluster scale out started.',
desired_capacity=10)
cluster.eval_status.assert_called_once_with(action.context,
consts.CLUSTER_SCALE_OUT)
def test_delete_nodes_with_lifecycle_hook_failed_remove_stop_node(
self, mock_remove_normally, mock_remove_hook, mock_load):
# prepare mocks
cluster = mock.Mock(id='ID',
config={'cluster.stop_node_before_delete': True})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
lifecycle_hook = {
'timeout': 10,
'type': 'zaqar',
'params': {
'queue': 'myqueue'
}
}
action.data = {
'hooks': lifecycle_hook,
}
mock_remove_hook.return_value = (action.RES_TIMEOUT, 'Timeout!')
mock_remove_normally.return_value = (action.RES_OK, '')
# do it
res_code, res_msg = action._delete_nodes(['NODE_ID'])
# assertions (other assertions are skipped)
self.assertEqual(action.RES_OK, res_code)
mock_remove_hook.assert_called_once_with(
'NODE_OPERATION', ['NODE_ID'], lifecycle_hook,
{'operation': 'stop', 'update_parent_status': False})
mock_remove_normally.assert_called_once_with('NODE_DELETE',
['NODE_ID'])
def test_do_resize_shrink_failed_delete(self, mock_delete, mock_size,
mock_count, mock_load):
cluster = mock.Mock(id='CLID', nodes=[], RESIZING='RESIZING')
mock_count.return_value = 3
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id,
'CLUSTER_RESIZE',
self.ctx,
data={
'deletion': {
'count': 2,
'grace_period': 2,
'candidates': ['NODE1', 'NODE2']
}
})
mock_delete.return_value = (action.RES_ERROR, 'Bad things happened.')
# do it
res_code, res_msg = action.do_resize()
# assertions
self.assertEqual(action.RES_ERROR, res_code)
self.assertEqual('Bad things happened.', res_msg)
mock_size.assert_called_once_with(1)
mock_delete.assert_called_once_with(['NODE1', 'NODE2'])
cluster.eval_status.assert_called_once_with(action.context,
consts.CLUSTER_RESIZE)
def test_remove_nodes_hook_failed_wait(self, mock_wait, mock_start,
mock_dep, mock_action,
mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='ID', config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
action.data = {
'hooks': {
'timeout': 10,
'type': 'zaqar',
'params': {
'queue': 'myqueue'
}
}
}
mock_wait.return_value = (action.RES_TIMEOUT, 'Timeout!')
mock_action.return_value = 'NODE_ACTION_ID'
# do it
res_code, res_msg = action._remove_nodes_normally('NODE_REMOVE',
['NODE_ID'])
# assertions (other assertions are skipped)
self.assertEqual(action.RES_TIMEOUT, res_code)
self.assertEqual('Timeout!', res_msg)
def test_do_resize_grow(self, mock_create, mock_size, mock_count,
mock_load):
cluster = mock.Mock(id='ID', nodes=[], RESIZING='RESIZING')
mock_load.return_value = cluster
mock_count.return_value = 10
action = ca.ClusterAction(cluster.id,
'CLUSTER_ACTION',
self.ctx,
data={'creation': {
'count': 2
}})
mock_create.return_value = (action.RES_OK, 'All dependents completed.')
# do it
res_code, res_msg = action.do_resize()
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('Cluster resize succeeded.', res_msg)
mock_size.assert_called_once_with(12)
mock_create.assert_called_once_with(2)
cluster.eval_status.assert_called_once_with(action.context,
consts.CLUSTER_RESIZE)
def test_do_delete_success(self, mock_action, mock_load):
node1 = mock.Mock(id='NODE_1')
node2 = mock.Mock(id='NODE_2')
cluster = mock.Mock(id='FAKE_CLUSTER',
nodes=[node1, node2],
DELETING='DELETING')
cluster.do_delete.return_value = True
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.data = {}
mock_delete = self.patchobject(action,
'_delete_nodes',
return_value=(action.RES_OK, 'Good'))
# do it
res_code, res_msg = action.do_delete()
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('Good', res_msg)
self.assertEqual({'deletion': {
'destroy_after_deletion': True
}}, action.data)
cluster.set_status.assert_called_once_with(action.context, 'DELETING',
'Deletion in progress.')
mock_delete.assert_called_once_with(['NODE_1', 'NODE_2'])
cluster.do_delete.assert_called_once_with(action.context)
mock_action.assert_called_once_with(action.context,
'FAKE_CLUSTER',
action_excluded=['CLUSTER_DELETE'],
status=['SUCCEEDED', 'FAILED'])
def test_do_resize_grow_with_parsing(self, mock_create, mock_parse,
mock_size, mock_count, mock_load):
def fake_parse(*args, **kwargs):
action.data = {'creation': {'count': 3}}
return action.RES_OK, ''
cluster = mock.Mock(id='ID', nodes=[], RESIZING='RESIZING')
mock_load.return_value = cluster
mock_count.return_value = 10
mock_parse.side_effect = fake_parse
action = ca.ClusterAction(cluster.id,
'CLUSTER_ACTION',
self.ctx,
data={},
inputs={'blah': 'blah'})
mock_create.return_value = (action.RES_OK, 'All dependents completed.')
# do it
res_code, res_msg = action.do_resize()
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('Cluster resize succeeded.', res_msg)
mock_parse.assert_called_once_with(action, cluster, 10)
mock_size.assert_called_once_with(13)
mock_create.assert_called_once_with(3)
cluster.eval_status.assert_called_once_with(action.context,
consts.CLUSTER_RESIZE)
def test_do_scale_in_with_pd_no_input(self, mock_count, mock_delete,
mock_sleep, mock_load):
cluster = mock.Mock(id='CID', min_size=1, max_size=-1)
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
action.data = {
'deletion': {
'count': 2,
'grace_period': 2,
'candidates': ['NODE_ID_3', 'NODE_ID_4'],
}
}
action.inputs = {}
mock_count.return_value = 5
mock_delete.return_value = (action.RES_OK, 'Life is beautiful.')
# do it
res_code, res_msg = action.do_scale_in()
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('Cluster scaling succeeded.', res_msg)
# deleting 2 nodes
mock_count.assert_called_once_with(action.context, 'CID')
mock_delete.assert_called_once_with(mock.ANY)
self.assertEqual(2, len(mock_delete.call_args[0][0]))
self.assertIn('NODE_ID_3', mock_delete.call_args[0][0])
self.assertIn('NODE_ID_4', mock_delete.call_args[0][0])
cluster.set_status.assert_called_once_with(
action.context, consts.CS_RESIZING, 'Cluster scale in started.',
desired_capacity=3)
cluster.eval_status.assert_called_once_with(
action.context, consts.CLUSTER_SCALE_IN)
mock_sleep.assert_called_once_with(2)
def test_delete_nodes_with_lifecycle_hook_timeout(
self, mock_wait, mock_start, mock_post, mock_dep, mock_node_get,
mock_action, mock_update, mock_check_status, mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100, config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.data = {
'hooks': {
'timeout': 10,
'type': 'zaqar',
'params': {
'queue': 'myqueue'
}
}
}
action.owner = 'OWNER_ID'
mock_wait.side_effect = [(action.RES_LIFECYCLE_HOOK_TIMEOUT,
'Timeout'),
(action.RES_OK, 'All dependents completed')]
mock_action.return_value = 'NODE_ACTION_ID'
mock_node_get.return_value = mock.Mock(status=consts.NS_ACTIVE,
id='NODE_ID',
physical_id="nova-server")
mock_check_status.return_value = 'WAITING_LIFECYCLE_COMPLETION'
# do it
res_code, res_msg = action._delete_nodes(['NODE_ID'])
# assertions (other assertions are skipped)
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('All dependents completed', res_msg)
self.assertEqual(1, mock_dep.call_count)
mock_action.assert_called_once_with(
action.context,
'NODE_ID',
'NODE_DELETE',
name='node_delete_NODE_ID',
cluster_id='CLUSTER_ID',
cause='Derived Action with Lifecycle Hook',
inputs={})
update_calls = [
mock.call(action.context, 'NODE_ACTION_ID', {
'status': 'WAITING_LIFECYCLE_COMPLETION',
'owner': 'OWNER_ID'
}),
mock.call(action.context, 'NODE_ACTION_ID', {
'status': 'READY',
'owner': None
}),
]
mock_update.assert_has_calls(update_calls)
mock_post.assert_called_once_with('NODE_ACTION_ID', 'NODE_ID',
'nova-server',
consts.LIFECYCLE_NODE_TERMINATION)
mock_start.assert_has_calls([mock.call(), mock.call()])
wait_calls = [mock.call(action.data['hooks']['timeout']), mock.call()]
mock_wait.assert_has_calls(wait_calls)