def test_webhook(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) # Wait and verify cluster creation result cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') self.assertEqual('test-cluster', cluster['name']) self.assertEqual(desired_capacity, cluster['desired_capacity']) self.assertEqual(min_size, cluster['min_size']) self.assertEqual(max_size, cluster['max_size']) self.assertEqual(desired_capacity, len(cluster['nodes'])) # Create a webhook targets on cluster resize action with params params = { 'adjustment_type': 'EXACT_CAPACITY', 'number': 2, } webhook = test_api.create_webhook(self.client, 'webhook-c-resize', 'cluster', cluster['id'], 'CLUSTER_RESIZE', params=params) webhook_url = webhook['url'] # Verify webhook params self.assertEqual('webhook-c-resize', webhook['name']) self.assertEqual('cluster', webhook['obj_type']) self.assertEqual(cluster['id'], webhook['obj_id']) self.assertEqual('CLUSTER_RESIZE', webhook['action']) self.assertEqual(params, webhook['params']) # Trigger webhook and wait for action complete action_id = test_api.trigger_webhook(webhook_url) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify action is as expected action = test_api.get_action(self.client, action_id) self.assertEqual('CLUSTER_RESIZE', action['action']) self.assertEqual(cluster['id'], action['target']) self.assertEqual(params, action['inputs']) # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(2, len(cluster['nodes'])) # Delete webhook test_api.delete_webhook(self.client, webhook['id']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE', ignore_missing=True)
def test_cluster_update_basic_properties(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 3 cluster_name = test_utils.random_name('cluster') cluster = test_api.create_cluster(self.client, cluster_name, self.profile['id'], desired_capacity, min_size, max_size, metadata={'k1': 'v1'}, timeout=120) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') self.assertEqual(cluster_name, cluster['name']) self.assertEqual({'k1': 'v1'}, cluster['metadata']) self.assertEqual(120, cluster['timeout']) # Update basic properties of cluster action_id = test_api.update_cluster(self.client, cluster['id'], name='cluster_new_name', metadata={'k2': 'v2'}, timeout=240) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify update result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('cluster_new_name', cluster['name']) self.assertEqual({'k2': 'v2'}, cluster['metadata']) self.assertEqual(240, cluster['timeout']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_webhook(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, test_utils.random_name('cluster'), self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Create a webhook receiver targets on cluster resize action # with params params = { 'adjustment_type': 'EXACT_CAPACITY', 'number': 2, } webhook_name = test_utils.random_name('webhook') res = test_api.create_receiver(self.client, webhook_name, cluster['id'], 'CLUSTER_RESIZE', 'webhook', params=params) r_webhook = test_api.get_receiver(self.client, res['id']) # Verify webhook params self.assertEqual(webhook_name, r_webhook['name']) self.assertEqual(cluster['id'], r_webhook['cluster_id']) self.assertEqual('CLUSTER_RESIZE', r_webhook['action']) self.assertEqual(params, r_webhook['params']) webhook_url = r_webhook['channel']['alarm_url'] # Trigger webhook and wait for action complete action_id = test_api.trigger_webhook(webhook_url) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify action is as expected action = test_api.get_action(self.client, action_id) self.assertEqual('CLUSTER_RESIZE', action['action']) self.assertEqual(cluster['id'], action['target']) self.assertEqual(params, action['inputs']) # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(2, len(cluster['nodes'])) # Delete webhook receiver test_api.delete_receiver(self.client, r_webhook['id']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_cluster_create_delete(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 3 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) # Wait and verify cluster creation result cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') self.assertEqual('test-cluster', cluster['name']) self.assertEqual(desired_capacity, cluster['desired_capacity']) self.assertEqual(min_size, cluster['min_size']) self.assertEqual(max_size, cluster['max_size']) self.assertEqual(desired_capacity, len(cluster['nodes'])) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_cluster_resize_with_invalid_constraint_provided(self): # Create cluster desired_capacity = 3 min_size = 1 max_size = 4 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # New min_size is larger than current cluster # max_size with strict set to False params = {'min_size': 5, 'strict': False} action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) # Wait for cluster resize action failed action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (4) is less than " "the specified min_size (5).") self.assertEqual(reason, action['status_reason']) # New max_size is less than current cluster # min_size with strict set to True params = {'max_size': 0, 'strict': True} action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) # Wait for cluster resize action failed action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (3) is greater " "than the specified max_size (0).") self.assertEqual(reason, action['status_reason']) # New min_size > new max_size params = {'min_size': 5, 'max_size': 3} msg = _("The specified min_size (5) is greater than the " "specified max_size (3).") ex = self.assertRaises(api_client.SenlinApiException, test_api.action_cluster, self.client, cluster['id'], 'resize', params) self.assertEqual(400, ex.response.status_code) self.assertIn(msg, ex.response.content) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_node_create_join_leave_cluster(self): # Create a cluster desired_capacity = 0 min_size = 0 max_size = 3 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Create a node and let it join the cluster node1 = test_api.create_node(self.client, 'node1', self.profile['id']) node1 = test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'ACTIVE') action_id = test_api.action_node(self.client, node1['id'], 'join', {'cluster_id': cluster['id']}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') node1 = test_api.get_node(self.client, node1['id']) self.assertEqual(cluster['id'], node1['cluster_id']) cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(1, len(cluster['nodes'])) self.assertIn(node1['id'], cluster['nodes']) # Let node1 leave the cluster action_id = test_api.action_node(self.client, node1['id'], 'leave') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') node1 = test_api.get_node(self.client, node1['id']) self.assertIsNone(node1['cluster_id']) self.assertEqual(-1, node1['index']) cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(0, len(cluster['nodes'])) # Delete node1 test_api.delete_node(self.client, node1['id']) test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'DELETED', ignore_missing=True) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_node_create_join_leave_cluster(self): # Create a cluster desired_capacity = 0 min_size = 0 max_size = 3 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Create a node and let it join the cluster node1 = test_api.create_node(self.client, 'node1', self.profile['id']) node1 = test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'ACTIVE') action_id = test_api.action_node(self.client, node1['id'], 'join', {'cluster_id': cluster['id']}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') node1 = test_api.get_node(self.client, node1['id']) self.assertEqual(cluster['id'], node1['cluster_id']) self.assertEqual(1, node1['index']) cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(1, len(cluster['nodes'])) self.assertIn(node1['id'], cluster['nodes']) # Let node1 leave the cluster action_id = test_api.action_node(self.client, node1['id'], 'leave') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') node1 = test_api.get_node(self.client, node1['id']) self.assertIsNone(node1['cluster_id']) self.assertEqual(-1, node1['index']) cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(0, len(cluster['nodes'])) # Delete node1 test_api.delete_node(self.client, node1['id']) test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'DELETED', ignore_missing=True) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_node_create_delete_with_target_cluster(self): # Create a cluster desired_capacity = 1 min_size = 1 max_size = 3 cluster = test_api.create_cluster(self.client, test_utils.random_name('cluster'), self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Create a node belongs to the cluster node1 = test_api.create_node(self.client, test_utils.random_name('node1'), self.profile['id'], cluster_id=cluster['id'], role='master') node1 = test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'ACTIVE') self.assertEqual(cluster['id'], node1['cluster_id']) self.assertEqual(2, node1['index']) cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(2, cluster['desired_capacity']) self.assertEqual(2, len(cluster['nodes'])) self.assertIn(node1['id'], cluster['nodes']) # List nodes nodes = test_api.list_nodes(self.client) nodes_id = [n['id'] for n in nodes] self.assertIn(node1['id'], nodes_id) for n_id in cluster['nodes']: self.assertIn(n_id, nodes_id) # Delete node1 from cluster test_api.delete_node(self.client, node1['id']) test_utils.wait_for_delete(test_api.get_node, self.client, node1['id']) cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(1, len(cluster['nodes'])) self.assertNotIn(node1['id'], cluster['nodes']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_node_check_recover(self): # Create a node node1 = test_api.create_node(self.client, test_utils.random_name('node1'), self.profile['id'], role='master') node1 = test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'ACTIVE') # Check node health status action_id = test_api.action_node(self.client, node1['id'], 'check') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') node1 = test_api.get_node(self.client, node1['id']) self.assertEqual('ACTIVE', node1['status']) # Perform node recovering operation action_id = test_api.action_node(self.client, node1['id'], 'recover') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') action_id = test_api.action_node(self.client, node1['id'], 'recover', {'operation': 'REBUILD'}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Delete node test_api.delete_node(self.client, node1['id']) test_utils.wait_for_delete(test_api.get_node, self.client, node1['id'])
def test_cluster_check_recover(self): # Create cluster desired_capacity = 3 min_size = 2 max_size = 5 cluster = test_api.create_cluster(self.client, test_utils.random_name('cluster'), self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Check cluster health status action_id = test_api.action_cluster(self.client, cluster['id'], 'check') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) # Perform cluster recovering operation action_id = test_api.action_cluster(self.client, cluster['id'], 'recover') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') action_id = test_api.action_cluster(self.client, cluster['id'], 'recover', {'operation': 'REBUILD'}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_webhook(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 5 cluster = test_api.create_cluster( self.client, "test-cluster", self.profile["id"], desired_capacity, min_size, max_size ) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster["id"], "ACTIVE") # Create a webhook targets on cluster resize action with params params = {"adjustment_type": "EXACT_CAPACITY", "number": 2} webhook = test_api.create_webhook( self.client, "webhook-c-resize", "cluster", cluster["id"], "CLUSTER_RESIZE", params=params ) webhook_url = webhook["url"] # Verify webhook params self.assertEqual("webhook-c-resize", webhook["name"]) self.assertEqual("cluster", webhook["obj_type"]) self.assertEqual(cluster["id"], webhook["obj_id"]) self.assertEqual("CLUSTER_RESIZE", webhook["action"]) self.assertEqual(params, webhook["params"]) # Trigger webhook and wait for action complete action_id = test_api.trigger_webhook(webhook_url) test_utils.wait_for_status(test_api.get_action, self.client, action_id, "SUCCEEDED") # Verify action is as expected action = test_api.get_action(self.client, action_id) self.assertEqual("CLUSTER_RESIZE", action["action"]) self.assertEqual(cluster["id"], action["target"]) self.assertEqual(params, action["inputs"]) # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster["id"]) self.assertEqual("ACTIVE", cluster["status"]) self.assertEqual(2, len(cluster["nodes"])) # Delete webhook test_api.delete_webhook(self.client, webhook["id"]) # Delete cluster test_api.delete_cluster(self.client, cluster["id"]) test_utils.wait_for_status(test_api.get_cluster, self.client, cluster["id"], "DELETED", ignore_missing=True)
def test_node_get_detail(self): node = test_api.create_node(self.client, test_utils.random_name('node'), self.profile['id']) test_utils.wait_for_status(test_api.get_node, self.client, node['id'], 'ACTIVE') # Get node detail and verify node = test_api.get_node(self.client, node['id'], show_details=True) self.assertIn('details', node) self.assertEqual("new-server-test", node['details']['name']) self.assertEqual("1", node['details']['flavor']) self.assertEqual("FAKE_IMAGE_ID", node['details']['image']) self.assertEqual("ACTIVE", node['details']['status']) self.assertEqual({"My Server Name": "Apache1"}, node['details']['metadata']) # Delete node from cluster test_api.delete_node(self.client, node['id']) test_utils.wait_for_delete(test_api.get_node, self.client, node['id'])
def test_cluster_update_profile(self): spec_nova_server = copy.deepcopy(test_utils.spec_nova_server) spec_nova_server['properties']['flavor'] = 'new_flavor' spec_nova_server['properties']['image'] = 'new_image' new_profile = test_api.create_profile( self.client, test_utils.random_name('profile'), spec_nova_server) # Create cluster with original profile desired_capacity = 2 min_size = 1 max_size = 3 cluster_name = test_utils.random_name('cluster') cluster = test_api.create_cluster(self.client, cluster_name, self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Update cluster with new profile action_id = test_api.update_cluster(self.client, cluster['id'], profile=new_profile['id']) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify update result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(new_profile['id'], cluster['profile_id']) nodes = cluster['nodes'] for n in nodes: node = test_api.get_node(self.client, n, show_details=True) self.assertEqual(new_profile['id'], node['profile_id']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id']) # Delete new profile test_api.delete_profile(self.client, new_profile['id'])
def test_cluster_create_delete(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 3 cluster_name = test_utils.random_name('cluster') cluster = test_api.create_cluster(self.client, cluster_name, self.profile['id'], desired_capacity, min_size, max_size) # Wait and verify cluster creation result cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') self.assertEqual(cluster_name, cluster['name']) self.assertEqual(desired_capacity, cluster['desired_capacity']) self.assertEqual(min_size, cluster['min_size']) self.assertEqual(max_size, cluster['max_size']) self.assertEqual(desired_capacity, len(cluster['nodes'])) # Create cluster2 desired_capacity = 1 min_size = 1 max_size = 3 cluster2_name = test_utils.random_name('cluster') cluster2 = test_api.create_cluster(self.client, cluster2_name, self.profile['id'], desired_capacity, min_size, max_size) # List clusters clusters = test_api.list_clusters(self.client) clusters_id = [c['id'] for c in clusters] self.assertIn(cluster['id'], clusters_id) self.assertIn(cluster2['id'], clusters_id) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_api.delete_cluster(self.client, cluster2['id']) test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id']) test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster2['id'])
def test_cluster_resize_with_invalid_constraint_provided(self): # Create cluster desired_capacity = 3 min_size = 1 max_size = 4 cluster = test_api.create_cluster(self.client, test_utils.random_name('cluster'), self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # New min_size is larger than current cluster # max_size with strict set to False params = {'min_size': 5, 'strict': False} res = test_api.action_cluster(self.client, cluster['id'], 'resize', params) reason = _("The specified min_size (5) is greater than the current " "max_size (4) of the cluster.") self.assertIn(reason, res) # New max_size is less than current cluster # min_size with strict set to True params = {'max_size': 0, 'strict': True} res = test_api.action_cluster(self.client, cluster['id'], 'resize', params) reason = _("The specified max_size (0) is less than the current " "min_size (1) of the cluster.") self.assertIn(reason, res) # New min_size > new max_size params = {'min_size': 5, 'max_size': 3} res = test_api.action_cluster(self.client, cluster['id'], 'resize', params) reason = _("The specified min_size (5) is greater than the " "specified max_size (3).") self.assertIn(reason, res) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_node_create_delete_with_target_cluster(self): # Create a cluster desired_capacity = 1 min_size = 1 max_size = 3 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Create a node belongs to the cluster node1 = test_api.create_node(self.client, 'node1', self.profile['id'], cluster_id=cluster['id'], role='master') node1 = test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'ACTIVE') self.assertEqual(cluster['id'], node1['cluster_id']) cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(2, len(cluster['nodes'])) self.assertIn(node1['id'], cluster['nodes']) # Delete node1 from cluster test_api.delete_node(self.client, node1['id']) test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'DELETED', ignore_missing=True) cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(1, len(cluster['nodes'])) self.assertNotIn(node1['id'], cluster['nodes']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_node_create_delete_with_target_cluster(self): # Create a cluster desired_capacity = 1 min_size = 1 max_size = 3 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Create a node belongs to the cluster node1 = test_api.create_node(self.client, 'node1', self.profile['id'], cluster_id=cluster['id'], role='master') node1 = test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'ACTIVE') self.assertEqual(cluster['id'], node1['cluster_id']) self.assertEqual(2, node1['index']) cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(2, cluster['desired_capacity']) self.assertEqual(2, len(cluster['nodes'])) self.assertIn(node1['id'], cluster['nodes']) # Delete node1 from cluster test_api.delete_node(self.client, node1['id']) test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'DELETED', ignore_missing=True) cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(1, len(cluster['nodes'])) self.assertNotIn(node1['id'], cluster['nodes']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_lb_policy(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, test_utils.random_name('cluster'), self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Verify there is no lb information in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertNotIn('lb_member', node['data']) # Create a lb policy spec = test_utils.spec_lb_policy lb_policy = test_api.create_policy(self.client, test_utils.random_name('lb-policy'), spec) # Attach scaling in/out policies to cluster params = {"enabled": True, "policy_id": lb_policy['id']} action_id = test_api.action_cluster(self.client, cluster['id'], 'policy_attach', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify lb information recorded in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertIn('lb_member', node['data']) # Scale out cluster action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Verify lb information recorded in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertIn('lb_member', node['data']) # Scale in cluster action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Detach scaling lb policy from cluster params = {"policy_id": lb_policy['id']} action_id = test_api.action_cluster(self.client, cluster['id'], 'policy_detach', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify lb information recorded in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertNotIn('lb_member', node['data']) # Delete policies test_api.delete_policy(self.client, lb_policy['id']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_scaling_policy(self): # Create cluster desired_capacity = 1 min_size = 0 max_size = 5 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Create a scaling policy targets on CLUSTER_SCALE_OUT action spec = test_utils.spec_scaling_policy spec['properties'] = { 'event': 'CLUSTER_SCALE_OUT', 'adjustment': { 'type': 'CHANGE_IN_CAPACITY', 'number': 2, 'min_step': 1, 'best_effort': True } } scaling_out_policy = test_api.create_policy(self.client, 'scaling-out-policy', spec, 0, 0) # Create a scaling policy targets on CLUSTER_SCALE_IN action spec['properties'] = { 'event': 'CLUSTER_SCALE_IN', 'adjustment': { 'type': 'CHANGE_IN_PERCENTAGE', 'number': 50, 'min_step': 2, 'best_effort': False } } scaling_in_policy = test_api.create_policy(self.client, 'scaling-in-policy', spec, 0, 0) # Attach scaling in/out policies to cluster for policy in [scaling_in_policy, scaling_out_policy]: params = { "priority": 50, "level": 50, "enabled": True, "cooldown": 0, "policy_id": policy['id'] } action_id = test_api.action_cluster(self.client, cluster['id'], 'policy_attach', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Scale out cluster without params action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Scale out with count set to 1 action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out', {'count': 1}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(4, len(cluster['nodes'])) # Scale out with count set to 3 to verify best_effort param action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out', {'count': 3}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(5, len(cluster['nodes'])) # Scale in cluster without params action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Scale in without param to verify min_step param action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Scale in with count set to 2 to verify best_effort param action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in', {'count': 2}) action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("Policy check failure: Attempted scaling below minimum " "size.") self.assertEqual(reason, action['status_reason']) # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Detach scaling in/out policies from cluster for policy in [scaling_in_policy, scaling_out_policy]: params = { "policy_id": policy['id'] } action_id = test_api.action_cluster(self.client, cluster['id'], 'policy_detach', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Delete policies test_api.delete_policy(self.client, scaling_in_policy['id']) test_api.delete_policy(self.client, scaling_out_policy['id']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_cluster_resize_with_constraint_adjusting(self): # Create cluster desired_capacity = 3 min_size = 2 max_size = 5 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Increase cluster size with upper limit increasing params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': 3, 'max_size': 6 } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(6, len(cluster['nodes'])) # Decrease cluster size upper limit with strict set to False params = {'max_size': 4, 'strict': False} action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(4, len(cluster['nodes'])) # Reduce cluster size with lower limit change params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': -3, 'min_size': 1 } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Increase cluster size lower limit with strict set to True params = {'min_size': 2, 'strict': True} action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (1) is less than " "the specified min_size (2).") self.assertEqual(reason, action['status_reason']) # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_cluster_resize_with_invalid_constraint_provided(self): # Create cluster desired_capacity = 3 min_size = 1 max_size = 4 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # New min_size is larger than current cluster # max_size with strict set to False params = { 'min_size': 5, 'strict': False } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) # Wait for cluster resize action failed action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (4) is less than " "the specified min_size (5).") self.assertEqual(reason, action['status_reason']) # New max_size is less than current cluster # min_size with strict set to True params = { 'max_size': 0, 'strict': True } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) # Wait for cluster resize action failed action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (3) is greater " "than the specified max_size (0).") self.assertEqual(reason, action['status_reason']) # New min_size > new max_size params = { 'min_size': 5, 'max_size': 3 } msg = _("The specified min_size (5) is greater than the " "specified max_size (3).") ex = self.assertRaises(api_client.SenlinApiException, test_api.action_cluster, self.client, cluster['id'], 'resize', params) self.assertEqual(400, ex.response.status_code) self.assertIn(msg, ex.response.content) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_cluster_scale_in_out(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Scale out cluster without params action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Scale out with count set to 2 action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out', {'count': 2}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(5, len(cluster['nodes'])) # Keep scaling out and break the size constraint action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out') # Wait for cluster scale out action failed action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (6) is greater " "than the cluster's max_size (5).") self.assertEqual(reason, action['status_reason']) # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(5, len(cluster['nodes'])) # Scale in cluster without params action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(4, len(cluster['nodes'])) # Scale in with count set to 3 action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in', {'count': 3}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Keep scaling in and break the size constraint action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (0) is less " "than the cluster's min_size (1).") self.assertEqual(reason, action['status_reason']) # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_cluster_scale_in_out(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, test_utils.random_name('cluster'), self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Scale out cluster without params action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Scale out with count set to 2 action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out', {'count': 2}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(5, len(cluster['nodes'])) # Keep scaling out and break the size constraint action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out') # Wait for cluster scale out action failed action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (6) is greater " "than the cluster's max_size (5).") self.assertEqual(reason, action['status_reason']) # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(5, len(cluster['nodes'])) # Scale in cluster without params action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(4, len(cluster['nodes'])) # Scale in with count set to 3 action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in', {'count': 3}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Keep scaling in and break the size constraint action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (0) is less " "than the cluster's min_size (1).") self.assertEqual(reason, action['status_reason']) # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_cluster_resize_with_constraint_breaking(self): # Create cluster desired_capacity = 3 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Increase cluster size and break the size constraint params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': 3, 'strict': True } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) # Wait for cluster resize action failed action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (6) is greater " "than the cluster's max_size (5).") self.assertEqual(reason, action['status_reason']) # Do best-effort resizing params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': 3, 'strict': False } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) # Wait for cluster resize action succeeded test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(5, len(cluster['nodes'])) # Decrease cluster size and break the size constraint params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': -5, 'strict': True } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) # Wait for cluster resize action failed action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (0) is less " "than the cluster's min_size (1).") self.assertEqual(reason, action['status_reason']) # Do best-effort resizing params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': -5, 'strict': False } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) # Wait for cluster resize action succeeded test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_cluster_resize_basic(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Increase cluster size by specifying adjustment count params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': 3, } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(5, len(cluster['nodes'])) # Decrease cluster size by specifying adjustment percentage. params = { 'adjustment_type': 'CHANGE_IN_PERCENTAGE', 'number': -50, } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Decrease cluster size by specifying exact capacity params = { 'adjustment_type': 'EXACT_CAPACITY', 'number': 2, } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(2, len(cluster['nodes'])) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_cluster_node_add_del(self): # Create cluster desired_capacity = 1 min_size = 1 max_size = 3 cluster_name = test_utils.random_name('cluster') cluster = test_api.create_cluster(self.client, cluster_name, self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Create two orphan nodes node1 = test_api.create_node(self.client, test_utils.random_name('node1'), self.profile['id']) node2 = test_api.create_node(self.client, test_utils.random_name('node2'), self.profile['id']) node1 = test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'ACTIVE') node2 = test_utils.wait_for_status(test_api.get_node, self.client, node2['id'], 'ACTIVE') self.assertEqual('', node1['cluster_id']) self.assertEqual(-1, node1['index']) self.assertEqual('', node2['cluster_id']) self.assertEqual(-1, node2['index']) # Add orphan nodes to cluster params = { 'nodes': [node1['id'], node2['id']], } action_id = test_api.action_cluster(self.client, cluster['id'], 'add_nodes', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(3, len(cluster['nodes'])) self.assertIn(node1['id'], cluster['nodes']) self.assertIn(node2['id'], cluster['nodes']) node1 = test_api.get_node(self.client, node1['id']) node2 = test_api.get_node(self.client, node2['id']) self.assertEqual(cluster['id'], node1['cluster_id']) self.assertEqual(cluster['id'], node2['cluster_id']) self.assertEqual(2, node1['index']) self.assertEqual(3, node2['index']) # Remove nodes from cluster params = { 'nodes': [node1['id'], node2['id']], } action_id = test_api.action_cluster(self.client, cluster['id'], 'del_nodes', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(1, len(cluster['nodes'])) self.assertNotIn(node1['id'], cluster['nodes']) self.assertNotIn(node2['id'], cluster['nodes']) node1 = test_api.get_node(self.client, node1['id']) node2 = test_api.get_node(self.client, node2['id']) self.assertEqual('', node1['cluster_id']) self.assertEqual('', node2['cluster_id']) self.assertEqual(-1, node1['index']) self.assertEqual(-1, node2['index']) # Delete orphan nodes test_api.delete_node(self.client, node1['id']) test_api.delete_node(self.client, node2['id']) test_utils.wait_for_delete(test_api.get_node, self.client, node1['id']) test_utils.wait_for_delete(test_api.get_node, self.client, node2['id']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_lb_policy(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) # Wait and verify cluster creation result cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') self.assertEqual('test-cluster', cluster['name']) self.assertEqual(desired_capacity, cluster['desired_capacity']) self.assertEqual(min_size, cluster['min_size']) self.assertEqual(max_size, cluster['max_size']) self.assertEqual(desired_capacity, len(cluster['nodes'])) # Verify there is no lb information in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertNotIn('lb_member', node['data']) # Create a lb policy spec = test_utils.spec_lb_policy lb_policy = test_api.create_policy(self.client, 'test-lb-policy', spec, 0, 0) # Attach scaling in/out policies to cluster params = { "priority": 50, "level": 50, "enabled": True, "cooldown": 0, "policy_id": lb_policy['id'] } action_id = test_api.action_cluster(self.client, cluster['id'], 'policy_attach', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify lb information recorded in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertIn('lb_member', node['data']) # Scale out cluster action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Verify lb information recorded in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertIn('lb_member', node['data']) # Scale in cluster action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Detach scaling lb policy from cluster params = { "policy_id": lb_policy['id'] } action_id = test_api.action_cluster(self.client, cluster['id'], 'policy_detach', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify lb information recorded in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertNotIn('lb_member', node['data']) # Delete policies test_api.delete_policy(self.client, lb_policy['id']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_cluster_node_add_del(self): # Create cluster desired_capacity = 1 min_size = 1 max_size = 3 cluster_name = test_utils.random_name('cluster') cluster = test_api.create_cluster(self.client, cluster_name, self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Create two orphan nodes node1 = test_api.create_node(self.client, test_utils.random_name('node1'), self.profile['id']) node2 = test_api.create_node(self.client, test_utils.random_name('node2'), self.profile['id']) node1 = test_utils.wait_for_status(test_api.get_node, self.client, node1['id'], 'ACTIVE') node2 = test_utils.wait_for_status(test_api.get_node, self.client, node2['id'], 'ACTIVE') self.assertEqual('', node1['cluster_id']) self.assertEqual(-1, node1['index']) self.assertEqual('', node2['cluster_id']) self.assertEqual(-1, node2['index']) # Add orphan nodes to cluster params = { 'nodes': [node1['id'], node2['id']], } action_id = test_api.action_cluster(self.client, cluster['id'], 'add_nodes', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(3, cluster['desired_capacity']) self.assertEqual(3, len(cluster['nodes'])) self.assertIn(node1['id'], cluster['nodes']) self.assertIn(node2['id'], cluster['nodes']) node1 = test_api.get_node(self.client, node1['id']) node2 = test_api.get_node(self.client, node2['id']) self.assertEqual(cluster['id'], node1['cluster_id']) self.assertEqual(cluster['id'], node2['cluster_id']) self.assertEqual([2, 3], sorted([node1['index'], node2['index']])) # Create one more orphan node and add it to cluster node3 = test_api.create_node(self.client, test_utils.random_name('node1'), self.profile['id']) node3 = test_utils.wait_for_status(test_api.get_node, self.client, node3['id'], 'ACTIVE') params = { 'nodes': [node3['id']], } res = test_api.action_cluster(self.client, cluster['id'], 'add_nodes', params) reason = _("The target capacity (4) is greater than the cluster's " "max_size (3).") self.assertIn(reason, res) # Remove two nodes from cluster params = { 'nodes': [node1['id'], node2['id']], } action_id = test_api.action_cluster(self.client, cluster['id'], 'del_nodes', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual(1, cluster['desired_capacity']) self.assertEqual(1, len(cluster['nodes'])) self.assertNotIn(node1['id'], cluster['nodes']) self.assertNotIn(node2['id'], cluster['nodes']) node1 = test_api.get_node(self.client, node1['id']) node2 = test_api.get_node(self.client, node2['id']) self.assertEqual('', node1['cluster_id']) self.assertEqual('', node2['cluster_id']) self.assertEqual(-1, node1['index']) self.assertEqual(-1, node2['index']) # Try to delete the last node from cluster last_node_id = cluster['nodes'][0] params = { 'nodes': [last_node_id] } res = test_api.action_cluster(self.client, cluster['id'], 'del_nodes', params) reason = _("The target capacity (0) is less than the cluster's " "min_size (1).") self.assertIn(reason, res) # Delete orphan nodes test_api.delete_node(self.client, node1['id']) test_api.delete_node(self.client, node2['id']) test_api.delete_node(self.client, node3['id']) test_utils.wait_for_delete(test_api.get_node, self.client, node1['id']) test_utils.wait_for_delete(test_api.get_node, self.client, node2['id']) test_utils.wait_for_delete(test_api.get_node, self.client, node3['id']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_cluster_resize_with_constraint_breaking(self): # Create cluster desired_capacity = 3 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, test_utils.random_name('cluster'), self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Increase cluster size and break the size constraint params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': 3, 'strict': True } res = test_api.action_cluster(self.client, cluster['id'], 'resize', params) reason = _("The target capacity (6) is greater than the cluster's " "max_size (5).") self.assertIn(reason, res) # Do best-effort resizing params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': 3, 'strict': False } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) # Wait for cluster resize action succeeded test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(5, len(cluster['nodes'])) # Decrease cluster size and break the size constraint params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': -5, 'strict': True } res = test_api.action_cluster(self.client, cluster['id'], 'resize', params) reason = _("The target capacity (0) is less than the cluster's " "min_size (1).") self.assertIn(reason, res) # Do best-effort resizing params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': -5, 'strict': False } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) # Wait for cluster resize action succeeded test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_cluster_resize_with_constraint_adjusting(self): # Create cluster desired_capacity = 3 min_size = 2 max_size = 5 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Increase cluster size with upper limit increasing params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': 3, 'max_size': 6 } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(6, len(cluster['nodes'])) # Decrease cluster size upper limit with strict set to False params = { 'max_size': 4, 'strict': False } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(4, len(cluster['nodes'])) # Reduce cluster size with lower limit change params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': -3, 'min_size': 1 } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Increase cluster size lower limit with strict set to True params = { 'min_size': 2, 'strict': True } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("The target capacity (1) is less than " "the specified min_size (2).") self.assertEqual(reason, action['status_reason']) # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_scaling_policy(self): # Create cluster desired_capacity = 1 min_size = 0 max_size = 5 cluster = test_api.create_cluster(self.client, 'test-cluster', self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Create a scaling policy targets on CLUSTER_SCALE_OUT action spec = test_utils.spec_scaling_policy spec['properties'] = { 'event': 'CLUSTER_SCALE_OUT', 'adjustment': { 'type': 'CHANGE_IN_CAPACITY', 'number': 2, 'min_step': 1, 'best_effort': True } } scaling_out_policy = test_api.create_policy(self.client, 'scaling-out-policy', spec, 0, 0) # Create a scaling policy targets on CLUSTER_SCALE_IN action spec['properties'] = { 'event': 'CLUSTER_SCALE_IN', 'adjustment': { 'type': 'CHANGE_IN_PERCENTAGE', 'number': 50, 'min_step': 2, 'best_effort': False } } scaling_in_policy = test_api.create_policy(self.client, 'scaling-in-policy', spec, 0, 0) # Attach scaling in/out policies to cluster for policy in [scaling_in_policy, scaling_out_policy]: params = { "priority": 50, "level": 50, "enabled": True, "cooldown": 0, "policy_id": policy['id'] } action_id = test_api.action_cluster(self.client, cluster['id'], 'policy_attach', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Scale out cluster without params action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Scale out with count set to 1 action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out', {'count': 1}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(4, len(cluster['nodes'])) # Scale out with count set to 3 to verify best_effort param action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out', {'count': 3}) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(5, len(cluster['nodes'])) # Scale in cluster without params action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Scale in without param to verify min_step param action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Scale in with count set to 2 to verify best_effort param action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in', {'count': 2}) action = test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'FAILED') reason = _("Policy check failure: Attempted scaling below minimum " "size.") self.assertEqual(reason, action['status_reason']) # Verify cluster scale in result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Detach scaling in/out policies from cluster for policy in [scaling_in_policy, scaling_out_policy]: params = {"policy_id": policy['id']} action_id = test_api.action_cluster(self.client, cluster['id'], 'policy_detach', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Delete policies test_api.delete_policy(self.client, scaling_in_policy['id']) test_api.delete_policy(self.client, scaling_out_policy['id']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'DELETED', ignore_missing=True)
def test_cluster_resize_with_constraint_adjusting(self): # Create cluster desired_capacity = 3 min_size = 2 max_size = 5 cluster = test_api.create_cluster(self.client, test_utils.random_name('cluster'), self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Increase cluster size with upper limit increasing params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': 3, 'max_size': 6 } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(6, len(cluster['nodes'])) # Decrease cluster size upper limit with strict set to False params = {'max_size': 4, 'strict': False} action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(4, len(cluster['nodes'])) # Reduce cluster size with lower limit change params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': -3, 'min_size': 1 } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Increase cluster size lower limit with strict set to True params = {'min_size': 2, 'strict': True} res = test_api.action_cluster(self.client, cluster['id'], 'resize', params) reason = _("The specified min_size (2) is greater than the current " "desired_capacity (1) of the cluster.") self.assertIn(reason, res) # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(1, len(cluster['nodes'])) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_cluster_resize_basic(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, test_utils.random_name('cluster'), self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Increase cluster size by specifying adjustment count params = { 'adjustment_type': 'CHANGE_IN_CAPACITY', 'number': 3, } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(5, len(cluster['nodes'])) # Decrease cluster size by specifying adjustment percentage. params = { 'adjustment_type': 'CHANGE_IN_PERCENTAGE', 'number': -50, } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Decrease cluster size by specifying exact capacity params = { 'adjustment_type': 'EXACT_CAPACITY', 'number': 2, } action_id = test_api.action_cluster(self.client, cluster['id'], 'resize', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster resize result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(2, len(cluster['nodes'])) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])
def test_lb_policy(self): # Create cluster desired_capacity = 2 min_size = 1 max_size = 5 cluster = test_api.create_cluster(self.client, test_utils.random_name('cluster'), self.profile['id'], desired_capacity, min_size, max_size) cluster = test_utils.wait_for_status(test_api.get_cluster, self.client, cluster['id'], 'ACTIVE') # Verify there is no lb information in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertNotIn('lb_member', node['data']) # Create a lb policy spec = test_utils.spec_lb_policy lb_policy = test_api.create_policy(self.client, test_utils.random_name('lb-policy'), spec) # Attach scaling in/out policies to cluster params = { "enabled": True, "policy_id": lb_policy['id'] } action_id = test_api.action_cluster(self.client, cluster['id'], 'policy_attach', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify lb information recorded in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertIn('lb_member', node['data']) # Scale out cluster action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_out') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify cluster scale out result cluster = test_api.get_cluster(self.client, cluster['id']) self.assertEqual('ACTIVE', cluster['status']) self.assertEqual(3, len(cluster['nodes'])) # Verify lb information recorded in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertIn('lb_member', node['data']) # Scale in cluster action_id = test_api.action_cluster(self.client, cluster['id'], 'scale_in') test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Detach scaling lb policy from cluster params = { "policy_id": lb_policy['id'] } action_id = test_api.action_cluster(self.client, cluster['id'], 'policy_detach', params) test_utils.wait_for_status(test_api.get_action, self.client, action_id, 'SUCCEEDED') # Verify lb information recorded in node data cluster = test_api.get_cluster(self.client, cluster['id']) for n in cluster['nodes']: node = test_api.get_node(self.client, n) self.assertNotIn('lb_member', node['data']) # Delete policies test_api.delete_policy(self.client, lb_policy['id']) # Delete cluster test_api.delete_cluster(self.client, cluster['id']) cluster = test_utils.wait_for_delete(test_api.get_cluster, self.client, cluster['id'])