def test_restart_heketi_pod(self): """Validate restarting heketi pod""" # create heketi volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size=1, json=True) self.assertTrue(vol_info, "Failed to create heketi volume of size 1") self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id'], raise_on_error=False) topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) # get heketi-pod name heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0], self.heketi_dc_name) # delete heketi-pod (it restarts the pod) oc_delete(self.ocp_master_node[0], 'pod', heketi_pod_name, collect_logs=self.heketi_logs_before_delete) wait_for_resource_absence(self.ocp_master_node[0], 'pod', heketi_pod_name) # get new heketi-pod name heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0], self.heketi_dc_name) wait_for_pod_be_ready(self.ocp_master_node[0], heketi_pod_name) # check heketi server is running self.assertTrue( hello_heketi(self.heketi_client_node, self.heketi_server_url), "Heketi server %s is not alive" % self.heketi_server_url) # compare the topology new_topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) self.assertEqual( new_topo_info, topo_info, "topology info is not same," " difference - %s" % diff(topo_info, new_topo_info)) # create new volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size=2, json=True) self.assertTrue(vol_info, "Failed to create heketi volume of size 20") heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def validate_file_volumes_count(self, h_node, h_server, node_ip): # check volume count from heketi and gluster are same heketi_topology_info(h_node, h_server, json=True) h_volume_list = heketi_volume_list(h_node, h_server, json=True) vol_list = get_volume_list(node_ip) self.assertIsNotNone( vol_list, "Failed to get volumes list") self.assertEqual( len(h_volume_list['volumes']), len(vol_list), "Failed to verify volume count Expected:'{}', Actual:'{}'".format( len(h_volume_list['volumes']), len(vol_list)))
def test_restart_heketi_pod(self): """Validate restarting heketi pod""" # create heketi volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size=1, json=True) self.assertTrue(vol_info, "Failed to create heketi volume of size 1") self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id'], raise_on_error=False) topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) # get heketi-pod name heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0], self.heketi_dc_name) # delete heketi-pod (it restarts the pod) oc_delete(self.ocp_master_node[0], 'pod', heketi_pod_name) wait_for_resource_absence(self.ocp_master_node[0], 'pod', heketi_pod_name) # get new heketi-pod name heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0], self.heketi_dc_name) wait_for_pod_be_ready(self.ocp_master_node[0], heketi_pod_name) # check heketi server is running self.assertTrue( hello_heketi(self.heketi_client_node, self.heketi_server_url), "Heketi server %s is not alive" % self.heketi_server_url ) # compare the topology new_topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) self.assertEqual(new_topo_info, topo_info, "topology info is not same," " difference - %s" % diff(topo_info, new_topo_info)) # create new volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size=2, json=True) self.assertTrue(vol_info, "Failed to create heketi volume of size 20") heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def test_heketi_device_delete(self): """Test Heketi device delete operation""" # Get list of additional devices for one of the Gluster nodes ip_with_devices = {} for gluster_server in g.config["gluster_servers"].values(): if not gluster_server.get("additional_devices"): continue ip_with_devices = { gluster_server['storage']: gluster_server['additional_devices'] } break # Skip test if no additional device is available if not ip_with_devices: self.skipTest( "No additional devices attached to any of the gluster nodes") # Select any additional device and get the node id of the gluster node h_node, h_server = self.heketi_client_node, self.heketi_server_url node_id, device_name = None, list(ip_with_devices.values())[0][0] topology_info = heketi_topology_info(h_node, h_server, json=True) for node in topology_info["clusters"][0]["nodes"]: if list(ip_with_devices.keys())[0] == ( node['hostnames']["storage"][0]): node_id = node["id"] break self.assertTrue(node_id) # Add additional device to the cluster heketi_device_add(h_node, h_server, device_name, node_id) # Get the device id and number of bricks on the device node_info_after_addition = heketi_node_info(h_node, h_server, node_id, json=True) device_id, bricks = None, None for device in node_info_after_addition["devices"]: if device["name"] == device_name: device_id, bricks = device["id"], len(device['bricks']) break self.assertTrue(device_id, "Device not added in expected node") # Delete heketi device heketi_device_disable(h_node, h_server, device_id) heketi_device_remove(h_node, h_server, device_id) heketi_device_delete(h_node, h_server, device_id) # Verify that there were no bricks on the newly added device msg = ( "Number of bricks on the device %s of the node %s should be zero" % (device_name, list(ip_with_devices.keys())[0])) self.assertEqual(0, bricks, msg) # Verify device deletion node_info_after_deletion = heketi_node_info(h_node, h_server, node_id) msg = ("Device %s should not be shown in node info of the node %s" "after the device deletion" % (device_id, node_id)) self.assertNotIn(device_id, node_info_after_deletion, msg)
def verify_node_is_present_or_not_in_heketi( self, node_id, manage_hostname, storage_ip, state='present'): topology = heketi_ops.heketi_topology_info( self.h_node, self.h_url, json=True) if state == 'present': present = False for node in topology['clusters'][0]['nodes']: if node_id == node['id']: self.assertEqual( manage_hostname, node['hostnames']['manage'][0]) self.assertEqual( storage_ip, node['hostnames']['storage'][0]) present = True break self.assertTrue(present, 'Node %s not found in heketi' % node_id) elif state == 'absent': for node in topology['clusters'][0]['nodes']: self.assertNotEqual( manage_hostname, node['hostnames']['manage'][0]) self.assertNotEqual( storage_ip, node['hostnames']['storage'][0]) self.assertNotEqual(node_id, node['id']) else: msg = "State %s is other than present, absent" % state raise AssertionError(msg)
def test_heketi_authentication_with_user_credentials(self): """Heketi command authentication with invalid and valid credentials""" h_client, h_server = self.heketi_client_node, self.heketi_server_url err_msg = "Error: Invalid JWT token: Token missing iss claim" # Run heketi commands with invalid credentials for each_cmd in ("volume list", "topology info"): cmd = "timeout 120 heketi-cli -s {} {}".format( self.heketi_server_url, each_cmd) ret, _, err = g.run(h_client, cmd) self.assertTrue(ret, "Command execution with invalid credentials" " should not succeed") self.assertEqual( err_msg, err.strip(), "Error is different from the command" " execution {}".format(err.strip())) # Run heketi commands with valid credentials kwar = {'json_arg': True, 'secret': self.heketi_cli_key, 'user': self.heketi_cli_user} heketi_ops.heketi_volume_list(h_client, h_server, **kwar) heketi_ops.heketi_topology_info(h_client, h_server, **kwar)
def _get_online_devices_and_nodes_with_zone(self): """ This function returns the list of nodes and devices associated to zone Returns: dict: dict with zone, devices and nodes values e.g., { zone_1: { "nodes": [ node1, node2, node3], "devices": [ device1, device2] }, zone_2: { "nodes": [ node1, node2, node3], "devices": [ device1, device2] } } """ zone_devices_nodes = dict() topology_info = heketi_ops.heketi_topology_info(self.h_client, self.h_server, json=True) for cluster in topology_info['clusters']: for node in cluster['nodes']: if node['state'] == 'online': if node['zone'] not in zone_devices_nodes: zone_devices_nodes[node['zone']] = dict() if 'nodes' not in zone_devices_nodes[node['zone']]: zone_devices_nodes[node['zone']]['nodes'] = [] (zone_devices_nodes[node['zone']]['nodes'].append( node['id'])) for device in node['devices']: if device['state'] == 'online': if node['zone'] not in zone_devices_nodes: zone_devices_nodes[node['zone']] = dict() if 'devices' not in zone_devices_nodes[node['zone']]: zone_devices_nodes[node['zone']]['devices'] = [] (zone_devices_nodes[node['zone']]['devices'].append( device['id'])) return zone_devices_nodes
def test_verify_db_check(self, count_type): """Validate the nodes, devices and bricks count in heketi db""" # Get the total number of nodes, devices and bricks from db check db_info = heketi_ops.heketi_db_check(self.heketi_client_node, self.heketi_server_url) db_devices_count = db_info["devices"]["total"] db_nodes_count = db_info["nodes"]["total"] db_bricks_count = db_info["bricks"]["total"] # Get the total number of nodes, devices and bricks from topology info topology_info = heketi_ops.heketi_topology_info( self.heketi_client_node, self.heketi_server_url, json=True) topology_devices_count, topology_nodes_count = 0, 0 topology_bricks_count = 0 for cluster in topology_info['clusters']: topology_nodes_count += len(cluster['nodes']) if count_type == 'bricks_count' or 'device_count': for node in cluster['nodes']: topology_devices_count += len(node['devices']) if count_type == 'bricks_count': for device in node['devices']: topology_bricks_count += len(device['bricks']) # Compare the device count if count_type == 'device_count': msg = ("Devices count in db check {} and in topology info {} is " "not same".format(db_devices_count, topology_devices_count)) self.assertEqual(topology_devices_count, db_devices_count, msg) # Compare the node count elif count_type == 'node_count': msg = ( "Nodes count in db check {} and nodes count in topology info " "{} is not same".format(db_nodes_count, topology_nodes_count)) self.assertEqual(topology_nodes_count, db_nodes_count, msg) # Compare the bricks count elif count_type == 'bricks_count': msg = ("Bricks count in db check {} and bricks count in topology " "info {} is not same".format(db_bricks_count, topology_bricks_count)) self.assertEqual(topology_bricks_count, db_bricks_count, msg)
def test_heketi_device_remove_delete(self, is_delete_device, add_back_again): """Validate remove/delete device using heketi-cli""" try: gluster_server_0 = list(g.config["gluster_servers"].values())[0] manage_hostname = gluster_server_0["manage"] device_name = gluster_server_0["additional_devices"][0] except (KeyError, IndexError): self.skipTest( "Additional disk is not specified for node with following " "hostnames and IP addresses: {}, {}".format( gluster_server_0.get('manage', '?'), gluster_server_0.get('storage', '?'))) # Get node ID of the Gluster hostname node_list = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True)["clusters"][0]["nodes"] self.assertTrue(node_list, "Cluster info command returned empty list of nodes") node_id = None for node in node_list: if manage_hostname == node['hostnames']["manage"][0]: node_id = node["id"] break self.assertTrue( node_id, "No information about node_id" " for {}".format(manage_hostname)) deleted_device = self._add_new_device_and_remove_existing_device( is_delete_device, device_name, node_id, add_back_again) if add_back_again: self._add_new_device_and_remove_existing_device( is_delete_device, deleted_device, node_id, add_back_again, skip_cleanup_addition=True)
def test_dev_path_mapping_basic_validation(self): """Validate dev_path of all the devices""" node_with_devices = dict() h_node, h_url = self.heketi_client_node, self.heketi_server_url # Get the hostname and devices attached to each host topology_info = heketi_topology_info(h_node, h_url, json=True) for cluster in topology_info['clusters']: for node in cluster['nodes']: node_with_devices[node['hostnames']['manage'][0]] = [ device['id'] for device in node['devices'] ] # Validate dev_path of each device for node, devices in node_with_devices.items(): for dev in list(devices): is_true = validate_dev_path_vg_and_uuid( h_node, h_url, node, dev) self.assertTrue( is_true, "Failed to verify dv_path for the " "device {}".format(dev))
def test_heketi_device_replacement_in_node(self): """Validate device replacement operation on single node""" h_client, h_server = self.heketi_client_node, self.heketi_server_url try: gluster_server_0 = list(g.config["gluster_servers"].values())[0] manage_hostname = gluster_server_0["manage"] add_device_name = gluster_server_0["additional_devices"][0] except (KeyError, IndexError): self.skipTest( "Additional disk is not specified for node with following " "hostnames and IP addresses: {}, {}".format( gluster_server_0.get('manage', '?'), gluster_server_0.get('storage', '?'))) # Get existing heketi volume list existing_volumes = heketi_volume_list(h_client, h_server, json=True) # Add cleanup function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Get nodes info node_id_list = heketi_node_list(h_client, h_server) # Disable 4th and other nodes if len(node_id_list) > 3: for node_id in node_id_list[3:]: heketi_node_disable(h_client, h_server, node_id) self.addCleanup(heketi_node_enable, h_client, h_server, node_id) # Create volume when 3 nodes are online vol_size, vol_count = 2, 4 for _ in range(vol_count): vol_info = heketi_blockvolume_create(h_client, h_server, vol_size, json=True) self.addCleanup(heketi_blockvolume_delete, h_client, h_server, vol_info['id']) # Get node ID of the Gluster hostname topology_info = heketi_topology_info(h_client, h_server, json=True) self.assertIsNotNone(topology_info, "Failed to get topology info") self.assertIn("clusters", topology_info.keys(), "Failed to get cluster " "details from topology info") node_list = topology_info["clusters"][0]["nodes"] self.assertTrue(node_list, "Cluster info command returned empty list of nodes") node_id = None for node in node_list: if manage_hostname == node['hostnames']["manage"][0]: node_id = node["id"] break self.assertTrue( node_id, "Failed to get node info for node id '{}'".format( manage_hostname)) # Add extra device, then remember it's ID and size device_id_new, device_size_new = self._add_heketi_device( add_device_name, node_id) # Remove one of the existing devices on node except new device device_name, device_id = None, None node_info_after_addition = heketi_node_info(h_client, h_server, node_id, json=True) for device in node_info_after_addition["devices"]: if (device["name"] != add_device_name and device["storage"]["total"] == device_size_new): device_name = device["name"] device_id = device["id"] break self.assertIsNotNone(device_name, "Failed to get device name") self.assertIsNotNone(device_id, "Failed to get device id") self.addCleanup(heketi_device_enable, h_client, h_server, device_id, raise_on_error=False) self.addCleanup(heketi_device_add, h_client, h_server, device_name, node_id, raise_on_error=False) heketi_device_disable(h_client, h_server, device_id) heketi_device_remove(h_client, h_server, device_id) heketi_device_delete(h_client, h_server, device_id)
def test_heketi_device_info(self): """Validate whether device related information is displayed""" # Get devices from topology info devices_from_topology = {} topology_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(topology_info) self.assertIn('clusters', list(topology_info.keys())) self.assertGreater(len(topology_info['clusters']), 0) for cluster in topology_info['clusters']: self.assertIn('nodes', list(cluster.keys())) self.assertGreater(len(cluster['nodes']), 0) for node in cluster['nodes']: self.assertIn('devices', list(node.keys())) self.assertGreater(len(node['devices']), 0) for device in node['devices']: # Expected keys are state, storage, id, name and bricks. self.assertIn('id', list(device.keys())) devices_from_topology[device['id']] = device # Get devices info and make sure data are consistent and complete for device_id, device_from_t_info in devices_from_topology.items(): device_info = heketi_device_info(self.heketi_client_node, self.heketi_server_url, device_id, json=True) self.assertTrue(device_info) # Verify 'id', 'name', 'state' and 'storage' data for key in ('id', 'name', 'state', 'storage', 'bricks'): self.assertIn(key, list(device_from_t_info.keys())) self.assertIn(key, list(device_info.keys())) for key in ('id', 'name', 'state'): self.assertEqual(device_info[key], device_from_t_info[key]) device_info_storage = device_info['storage'] device_from_t_info_storage = device_from_t_info['storage'] device_info_storage_keys = list(device_info_storage.keys()) device_from_t_info_storage_keys = list( device_from_t_info_storage.keys()) for key in ('total', 'used', 'free'): self.assertIn(key, device_info_storage_keys) self.assertIn(key, device_from_t_info_storage_keys) self.assertEqual(device_info_storage[key], device_from_t_info_storage[key]) self.assertIsInstance(device_info_storage[key], int) self.assertGreater(device_info_storage[key], -1) # Verify 'bricks' data self.assertEqual(len(device_info['bricks']), len(device_from_t_info['bricks'])) brick_match_count = 0 for brick in device_info['bricks']: for brick_from_t in device_from_t_info['bricks']: if brick_from_t['id'] != brick['id']: continue brick_match_count += 1 brick_from_t_keys = list(brick_from_t.keys()) brick_keys = list(brick.keys()) for key in ('device', 'volume', 'size', 'path', 'id', 'node'): self.assertIn(key, brick_from_t_keys) self.assertIn(key, brick_keys) self.assertEqual(brick[key], brick_from_t[key]) self.assertEqual(brick_match_count, len(device_info['bricks']))
def verify_heketi_metrics_with_topology_info(self): topology = heketi_topology_info( self.heketi_client_node, self.heketi_server_url, json=True) metrics = get_heketi_metrics( self.heketi_client_node, self.heketi_server_url) self.assertTrue(topology) self.assertIn('clusters', list(topology.keys())) self.assertGreater(len(topology['clusters']), 0) self.assertTrue(metrics) self.assertGreater(len(metrics.keys()), 0) self.assertEqual( len(topology['clusters']), metrics['heketi_cluster_count']) for cluster in topology['clusters']: self.assertIn('nodes', list(cluster.keys())) self.assertGreater(len(cluster['nodes']), 0) cluster_id = cluster['id'] cluster_ids = ([obj['cluster'] for obj in metrics['heketi_nodes_count']]) self.assertIn(cluster_id, cluster_ids) for node_count in metrics['heketi_nodes_count']: if node_count['cluster'] == cluster_id: self.assertEqual( len(cluster['nodes']), node_count['value']) cluster_ids = ([obj['cluster'] for obj in metrics['heketi_volumes_count']]) self.assertIn(cluster_id, cluster_ids) for vol_count in metrics['heketi_volumes_count']: if vol_count['cluster'] == cluster_id: self.assertEqual( len(cluster['volumes']), vol_count['value']) for node in cluster['nodes']: self.assertIn('devices', list(node.keys())) self.assertGreater(len(node['devices']), 0) hostname = node['hostnames']['manage'][0] cluster_ids = ([obj['cluster'] for obj in metrics['heketi_device_count']]) self.assertIn(cluster_id, cluster_ids) hostnames = ([obj['hostname'] for obj in metrics['heketi_device_count']]) self.assertIn(hostname, hostnames) for device_count in metrics['heketi_device_count']: if (device_count['cluster'] == cluster_id and device_count['hostname'] == hostname): self.assertEqual( len(node['devices']), device_count['value']) for device in node['devices']: device_name = device['name'] device_size_t = device['storage']['total'] device_free_t = device['storage']['free'] device_used_t = device['storage']['used'] cluster_ids = ([obj['cluster'] for obj in metrics['heketi_device_brick_count']]) self.assertIn(cluster_id, cluster_ids) hostnames = ([obj['hostname'] for obj in metrics['heketi_device_brick_count']]) self.assertIn(hostname, hostnames) devices = ([obj['device'] for obj in metrics['heketi_device_brick_count']]) self.assertIn(device_name, devices) for brick_count in metrics['heketi_device_brick_count']: if (brick_count['cluster'] == cluster_id and brick_count['hostname'] == hostname and brick_count['device'] == device_name): self.assertEqual( len(device['bricks']), brick_count['value']) cluster_ids = ([obj['cluster'] for obj in metrics['heketi_device_size']]) self.assertIn(cluster_id, cluster_ids) hostnames = ([obj['hostname'] for obj in metrics['heketi_device_size']]) self.assertIn(hostname, hostnames) devices = ([obj['device'] for obj in metrics['heketi_device_size']]) self.assertIn(device_name, devices) for device_size in metrics['heketi_device_size']: if (device_size['cluster'] == cluster_id and device_size['hostname'] == hostname and device_size['device'] == device_name): self.assertEqual( device_size_t, device_size['value']) cluster_ids = ([obj['cluster'] for obj in metrics['heketi_device_free']]) self.assertIn(cluster_id, cluster_ids) hostnames = ([obj['hostname'] for obj in metrics['heketi_device_free']]) self.assertIn(hostname, hostnames) devices = ([obj['device'] for obj in metrics['heketi_device_free']]) self.assertIn(device_name, devices) for device_free in metrics['heketi_device_free']: if (device_free['cluster'] == cluster_id and device_free['hostname'] == hostname and device_free['device'] == device_name): self.assertEqual( device_free_t, device_free['value']) cluster_ids = ([obj['cluster'] for obj in metrics['heketi_device_used']]) self.assertIn(cluster_id, cluster_ids) hostnames = ([obj['hostname'] for obj in metrics['heketi_device_used']]) self.assertIn(hostname, hostnames) devices = ([obj['device'] for obj in metrics['heketi_device_used']]) self.assertIn(device_name, devices) for device_used in metrics['heketi_device_used']: if (device_used['cluster'] == cluster_id and device_used['hostname'] == hostname and device_used['device'] == device_name): self.assertEqual( device_used_t, device_used['value'])
def test_heketi_device_remove(self, delete_device): """Validate remove/delete device using heketi-cli""" gluster_server_0 = list(g.config["gluster_servers"].values())[0] try: device_name = gluster_server_0["additional_devices"][0] except (KeyError, IndexError): self.skipTest( "Additional disk is not specified for node with following " "hostnames and IP addresses: %s, %s." % (gluster_server_0.get( 'manage', '?'), gluster_server_0.get('storage', '?'))) manage_hostname = gluster_server_0["manage"] # Get node ID of the Gluster hostname topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(topo_info["clusters"][0]["nodes"], "Cluster info command returned empty list of nodes.") node_id = None for node in topo_info["clusters"][0]["nodes"]: if manage_hostname == node['hostnames']["manage"][0]: node_id = node["id"] break self.assertNotEqual( node_id, None, "No information about node_id for %s" % manage_hostname) # Iterate chosen node devices and pick the smallest online one. lowest_device_size = lowest_device_id = None online_hosts = self.get_online_nodes_disable_redundant() for host in online_hosts[0:3]: if node_id != host["id"]: continue for device in host["devices"]: if device["state"].strip().lower() != "online": continue if (lowest_device_size is None or device["storage"]["total"] < lowest_device_size): lowest_device_size = device["storage"]["total"] lowest_device_id = device["id"] lowest_device_name = device["name"] if lowest_device_id is None: self.skipTest( "Didn't find suitable device for disablement on '%s' node." % (node_id)) # Create volume vol_size = 1 vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, vol_size, json=True) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # Add extra device, then remember it's ID and size heketi_device_add(self.heketi_client_node, self.heketi_server_url, device_name, node_id) node_info_after_addition = heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) for device in node_info_after_addition["devices"]: if device["name"] != device_name: continue device_id_new = device["id"] device_size_new = device["storage"]["total"] self.addCleanup(heketi_device_delete, self.heketi_client_node, self.heketi_server_url, device_id_new) self.addCleanup(heketi_device_remove, self.heketi_client_node, self.heketi_server_url, device_id_new) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, device_id_new) if lowest_device_size > device_size_new: skip_msg = ("Skip test case, because newly added disk %s is " "smaller than device which we want to remove %s." % (device_size_new, lowest_device_size)) self.skipTest(skip_msg) g.log.info("Removing device id %s" % lowest_device_id) with self.assertRaises(AssertionError): out = heketi_device_remove(self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.assertFalse(True, "Device removal didn't fail: %s" % out) g.log.info("Device removal failed as expected") # Need to disable device before removing heketi_device_disable(self.heketi_client_node, self.heketi_server_url, lowest_device_id) if not delete_device: self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) # Remove device from Heketi try: heketi_device_remove(self.heketi_client_node, self.heketi_server_url, lowest_device_id) except Exception: if delete_device: self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) raise if not delete_device: self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) if delete_device: try: heketi_device_delete(self.heketi_client_node, self.heketi_server_url, lowest_device_id) except Exception: self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) raise self.addCleanup(heketi_device_add, self.heketi_client_node, self.heketi_server_url, lowest_device_name, node_id) # Create volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, vol_size, json=True) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) if delete_device: return # Check that none of volume's bricks is present on the device present = self.check_any_of_bricks_present_in_device( vol_info['bricks'], lowest_device_id) self.assertFalse( present, "Some of the '%s' volume bricks is present of the removed " "'%s' device." % (vol_info['id'], lowest_device_id))
def verify_heketi_metrics_with_topology_info(self): topology = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) metrics = get_heketi_metrics(self.heketi_client_node, self.heketi_server_url) self.assertTrue(topology) self.assertIn('clusters', list(topology.keys())) self.assertGreater(len(topology['clusters']), 0) self.assertTrue(metrics) self.assertGreater(len(metrics.keys()), 0) self.assertEqual(len(topology['clusters']), metrics['heketi_cluster_count']) for cluster in topology['clusters']: self.assertIn('nodes', list(cluster.keys())) self.assertGreater(len(cluster['nodes']), 0) cluster_id = cluster['id'] cluster_ids = ([ obj['cluster'] for obj in metrics['heketi_nodes_count'] ]) self.assertIn(cluster_id, cluster_ids) for node_count in metrics['heketi_nodes_count']: if node_count['cluster'] == cluster_id: self.assertEqual(len(cluster['nodes']), node_count['value']) cluster_ids = ([ obj['cluster'] for obj in metrics['heketi_volumes_count'] ]) self.assertIn(cluster_id, cluster_ids) for vol_count in metrics['heketi_volumes_count']: if vol_count['cluster'] == cluster_id: self.assertEqual(len(cluster['volumes']), vol_count['value']) for node in cluster['nodes']: self.assertIn('devices', list(node.keys())) self.assertGreater(len(node['devices']), 0) hostname = node['hostnames']['manage'][0] cluster_ids = ([ obj['cluster'] for obj in metrics['heketi_device_count'] ]) self.assertIn(cluster_id, cluster_ids) hostnames = ([ obj['hostname'] for obj in metrics['heketi_device_count'] ]) self.assertIn(hostname, hostnames) for device_count in metrics['heketi_device_count']: if (device_count['cluster'] == cluster_id and device_count['hostname'] == hostname): self.assertEqual(len(node['devices']), device_count['value']) for device in node['devices']: device_name = device['name'] device_size_t = device['storage']['total'] device_free_t = device['storage']['free'] device_used_t = device['storage']['used'] cluster_ids = ([ obj['cluster'] for obj in metrics['heketi_device_brick_count'] ]) self.assertIn(cluster_id, cluster_ids) hostnames = ([ obj['hostname'] for obj in metrics['heketi_device_brick_count'] ]) self.assertIn(hostname, hostnames) devices = ([ obj['device'] for obj in metrics['heketi_device_brick_count'] ]) self.assertIn(device_name, devices) for brick_count in metrics['heketi_device_brick_count']: if (brick_count['cluster'] == cluster_id and brick_count['hostname'] == hostname and brick_count['device'] == device_name): self.assertEqual(len(device['bricks']), brick_count['value']) cluster_ids = ([ obj['cluster'] for obj in metrics['heketi_device_size'] ]) self.assertIn(cluster_id, cluster_ids) hostnames = ([ obj['hostname'] for obj in metrics['heketi_device_size'] ]) self.assertIn(hostname, hostnames) devices = ([ obj['device'] for obj in metrics['heketi_device_size'] ]) self.assertIn(device_name, devices) for device_size in metrics['heketi_device_size']: if (device_size['cluster'] == cluster_id and device_size['hostname'] == hostname and device_size['device'] == device_name): self.assertEqual(device_size_t, device_size['value']) cluster_ids = ([ obj['cluster'] for obj in metrics['heketi_device_free'] ]) self.assertIn(cluster_id, cluster_ids) hostnames = ([ obj['hostname'] for obj in metrics['heketi_device_free'] ]) self.assertIn(hostname, hostnames) devices = ([ obj['device'] for obj in metrics['heketi_device_free'] ]) self.assertIn(device_name, devices) for device_free in metrics['heketi_device_free']: if (device_free['cluster'] == cluster_id and device_free['hostname'] == hostname and device_free['device'] == device_name): self.assertEqual(device_free_t, device_free['value']) cluster_ids = ([ obj['cluster'] for obj in metrics['heketi_device_used'] ]) self.assertIn(cluster_id, cluster_ids) hostnames = ([ obj['hostname'] for obj in metrics['heketi_device_used'] ]) self.assertIn(hostname, hostnames) devices = ([ obj['device'] for obj in metrics['heketi_device_used'] ]) self.assertIn(device_name, devices) for device_used in metrics['heketi_device_used']: if (device_used['cluster'] == cluster_id and device_used['hostname'] == hostname and device_used['device'] == device_name): self.assertEqual(device_used_t, device_used['value'])
def test_create_heketi_cluster_and_add_node(self): """Test heketi node add to a newly created cluster""" storage_host_info = g.config.get("additional_gluster_servers") if not storage_host_info: self.skipTest( "Skip test case as 'additional_gluster_servers' option is " "not provided in config file") storage_host_info = list(storage_host_info.values())[0] try: storage_hostname = storage_host_info["manage"] storage_ip = storage_host_info["storage"] storage_device = storage_host_info["devices"][0] except KeyError: msg = ("Config options 'additional_gluster_servers.manage' " "'additional_gluster_servers.storage' and " "'additional_gluster_servers.devices' " "must be set.") g.log.error(msg) raise exceptions.ConfigError(msg) h_client, h_server = self.heketi_client_node, self.heketi_server_url storage_zone = 1 cluster_id = heketi_ops.heketi_cluster_create(self.heketi_client_node, self.heketi_server_url, json=True)["id"] self.addCleanup(heketi_ops.heketi_cluster_delete, self.heketi_client_node, self.heketi_server_url, cluster_id) self.configure_node_to_run_gluster(storage_hostname) heketi_node_info = heketi_ops.heketi_node_add(h_client, h_server, storage_zone, cluster_id, storage_hostname, storage_ip, json=True) heketi_node_id = heketi_node_info["id"] self.addCleanup(heketi_ops.heketi_node_delete, h_client, h_server, heketi_node_id) self.addCleanup(heketi_ops.heketi_node_remove, h_client, h_server, heketi_node_id) self.addCleanup(heketi_ops.heketi_node_disable, h_client, h_server, heketi_node_id) self.assertEqual( heketi_node_info["cluster"], cluster_id, "Node got added in unexpected cluster exp: %s, act: %s" % (cluster_id, heketi_node_info["cluster"])) heketi_ops.heketi_device_add(h_client, h_server, storage_device, heketi_node_id) heketi_node_info = heketi_ops.heketi_node_info(h_client, h_server, heketi_node_id, json=True) device_id = None for device in heketi_node_info["devices"]: if device["name"] == storage_device: device_id = device["id"] break err_msg = ("Failed to add device %s on node %s" % (storage_device, heketi_node_id)) self.assertTrue(device_id, err_msg) self.addCleanup(heketi_ops.heketi_device_delete, h_client, h_server, device_id) self.addCleanup(heketi_ops.heketi_device_remove, h_client, h_server, device_id) self.addCleanup(heketi_ops.heketi_device_disable, h_client, h_server, device_id) cluster_info = heketi_ops.heketi_cluster_info(h_client, h_server, cluster_id, json=True) self.assertIn( heketi_node_info["id"], cluster_info["nodes"], "Newly added node %s not found in cluster %s, cluster info %s" % (heketi_node_info["id"], cluster_id, cluster_info)) topology_info = heketi_ops.heketi_topology_info(h_client, h_server, json=True) cluster_details = [ cluster for cluster in topology_info["clusters"] if cluster["id"] == cluster_id ] err_msg = "Cluster details for id '%s' not found" % cluster_id self.assertTrue(cluster_details, err_msg) err_msg = ("Multiple clusters with same id '%s' found %s" % (cluster_id, cluster_details)) self.assertEqual(len(cluster_details), 1, err_msg) node_details = [ node for node in cluster_details[0]["nodes"] if node["id"] == heketi_node_id ] err_msg = "Node details for id '%s' not found" % heketi_node_id self.assertTrue(node_details, err_msg) err_msg = ("Multiple nodes with same id '%s' found %s" % (heketi_node_id, node_details)) self.assertEqual(len(node_details), 1, err_msg) err_msg = "Unexpected %s found '%s', expected '%s'" exp_storage_hostname = node_details[0]["hostnames"]["manage"][0] self.assertEqual( exp_storage_hostname, storage_hostname, err_msg % ( "hostname", exp_storage_hostname, storage_hostname, )) exp_storage_ip = node_details[0]["hostnames"]["storage"][0] self.assertEqual(exp_storage_ip, storage_ip, err_msg % ("IP address", exp_storage_ip, storage_ip)) zone = node_details[0]["zone"] self.assertEqual(zone, storage_zone, err_msg % ("zone", zone, storage_zone))
def test_heketi_prometheus_usedbytes_brickcount_on_device_delete( self, operation): """Validate used bytes,device count on heketi and prometheus""" h_node, h_server = self.heketi_client_node, self.heketi_server_url # Get list of additional devices for one of the Gluster nodes gluster_server_0 = list(self.gluster_servers_info.values())[0] manage_hostname = gluster_server_0.get("manage") self.assertTrue( manage_hostname, "IP Address is not specified for " "node {}".format(gluster_server_0)) device_name = gluster_server_0.get("additional_devices")[0] self.assertTrue( device_name, "Additional devices are not specified for " "node {}".format(gluster_server_0)) # Get node ID of the Gluster hostname node_list = heketi_ops.heketi_topology_info( h_node, h_server, json=True).get("clusters")[0].get("nodes") self.assertTrue( node_list, "Cluster info command returned empty list of nodes") node_id = [ node.get("id") for node in node_list if manage_hostname == node.get("hostnames").get("manage")[0]] self.assertTrue( node_id, "Failed to get node_id for {}".format(manage_hostname)) node_id = node_id[0] # Adding heketi device heketi_ops.heketi_device_add(h_node, h_server, device_name, node_id) node_info_after_addition = heketi_ops.heketi_node_info( h_node, h_server, node_id, json=True) device_id, bricks = None, None for device in node_info_after_addition.get("devices"): if device.get("name") == device_name: device_id, bricks = ( device.get("id"), len(device.get("bricks"))) break # Verify zero bricks on the device msg = ( "Number of bricks on the device {} of the nodes should be" "zero".format(device_name)) self.assertFalse(bricks, msg) self.addCleanup( heketi_ops.heketi_device_delete, h_node, h_server, device_id, raise_on_error=False) self.addCleanup( heketi_ops.heketi_device_remove, h_node, h_server, device_id, raise_on_error=False) self.addCleanup( heketi_ops.heketi_device_disable, h_node, h_server, device_id, raise_on_error=False) # Disable,Remove and Delete heketi device heketi_ops.heketi_device_disable(h_node, h_server, device_id) heketi_ops.heketi_device_remove(h_node, h_server, device_id) heketi_ops.heketi_device_delete(h_node, h_server, device_id) # Verify device deletion node_info_after_deletion = ( heketi_ops.heketi_node_info(h_node, h_server, node_id)) msg = ("Device {} should not be shown in node info of the node {}" "after the device deletion".format(device_id, node_id)) self.assertNotIn(device_id, node_info_after_deletion, msg) if operation == "usedbytes": # Validate heketi and prometheus device used bytes for w in waiter.Waiter(timeout=60, interval=10): device_used_bytes_prometheus = 0 device_used_bytes_metrics = 0 openshift_ops.switch_oc_project( self.ocp_master_node[0], 'openshift-monitoring') metric_result = self._fetch_metric_from_promtheus_pod( metric='heketi_device_used_bytes') for result in metric_result: if (node_id == result.get('cluster') and device_name == result.get('device')): device_used_bytes_prometheus += ( int(result.get('value')[1])) openshift_ops.switch_oc_project( self.ocp_master_node[0], 'glusterfs') metrics = heketi_ops.get_heketi_metrics(h_node, h_server) heketi_device_count_metric = ( metrics.get('heketi_device_used_bytes')) for result in heketi_device_count_metric: if (node_id == result.get('cluster') and device_name == result.get('device')): device_used_bytes_metrics = int(result.get('value')) if device_used_bytes_prometheus == device_used_bytes_metrics: break if w.expired: raise exceptions.ExecutionError( "Failed to update device details in prometheus") elif operation == "brickcount": # Validate heketi and prometheus device brick count for w in waiter.Waiter(timeout=60, interval=10): device_brick_count_prometheus = 0 device_brick_count_metrics = 0 metrics = heketi_ops.get_heketi_metrics(h_node, h_server) heketi_device_count_metric = metrics.get( 'heketi_device_brick_count') for result in heketi_device_count_metric: device_brick_count_metrics += int(result.get('value')) openshift_ops.switch_oc_project( self.ocp_master_node[0], 'openshift-monitoring') metric_result = self._fetch_metric_from_promtheus_pod( metric='heketi_device_brick_count') for result in metric_result: device_brick_count_prometheus += ( int(result.get('value')[1])) if device_brick_count_prometheus == device_brick_count_metrics: break if w.expired: raise exceptions.ExecutionError( "Failed to update device details in prometheus")
def test_volume_creation_of_size_greater_than_the_device_size(self): """Validate creation of a volume of size greater than the size of a device. """ h_node, h_url = self.heketi_client_node, self.heketi_server_url # Remove existing BHV to calculate freespace bhv_list = heketi_ops.get_block_hosting_volume_list(h_node, h_url) if bhv_list: for bhv in bhv_list: bhv_info = heketi_ops.heketi_volume_info(h_node, h_url, bhv, json=True) if bhv_info['blockinfo'].get('blockvolume') is None: heketi_ops.heketi_volume_delete(h_node, h_url, bhv) topology = heketi_ops.heketi_topology_info(h_node, h_url, json=True) nodes_free_space, nodes_ips = [], [] selected_nodes, selected_devices = [], [] cluster = topology['clusters'][0] node_count = len(cluster['nodes']) msg = ("At least 3 Nodes are required in cluster. " "But only %s Nodes are present." % node_count) if node_count < 3: self.skipTest(msg) online_nodes_count = 0 for node in cluster['nodes']: nodes_ips.append(node['hostnames']['storage'][0]) if node['state'] != 'online': continue online_nodes_count += 1 # Disable nodes after 3rd online nodes if online_nodes_count > 3: heketi_ops.heketi_node_disable(h_node, h_url, node['id']) self.addCleanup(heketi_ops.heketi_node_enable, h_node, h_url, node['id']) continue selected_nodes.append(node['id']) device_count = len(node['devices']) msg = ("At least 2 Devices are required on each Node." "But only %s Devices are present." % device_count) if device_count < 2: self.skipTest(msg) sel_devices, online_devices_count, free_space = [], 0, 0 for device in node['devices']: if device['state'] != 'online': continue online_devices_count += 1 # Disable devices after 2nd online devices if online_devices_count > 2: heketi_ops.heketi_device_disable(h_node, h_url, device['id']) self.addCleanup(heketi_ops.heketi_device_enable, h_node, h_url, device['id']) continue sel_devices.append(device['id']) free_space += int(device['storage']['free'] / (1024**2)) selected_devices.append(sel_devices) nodes_free_space.append(free_space) msg = ("At least 2 online Devices are required on each Node. " "But only %s Devices are online on Node: %s." % (online_devices_count, node['id'])) if online_devices_count < 2: self.skipTest(msg) msg = ("At least 3 online Nodes are required in cluster. " "But only %s Nodes are online in Cluster: %s." % (online_nodes_count, cluster['id'])) if online_nodes_count < 3: self.skipTest(msg) # Select node with minimum free space min_free_size = min(nodes_free_space) index = nodes_free_space.index(min_free_size) # Get max device size from selected node device_size = 0 for device in selected_devices[index]: device_info = heketi_ops.heketi_device_info(h_node, h_url, device, json=True) device_size = max(device_size, (int(device_info['storage']['total'] / (1024**2)))) vol_size = device_size + 1 if vol_size >= min_free_size: self.skipTest('Required free space %s is not available' % vol_size) # Create heketi volume with device size + 1 vol_info = self.create_heketi_volume_with_name_and_wait( name="volume_size_greater_than_device_size", size=vol_size, json=True) # Get gluster server IP's from heketi volume info glusterfs_servers = heketi_ops.get_vol_file_servers_and_hosts( h_node, h_url, vol_info['id']) # Verify gluster server IP's in heketi volume info msg = ("gluster IP's '%s' does not match with IP's '%s' found in " "heketi volume info" % (nodes_ips, glusterfs_servers['vol_servers'])) self.assertEqual(set(glusterfs_servers['vol_servers']), set(nodes_ips), msg) vol_name = vol_info['name'] gluster_v_info = self.get_gluster_vol_info(vol_name) # Verify replica count in gluster v info msg = "Volume %s is replica %s instead of replica 3" % ( vol_name, gluster_v_info['replicaCount']) self.assertEqual('3', gluster_v_info['replicaCount']) # Verify distCount in gluster v info msg = "Volume %s distCount is %s instead of distCount as 3" % ( vol_name, int(gluster_v_info['distCount'])) self.assertEqual( int(gluster_v_info['brickCount']) // 3, int(gluster_v_info['distCount']), msg) # Verify bricks count in gluster v info msg = ( "Volume %s does not have bricks count multiple of 3. It has %s" % (vol_name, gluster_v_info['brickCount'])) self.assertFalse(int(gluster_v_info['brickCount']) % 3, msg)