def test_heketi_device_delete(self): """Test Heketi device delete operation""" # Get list of additional devices for one of the Gluster nodes ip_with_devices = {} for gluster_server in g.config["gluster_servers"].values(): if not gluster_server.get("additional_devices"): continue ip_with_devices = { gluster_server['storage']: gluster_server['additional_devices'] } break # Skip test if no additional device is available if not ip_with_devices: self.skipTest( "No additional devices attached to any of the gluster nodes") # Select any additional device and get the node id of the gluster node h_node, h_server = self.heketi_client_node, self.heketi_server_url node_id, device_name = None, list(ip_with_devices.values())[0][0] topology_info = heketi_topology_info(h_node, h_server, json=True) for node in topology_info["clusters"][0]["nodes"]: if list(ip_with_devices.keys())[0] == ( node['hostnames']["storage"][0]): node_id = node["id"] break self.assertTrue(node_id) # Add additional device to the cluster heketi_device_add(h_node, h_server, device_name, node_id) # Get the device id and number of bricks on the device node_info_after_addition = heketi_node_info(h_node, h_server, node_id, json=True) device_id, bricks = None, None for device in node_info_after_addition["devices"]: if device["name"] == device_name: device_id, bricks = device["id"], len(device['bricks']) break self.assertTrue(device_id, "Device not added in expected node") # Delete heketi device heketi_device_disable(h_node, h_server, device_id) heketi_device_remove(h_node, h_server, device_id) heketi_device_delete(h_node, h_server, device_id) # Verify that there were no bricks on the newly added device msg = ( "Number of bricks on the device %s of the node %s should be zero" % (device_name, list(ip_with_devices.keys())[0])) self.assertEqual(0, bricks, msg) # Verify device deletion node_info_after_deletion = heketi_node_info(h_node, h_server, node_id) msg = ("Device %s should not be shown in node info of the node %s" "after the device deletion" % (device_id, node_id)) self.assertNotIn(device_id, node_info_after_deletion, msg)
def _add_heketi_device(self, device_name, node_id, raise_on_error=True): # Add device to heketi node device_id, device_size = None, None h_client, h_url = self.heketi_client_node, self.heketi_server_url heketi_device_add(h_client, h_url, device_name, node_id) node_info_addition = heketi_node_info(h_client, h_url, node_id, json=True) for device in node_info_addition.get("devices"): if device.get("name") == device_name: device_id = device.get("id") device_size = device.get("storage", {}).get("total") break self.addCleanup(heketi_device_delete, h_client, h_url, device_id, raise_on_error=raise_on_error) self.addCleanup(heketi_device_remove, h_client, h_url, device_id, raise_on_error=raise_on_error) self.addCleanup(heketi_device_disable, h_client, h_url, device_id, raise_on_error=raise_on_error) return device_id, device_size
def add_device_on_heketi_node(self, node_id, device_name): # Add Devices on heketi node heketi_ops.heketi_device_add( self.h_node, self.h_url, device_name, node_id) # Get device id of newly added device node_info = heketi_ops.heketi_node_info( self.h_node, self.h_url, node_id, json=True) for device in node_info['devices']: if device['name'] == device_name: return device['id'] raise AssertionError('Device %s did not found on node %s' % ( device_name, node_id))
def test_heketi_device_remove(self, delete_device): """Validate remove/delete device using heketi-cli""" gluster_server_0 = list(g.config["gluster_servers"].values())[0] try: device_name = gluster_server_0["additional_devices"][0] except (KeyError, IndexError): self.skipTest( "Additional disk is not specified for node with following " "hostnames and IP addresses: %s, %s." % (gluster_server_0.get( 'manage', '?'), gluster_server_0.get('storage', '?'))) manage_hostname = gluster_server_0["manage"] # Get node ID of the Gluster hostname topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(topo_info["clusters"][0]["nodes"], "Cluster info command returned empty list of nodes.") node_id = None for node in topo_info["clusters"][0]["nodes"]: if manage_hostname == node['hostnames']["manage"][0]: node_id = node["id"] break self.assertNotEqual( node_id, None, "No information about node_id for %s" % manage_hostname) # Iterate chosen node devices and pick the smallest online one. lowest_device_size = lowest_device_id = None online_hosts = self.get_online_nodes_disable_redundant() for host in online_hosts[0:3]: if node_id != host["id"]: continue for device in host["devices"]: if device["state"].strip().lower() != "online": continue if (lowest_device_size is None or device["storage"]["total"] < lowest_device_size): lowest_device_size = device["storage"]["total"] lowest_device_id = device["id"] lowest_device_name = device["name"] if lowest_device_id is None: self.skipTest( "Didn't find suitable device for disablement on '%s' node." % (node_id)) # Create volume vol_size = 1 vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, vol_size, json=True) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # Add extra device, then remember it's ID and size heketi_device_add(self.heketi_client_node, self.heketi_server_url, device_name, node_id) node_info_after_addition = heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) for device in node_info_after_addition["devices"]: if device["name"] != device_name: continue device_id_new = device["id"] device_size_new = device["storage"]["total"] self.addCleanup(heketi_device_delete, self.heketi_client_node, self.heketi_server_url, device_id_new) self.addCleanup(heketi_device_remove, self.heketi_client_node, self.heketi_server_url, device_id_new) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, device_id_new) if lowest_device_size > device_size_new: skip_msg = ("Skip test case, because newly added disk %s is " "smaller than device which we want to remove %s." % (device_size_new, lowest_device_size)) self.skipTest(skip_msg) g.log.info("Removing device id %s" % lowest_device_id) with self.assertRaises(AssertionError): out = heketi_device_remove(self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.assertFalse(True, "Device removal didn't fail: %s" % out) g.log.info("Device removal failed as expected") # Need to disable device before removing heketi_device_disable(self.heketi_client_node, self.heketi_server_url, lowest_device_id) if not delete_device: self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) # Remove device from Heketi try: heketi_device_remove(self.heketi_client_node, self.heketi_server_url, lowest_device_id) except Exception: if delete_device: self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) raise if not delete_device: self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) if delete_device: try: heketi_device_delete(self.heketi_client_node, self.heketi_server_url, lowest_device_id) except Exception: self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) raise self.addCleanup(heketi_device_add, self.heketi_client_node, self.heketi_server_url, lowest_device_name, node_id) # Create volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, vol_size, json=True) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) if delete_device: return # Check that none of volume's bricks is present on the device present = self.check_any_of_bricks_present_in_device( vol_info['bricks'], lowest_device_id) self.assertFalse( present, "Some of the '%s' volume bricks is present of the removed " "'%s' device." % (vol_info['id'], lowest_device_id))
def test_volume_expansion_no_free_space(self): """Validate volume expansion when there is no free space""" vol_size, expand_size, additional_devices_attached = None, 10, {} h_node, h_server_url = self.heketi_client_node, self.heketi_server_url # Get nodes info heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url) if len(heketi_node_id_list) < 3: self.skipTest("3 Heketi nodes are required.") # Disable 4th and other nodes for node_id in heketi_node_id_list[3:]: heketi_ops.heketi_node_disable(h_node, h_server_url, node_id) self.addCleanup( heketi_ops.heketi_node_enable, h_node, h_server_url, node_id) # Prepare first 3 nodes smallest_size = None err_msg = '' for node_id in heketi_node_id_list[0:3]: node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) # Disable second and other devices devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_id) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if (smallest_size is None or devices[0]["storage"]["free"] < smallest_size): smallest_size = devices[0]["storage"]["free"] for device in node_info["devices"][1:]: heketi_ops.heketi_device_disable( h_node, h_server_url, device["id"]) self.addCleanup( heketi_ops.heketi_device_enable, h_node, h_server_url, device["id"]) # Gather info about additional devices additional_device_name = None for gluster_server in self.gluster_servers: gluster_server_data = self.gluster_servers_info[gluster_server] g_manage = gluster_server_data["manage"] g_storage = gluster_server_data["storage"] if not (g_manage in node_info["hostnames"]["manage"] or g_storage in node_info["hostnames"]["storage"]): continue additional_device_name = (( gluster_server_data.get("additional_devices") or [''])[0]) break if not additional_device_name: err_msg += ("No 'additional_devices' are configured for " "'%s' node, which has following hostnames and " "IP addresses: %s.\n" % ( node_id, ', '.join(node_info["hostnames"]["manage"] + node_info["hostnames"]["storage"]))) continue heketi_ops.heketi_device_add( h_node, h_server_url, additional_device_name, node_id) additional_devices_attached.update( {node_id: additional_device_name}) # Schedule cleanup of the added devices for node_id in additional_devices_attached.keys(): node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) for device in node_info["devices"]: if device["name"] != additional_devices_attached[node_id]: continue self.addCleanup(self.detach_devices_attached, device["id"]) break else: self.fail("Could not find ID for added device on " "'%s' node." % node_id) if err_msg: self.skipTest(err_msg) # Temporary disable new devices self.disable_devices(additional_devices_attached) # Create volume and save info about it vol_size = int(smallest_size / (1024**2)) - 1 creation_info = heketi_ops.heketi_volume_create( h_node, h_server_url, vol_size, json=True) volume_name, volume_id = creation_info["name"], creation_info["id"] self.addCleanup( heketi_ops.heketi_volume_delete, h_node, h_server_url, volume_id, raise_on_error=False) volume_info_before_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) free_space_before_expansion = self.get_devices_summary_free_space() # Try to expand volume with not enough device space self.assertRaises( ExecutionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume self.enable_devices(additional_devices_attached) # Expand volume and validate results heketi_ops.heketi_volume_expand( h_node, h_server_url, volume_id, expand_size, json=True) free_space_after_expansion = self.get_devices_summary_free_space() self.assertGreater( free_space_before_expansion, free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) volume_info_after_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) self.assertGreater( volume_info_after_expansion["size"], volume_info_before_expansion["size"], "Size of %s not increased" % volume_id) self.assertGreater( num_of_bricks_after_expansion, num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0) # Delete volume and validate release of the used space heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertGreater( free_space_after_deletion, free_space_after_expansion, "Free space not reclaimed after deletion of volume %s" % volume_id)
def test_volume_expansion_no_free_space(self): """Validate volume expansion when there is no free space""" vol_size, expand_size, additional_devices_attached = None, 10, {} h_node, h_server_url = self.heketi_client_node, self.heketi_server_url # Get nodes info heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url) if len(heketi_node_id_list) < 3: self.skipTest("3 Heketi nodes are required.") # Disable 4th and other nodes for node_id in heketi_node_id_list[3:]: heketi_ops.heketi_node_disable(h_node, h_server_url, node_id) self.addCleanup( heketi_ops.heketi_node_enable, h_node, h_server_url, node_id) # Prepare first 3 nodes smallest_size = None err_msg = '' for node_id in heketi_node_id_list[0:3]: node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) # Disable second and other devices devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_id) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if (smallest_size is None or devices[0]["storage"]["free"] < smallest_size): smallest_size = devices[0]["storage"]["free"] for device in node_info["devices"][1:]: heketi_ops.heketi_device_disable( h_node, h_server_url, device["id"]) self.addCleanup( heketi_ops.heketi_device_enable, h_node, h_server_url, device["id"]) # Gather info about additional devices additional_device_name = None for gluster_server in self.gluster_servers: gluster_server_data = self.gluster_servers_info[gluster_server] g_manage = gluster_server_data["manage"] g_storage = gluster_server_data["storage"] if not (g_manage in node_info["hostnames"]["manage"] or g_storage in node_info["hostnames"]["storage"]): continue additional_device_name = (( gluster_server_data.get("additional_devices") or [''])[0]) break if not additional_device_name: err_msg += ("No 'additional_devices' are configured for " "'%s' node, which has following hostnames and " "IP addresses: %s.\n" % ( node_id, ', '.join( node_info["hostnames"]["manage"] + node_info["hostnames"]["storage"]))) continue heketi_ops.heketi_device_add( h_node, h_server_url, additional_device_name, node_id) additional_devices_attached.update( {node_id: additional_device_name}) # Schedule cleanup of the added devices for node_id in additional_devices_attached.keys(): node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) for device in node_info["devices"]: if device["name"] != additional_devices_attached[node_id]: continue self.addCleanup(self.detach_devices_attached, device["id"]) break else: self.fail("Could not find ID for added device on " "'%s' node." % node_id) if err_msg: self.skipTest(err_msg) # Temporary disable new devices self.disable_devices(additional_devices_attached) # Create volume and save info about it vol_size = int(smallest_size / (1024**2)) - 1 creation_info = heketi_ops.heketi_volume_create( h_node, h_server_url, vol_size, json=True) volume_name, volume_id = creation_info["name"], creation_info["id"] self.addCleanup( heketi_ops.heketi_volume_delete, h_node, h_server_url, volume_id, raise_on_error=False) volume_info_before_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) free_space_before_expansion = self.get_devices_summary_free_space() # Try to expand volume with not enough device space self.assertRaises( AssertionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume self.enable_devices(additional_devices_attached) # Expand volume and validate results heketi_ops.heketi_volume_expand( h_node, h_server_url, volume_id, expand_size, json=True) free_space_after_expansion = self.get_devices_summary_free_space() self.assertGreater( free_space_before_expansion, free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) volume_info_after_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) self.assertGreater( volume_info_after_expansion["size"], volume_info_before_expansion["size"], "Size of %s not increased" % volume_id) self.assertGreater( num_of_bricks_after_expansion, num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0) # Delete volume and validate release of the used space heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertGreater( free_space_after_deletion, free_space_after_expansion, "Free space not reclaimed after deletion of volume %s" % volume_id)
def test_heketi_prometheus_usedbytes_brickcount_on_device_delete( self, operation): """Validate used bytes,device count on heketi and prometheus""" h_node, h_server = self.heketi_client_node, self.heketi_server_url # Get list of additional devices for one of the Gluster nodes gluster_server_0 = list(self.gluster_servers_info.values())[0] manage_hostname = gluster_server_0.get("manage") self.assertTrue( manage_hostname, "IP Address is not specified for " "node {}".format(gluster_server_0)) device_name = gluster_server_0.get("additional_devices")[0] self.assertTrue( device_name, "Additional devices are not specified for " "node {}".format(gluster_server_0)) # Get node ID of the Gluster hostname node_list = heketi_ops.heketi_topology_info( h_node, h_server, json=True).get("clusters")[0].get("nodes") self.assertTrue( node_list, "Cluster info command returned empty list of nodes") node_id = [ node.get("id") for node in node_list if manage_hostname == node.get("hostnames").get("manage")[0]] self.assertTrue( node_id, "Failed to get node_id for {}".format(manage_hostname)) node_id = node_id[0] # Adding heketi device heketi_ops.heketi_device_add(h_node, h_server, device_name, node_id) node_info_after_addition = heketi_ops.heketi_node_info( h_node, h_server, node_id, json=True) device_id, bricks = None, None for device in node_info_after_addition.get("devices"): if device.get("name") == device_name: device_id, bricks = ( device.get("id"), len(device.get("bricks"))) break # Verify zero bricks on the device msg = ( "Number of bricks on the device {} of the nodes should be" "zero".format(device_name)) self.assertFalse(bricks, msg) self.addCleanup( heketi_ops.heketi_device_delete, h_node, h_server, device_id, raise_on_error=False) self.addCleanup( heketi_ops.heketi_device_remove, h_node, h_server, device_id, raise_on_error=False) self.addCleanup( heketi_ops.heketi_device_disable, h_node, h_server, device_id, raise_on_error=False) # Disable,Remove and Delete heketi device heketi_ops.heketi_device_disable(h_node, h_server, device_id) heketi_ops.heketi_device_remove(h_node, h_server, device_id) heketi_ops.heketi_device_delete(h_node, h_server, device_id) # Verify device deletion node_info_after_deletion = ( heketi_ops.heketi_node_info(h_node, h_server, node_id)) msg = ("Device {} should not be shown in node info of the node {}" "after the device deletion".format(device_id, node_id)) self.assertNotIn(device_id, node_info_after_deletion, msg) if operation == "usedbytes": # Validate heketi and prometheus device used bytes for w in waiter.Waiter(timeout=60, interval=10): device_used_bytes_prometheus = 0 device_used_bytes_metrics = 0 openshift_ops.switch_oc_project( self.ocp_master_node[0], 'openshift-monitoring') metric_result = self._fetch_metric_from_promtheus_pod( metric='heketi_device_used_bytes') for result in metric_result: if (node_id == result.get('cluster') and device_name == result.get('device')): device_used_bytes_prometheus += ( int(result.get('value')[1])) openshift_ops.switch_oc_project( self.ocp_master_node[0], 'glusterfs') metrics = heketi_ops.get_heketi_metrics(h_node, h_server) heketi_device_count_metric = ( metrics.get('heketi_device_used_bytes')) for result in heketi_device_count_metric: if (node_id == result.get('cluster') and device_name == result.get('device')): device_used_bytes_metrics = int(result.get('value')) if device_used_bytes_prometheus == device_used_bytes_metrics: break if w.expired: raise exceptions.ExecutionError( "Failed to update device details in prometheus") elif operation == "brickcount": # Validate heketi and prometheus device brick count for w in waiter.Waiter(timeout=60, interval=10): device_brick_count_prometheus = 0 device_brick_count_metrics = 0 metrics = heketi_ops.get_heketi_metrics(h_node, h_server) heketi_device_count_metric = metrics.get( 'heketi_device_brick_count') for result in heketi_device_count_metric: device_brick_count_metrics += int(result.get('value')) openshift_ops.switch_oc_project( self.ocp_master_node[0], 'openshift-monitoring') metric_result = self._fetch_metric_from_promtheus_pod( metric='heketi_device_brick_count') for result in metric_result: device_brick_count_prometheus += ( int(result.get('value')[1])) if device_brick_count_prometheus == device_brick_count_metrics: break if w.expired: raise exceptions.ExecutionError( "Failed to update device details in prometheus")
def test_create_heketi_cluster_and_add_node(self): """Test heketi node add to a newly created cluster""" storage_host_info = g.config.get("additional_gluster_servers") if not storage_host_info: self.skipTest( "Skip test case as 'additional_gluster_servers' option is " "not provided in config file") storage_host_info = list(storage_host_info.values())[0] try: storage_hostname = storage_host_info["manage"] storage_ip = storage_host_info["storage"] storage_device = storage_host_info["devices"][0] except KeyError: msg = ("Config options 'additional_gluster_servers.manage' " "'additional_gluster_servers.storage' and " "'additional_gluster_servers.devices' " "must be set.") g.log.error(msg) raise exceptions.ConfigError(msg) h_client, h_server = self.heketi_client_node, self.heketi_server_url storage_zone = 1 cluster_id = heketi_ops.heketi_cluster_create(self.heketi_client_node, self.heketi_server_url, json=True)["id"] self.addCleanup(heketi_ops.heketi_cluster_delete, self.heketi_client_node, self.heketi_server_url, cluster_id) self.configure_node_to_run_gluster(storage_hostname) heketi_node_info = heketi_ops.heketi_node_add(h_client, h_server, storage_zone, cluster_id, storage_hostname, storage_ip, json=True) heketi_node_id = heketi_node_info["id"] self.addCleanup(heketi_ops.heketi_node_delete, h_client, h_server, heketi_node_id) self.addCleanup(heketi_ops.heketi_node_remove, h_client, h_server, heketi_node_id) self.addCleanup(heketi_ops.heketi_node_disable, h_client, h_server, heketi_node_id) self.assertEqual( heketi_node_info["cluster"], cluster_id, "Node got added in unexpected cluster exp: %s, act: %s" % (cluster_id, heketi_node_info["cluster"])) heketi_ops.heketi_device_add(h_client, h_server, storage_device, heketi_node_id) heketi_node_info = heketi_ops.heketi_node_info(h_client, h_server, heketi_node_id, json=True) device_id = None for device in heketi_node_info["devices"]: if device["name"] == storage_device: device_id = device["id"] break err_msg = ("Failed to add device %s on node %s" % (storage_device, heketi_node_id)) self.assertTrue(device_id, err_msg) self.addCleanup(heketi_ops.heketi_device_delete, h_client, h_server, device_id) self.addCleanup(heketi_ops.heketi_device_remove, h_client, h_server, device_id) self.addCleanup(heketi_ops.heketi_device_disable, h_client, h_server, device_id) cluster_info = heketi_ops.heketi_cluster_info(h_client, h_server, cluster_id, json=True) self.assertIn( heketi_node_info["id"], cluster_info["nodes"], "Newly added node %s not found in cluster %s, cluster info %s" % (heketi_node_info["id"], cluster_id, cluster_info)) topology_info = heketi_ops.heketi_topology_info(h_client, h_server, json=True) cluster_details = [ cluster for cluster in topology_info["clusters"] if cluster["id"] == cluster_id ] err_msg = "Cluster details for id '%s' not found" % cluster_id self.assertTrue(cluster_details, err_msg) err_msg = ("Multiple clusters with same id '%s' found %s" % (cluster_id, cluster_details)) self.assertEqual(len(cluster_details), 1, err_msg) node_details = [ node for node in cluster_details[0]["nodes"] if node["id"] == heketi_node_id ] err_msg = "Node details for id '%s' not found" % heketi_node_id self.assertTrue(node_details, err_msg) err_msg = ("Multiple nodes with same id '%s' found %s" % (heketi_node_id, node_details)) self.assertEqual(len(node_details), 1, err_msg) err_msg = "Unexpected %s found '%s', expected '%s'" exp_storage_hostname = node_details[0]["hostnames"]["manage"][0] self.assertEqual( exp_storage_hostname, storage_hostname, err_msg % ( "hostname", exp_storage_hostname, storage_hostname, )) exp_storage_ip = node_details[0]["hostnames"]["storage"][0] self.assertEqual(exp_storage_ip, storage_ip, err_msg % ("IP address", exp_storage_ip, storage_ip)) zone = node_details[0]["zone"] self.assertEqual(zone, storage_zone, err_msg % ("zone", zone, storage_zone))
def _add_new_device_and_remove_existing_device( self, is_delete_device, add_device_name, node_id, add_back_again=False, skip_cleanup_addition=False): """Delete or remove device and also add one device on the same node. """ h_client, h_url = self.heketi_client_node, self.heketi_server_url raise_on_error = False if add_back_again else True # Iterate chosen node devices and pick the smallest online one. lowest_device_size = lowest_device_id = None online_hosts = self.get_online_nodes_disable_redundant() for host in online_hosts[0:3]: if node_id != host["id"]: continue for device in host["devices"]: if device["state"].strip().lower() != "online": continue if (lowest_device_size is None or device["storage"]["total"] < lowest_device_size): lowest_device_size = device["storage"]["total"] lowest_device_id = device["id"] lowest_device_name = device["name"] if lowest_device_id is None: self.skipTest( "Didn't find suitable device for disablement on '%s' node." % (node_id)) # Create volume vol_size = 1 vol_info = heketi_volume_create(h_client, h_url, vol_size, json=True) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # Add extra device, then remember it's ID and size heketi_device_add(h_client, h_url, add_device_name, node_id) node_info_after_addition = heketi_node_info(h_client, h_url, node_id, json=True) for device in node_info_after_addition["devices"]: if device["name"] != add_device_name: continue device_id_new = device["id"] device_size_new = device["storage"]["total"] self.addCleanup(heketi_device_delete, h_client, h_url, device_id_new, raise_on_error=raise_on_error) self.addCleanup(heketi_device_remove, h_client, h_url, device_id_new, raise_on_error=raise_on_error) self.addCleanup(heketi_device_disable, h_client, h_url, device_id_new, raise_on_error=raise_on_error) if lowest_device_size > device_size_new: skip_msg = ("Skip test case, because newly added disk %s is " "smaller than device which we want to remove %s." % (device_size_new, lowest_device_size)) self.skipTest(skip_msg) g.log.info("Removing device id %s" % lowest_device_id) with self.assertRaises(AssertionError): out = heketi_device_remove(h_client, h_url, lowest_device_id) self.addCleanup(heketi_device_enable, h_client, h_url, lowest_device_id) self.addCleanup(heketi_device_disable, h_client, h_url, lowest_device_id) self.assertFalse(True, "Device removal didn't fail: %s" % out) g.log.info("Device removal failed as expected") # Need to disable device before removing heketi_device_disable(h_client, h_url, lowest_device_id) if not is_delete_device: self.addCleanup(heketi_device_enable, h_client, h_url, lowest_device_id) # Remove device from Heketi try: heketi_device_remove(h_client, h_url, lowest_device_id) except Exception: if is_delete_device: self.addCleanup(heketi_device_enable, h_client, h_url, lowest_device_id, raise_on_error=raise_on_error) raise if not is_delete_device: self.addCleanup(heketi_device_disable, h_client, h_url, lowest_device_id) if is_delete_device: try: heketi_device_delete(h_client, h_url, lowest_device_id) except Exception: self.addCleanup(heketi_device_enable, h_client, h_url, lowest_device_id, raise_on_error=raise_on_error) self.addCleanup(heketi_device_disable, h_client, h_url, lowest_device_id, raise_on_error=raise_on_error) raise if not skip_cleanup_addition: # Do not add the additional device back, intially added self.addCleanup(heketi_device_add, h_client, h_url, lowest_device_name, node_id, raise_on_error=raise_on_error) # Create volume vol_info = heketi_volume_create(h_client, h_url, vol_size, json=True) self.addCleanup(heketi_volume_delete, h_client, h_url, vol_info['id']) if is_delete_device: return lowest_device_name # Check that none of volume's bricks is present on the device present = self.check_any_of_bricks_present_in_device( vol_info['bricks'], lowest_device_id) self.assertFalse( present, "Some of the '%s' volume bricks is present of the removed " "'%s' device." % (vol_info['id'], lowest_device_id))
def test_dev_path_mapping_heketi_node_delete(self): """Validate dev path mapping for heketi node deletion lifecycle""" h_client, h_url = self.heketi_client_node, self.heketi_server_url node_ids = heketi_ops.heketi_node_list(h_client, h_url) self.assertTrue(node_ids, "Failed to get heketi node list") # Fetch #4th node for the operations h_disable_node = node_ids[3] # Fetch bricks on the devices before volume create h_node_details_before, h_node = self._get_bricks_and_device_details() # Bricks count on the node before pvc creation brick_count_before = [count[1] for count in h_node_details_before] # Create file volume with app pod and verify IO's # and compare path, UUID, vg_name pod_name, dc_name, use_percent = self._create_app_pod_and_verify_pvs() # Check if IO's are running use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name)) # Fetch bricks on the devices after volume create h_node_details_after, h_node = self._get_bricks_and_device_details() # Bricks count on the node after pvc creation brick_count_after = [count[1] for count in h_node_details_after] self.assertGreater( sum(brick_count_after), sum(brick_count_before), "Failed to add bricks on the node {}".format(h_node)) self.addCleanup(heketi_ops.heketi_node_disable, h_client, h_url, h_disable_node) # Enable the #4th node heketi_ops.heketi_node_enable(h_client, h_url, h_disable_node) node_info = heketi_ops.heketi_node_info(h_client, h_url, h_disable_node, json=True) h_node_id = node_info['id'] self.assertEqual(node_info['state'], "online", "Failed to enable node {}".format(h_disable_node)) # Disable the node and check for brick migrations self.addCleanup(heketi_ops.heketi_node_enable, h_client, h_url, h_node, raise_on_error=False) heketi_ops.heketi_node_disable(h_client, h_url, h_node) node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) self.assertEqual(node_info['state'], "offline", "Failed to disable node {}".format(h_node)) # Before bricks migration h_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) # Bricks before migration on the node i.e to be deleted bricks_counts_before = 0 for device in h_node_info['devices']: bricks_counts_before += (len(device['bricks'])) # Remove the node heketi_ops.heketi_node_remove(h_client, h_url, h_node) # After bricks migration h_node_info_after = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) # Bricks after migration on the node i.e to be delete bricks_counts = 0 for device in h_node_info_after['devices']: bricks_counts += (len(device['bricks'])) self.assertFalse( bricks_counts, "Failed to remove all the bricks from node {}".format(h_node)) # Old node which is to deleted, new node were bricks resides old_node, new_node = h_node, h_node_id # Node info for the new node were brick reside after migration h_node_info_new = heketi_ops.heketi_node_info(h_client, h_url, new_node, json=True) bricks_counts_after = 0 for device in h_node_info_new['devices']: bricks_counts_after += (len(device['bricks'])) self.assertEqual( bricks_counts_before, bricks_counts_after, "Failed to migrated bricks from {} node to {}".format( old_node, new_node)) # Fetch device list i.e to be deleted h_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) devices_list = [[device['id'], device['name']] for device in h_node_info['devices']] for device in devices_list: device_id = device[0] device_name = device[1] self.addCleanup(heketi_ops.heketi_device_add, h_client, h_url, device_name, h_node, raise_on_error=False) # Device deletion from heketi node device_delete = heketi_ops.heketi_device_delete( h_client, h_url, device_id) self.assertTrue(device_delete, "Failed to delete the device {}".format(device_id)) node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) cluster_id = node_info['cluster'] zone = node_info['zone'] storage_hostname = node_info['hostnames']['manage'][0] storage_ip = node_info['hostnames']['storage'][0] # Delete the node self.addCleanup(heketi_ops.heketi_node_add, h_client, h_url, zone, cluster_id, storage_hostname, storage_ip, raise_on_error=False) heketi_ops.heketi_node_delete(h_client, h_url, h_node) # Verify if the node is deleted node_ids = heketi_ops.heketi_node_list(h_client, h_url) self.assertNotIn(old_node, node_ids, "Failed to delete the node {}".format(old_node)) # Check if IO's are running use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name)) # Adding node back h_node_info = heketi_ops.heketi_node_add(h_client, h_url, zone, cluster_id, storage_hostname, storage_ip, json=True) self.assertTrue( h_node_info, "Failed to add the node in the cluster {}".format(cluster_id)) h_node_id = h_node_info["id"] # Adding devices to the new node for device in devices_list: storage_device = device[1] # Add device to the new heketi node heketi_ops.heketi_device_add(h_client, h_url, storage_device, h_node_id) heketi_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node_id, json=True) device_id = None for device in heketi_node_info["devices"]: if device["name"] == storage_device: device_id = device["id"] break self.assertTrue( device_id, "Failed to add device {} on node {}".format( storage_device, h_node_id)) # Create n pvc in order to verfiy if the bricks reside on the new node pvc_amount, pvc_size = 5, 1 # Fetch bricks on the devices before volume create h_node_details_before, h_node = self._get_bricks_and_device_details() # Bricks count on the node before pvc creation brick_count_before = [count[1] for count in h_node_details_before] # Create file volumes pvc_name = self.create_and_wait_for_pvcs(pvc_size=pvc_size, pvc_amount=pvc_amount) self.assertEqual(len(pvc_name), pvc_amount, "Failed to create {} pvc".format(pvc_amount)) # Fetch bricks on the devices before volume create h_node_details_after, h_node = self._get_bricks_and_device_details() # Bricks count on the node after pvc creation brick_count_after = [count[1] for count in h_node_details_after] self.assertGreater( sum(brick_count_after), sum(brick_count_before), "Failed to add bricks on the new node {}".format(new_node)) # Check if IO's are running after new node is added use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name))
def test_dev_path_mapping_heketi_device_delete(self): """Validate dev path mapping for heketi device delete lifecycle""" h_client, h_url = self.heketi_client_node, self.heketi_server_url node_ids = heketi_ops.heketi_node_list(h_client, h_url) self.assertTrue(node_ids, "Failed to get heketi node list") # Fetch #4th node for the operations h_disable_node = node_ids[3] # Fetch bricks on the devices before volume create h_node_details_before, h_node = self._get_bricks_and_device_details() # Bricks count on the node before pvc creation brick_count_before = [count[1] for count in h_node_details_before] # Create file volume with app pod and verify IO's # and compare path, UUID, vg_name pod_name, dc_name, use_percent = self._create_app_pod_and_verify_pvs() # Check if IO's are running use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name)) # Fetch bricks on the devices after volume create h_node_details_after, h_node = self._get_bricks_and_device_details() # Bricks count on the node after pvc creation brick_count_after = [count[1] for count in h_node_details_after] self.assertGreater( sum(brick_count_after), sum(brick_count_before), "Failed to add bricks on the node {}".format(h_node)) # Enable the #4th node heketi_ops.heketi_node_enable(h_client, h_url, h_disable_node) node_info = heketi_ops.heketi_node_info(h_client, h_url, h_disable_node, json=True) h_node_id = node_info['id'] self.assertEqual(node_info['state'], "online", "Failed to enable node {}".format(h_disable_node)) # Fetch device list i.e to be deleted h_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) devices_list = [[device['id'], device['name']] for device in h_node_info['devices']] # Device deletion operation for device in devices_list: device_id, device_name = device[0], device[1] self.addCleanup(heketi_ops.heketi_device_enable, h_client, h_url, device_id, raise_on_error=False) # Disable device from heketi device_disable = heketi_ops.heketi_device_disable( h_client, h_url, device_id) self.assertTrue( device_disable, "Device {} could not be disabled".format(device_id)) device_info = heketi_ops.heketi_device_info(h_client, h_url, device_id, json=True) self.assertEqual(device_info['state'], "offline", "Failed to disable device {}".format(device_id)) # Remove device from heketi device_remove = heketi_ops.heketi_device_remove( h_client, h_url, device_id) self.assertTrue(device_remove, "Device {} could not be removed".format(device_id)) # Bricks after device removal device_info = heketi_ops.heketi_device_info(h_client, h_url, device_id, json=True) bricks_count_after = len(device_info['bricks']) self.assertFalse( bricks_count_after, "Failed to remove the bricks from the device {}".format( device_id)) # Delete device from heketi self.addCleanup(heketi_ops.heketi_device_add, h_client, h_url, device_name, h_node, raise_on_error=False) device_delete = heketi_ops.heketi_device_delete( h_client, h_url, device_id) self.assertTrue(device_delete, "Device {} could not be deleted".format(device_id)) # Check if IO's are running after device is deleted use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name)) # Add device operations for device in devices_list: device_name = device[1] # Add device back to the node heketi_ops.heketi_device_add(h_client, h_url, device_name, h_node) # Fetch device info after device add node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) device_id = None for device in node_info["devices"]: if device["name"] == device_name: device_id = device["id"] break self.assertTrue( device_id, "Failed to add device {} on node" " {}".format(device_name, h_node)) # Disable the #4th node heketi_ops.heketi_node_disable(h_client, h_url, h_node_id) node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node_id, json=True) self.assertEqual(node_info['state'], "offline", "Failed to disable node {}".format(h_node_id)) pvc_amount, pvc_size = 5, 1 # Fetch bricks on the devices before volume create h_node_details_before, h_node = self._get_bricks_and_device_details() # Bricks count on the node before pvc creation brick_count_before = [count[1] for count in h_node_details_before] # Create file volumes pvc_name = self.create_and_wait_for_pvcs(pvc_size=pvc_size, pvc_amount=pvc_amount) self.assertEqual(len(pvc_name), pvc_amount, "Failed to create {} pvc".format(pvc_amount)) # Fetch bricks on the devices after volume create h_node_details_after, h_node = self._get_bricks_and_device_details() # Bricks count on the node after pvc creation brick_count_after = [count[1] for count in h_node_details_after] self.assertGreater( sum(brick_count_after), sum(brick_count_before), "Failed to add bricks on the node {}".format(h_node)) # Check if IO's are running after new device is added use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name))
def test_udev_usage_in_container(self): """Validate LVM inside container does not use udev""" # Skip the TC if independent mode deployment if not self.is_containerized_gluster(): self.skipTest("Skipping this test case as it needs to run on " "converged mode deployment") h_client, h_url = self.heketi_client_node, self.heketi_server_url server_info = list(g.config.get('gluster_servers').values())[0] server_node = server_info.get('manage') additional_device = server_info.get('additional_devices')[0] # command to run pvscan cmd_pvscan = "timeout 300 pvscan" # Get pod name from on host for pod_info in self.pod_name: if pod_info.get('pod_hostname') == server_node: pod_name = pod_info.get('pod_name') break # Create file volume vol_info = heketi_ops.heketi_volume_create(h_client, h_url, self.volume_size, json=True) self.addCleanup(heketi_ops.heketi_volume_delete, h_client, h_url, vol_info.get("id")) # Create block volume block_vol_info = heketi_ops.heketi_blockvolume_create(h_client, h_url, self.volume_size, json=True) self.addCleanup(heketi_ops.heketi_blockvolume_delete, h_client, h_url, block_vol_info.get("id")) # Check dmeventd service in container err_msg = "dmeventd.service is running on setup" with self.assertRaises(AssertionError, msg=err_msg): openshift_ops.oc_rsh(self.oc_node, pod_name, "systemctl is-active dmeventd.service") # Service dmeventd should not be running in background with self.assertRaises(AssertionError, msg=err_msg): openshift_ops.oc_rsh(self.oc_node, pod_name, "ps aux | grep dmeventd.service") # Perform a pvscan in contaier openshift_ops.oc_rsh(self.oc_node, pod_name, cmd_pvscan) # Get heketi node to add new device heketi_node_list = heketi_ops.heketi_node_list(h_client, h_url) for h_node_id in heketi_node_list: h_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node_id, json=True) h_node_host = h_node_info.get('hostnames', {}).get('manage')[0] if h_node_host == server_node: break # Add new device to the node heketi_ops.heketi_device_add(h_client, h_url, additional_device, h_node_id) h_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node_id, json=True) h_device_id = [ device.get('id') for device in h_node_info.get('devices') if device.get('name') == additional_device ] self.addCleanup(heketi_ops.heketi_device_delete, h_client, h_url, h_device_id[0]) self.addCleanup(heketi_ops.heketi_device_remove, h_client, h_url, h_device_id[0]) self.addCleanup(heketi_ops.heketi_device_disable, h_client, h_url, h_device_id[0]) # Reboot the node on which device is added self.addCleanup(self._check_heketi_and_gluster_pod_after_node_reboot, server_node) node_ops.node_reboot_by_command(server_node) # Wait node to become NotReady custom = r'":.status.conditions[?(@.type==\"Ready\")]".status' for w in waiter.Waiter(300, 10): status = openshift_ops.oc_get_custom_resource( self.oc_node, 'node', custom, server_node) if status[0] == 'False': break if w.expired: raise exceptions.ExecutionError( "Failed to bring node down {}".format(server_node)) # Wait for node to become ready openshift_ops.wait_for_ocp_node_be_ready(self.oc_node, server_node) # Wait for heketi and glusterfs pod to become ready self._check_heketi_and_gluster_pod_after_node_reboot(server_node) # Perform a pvscan in contaier openshift_ops.oc_rsh(self.oc_node, pod_name, cmd_pvscan)