def test_to_check_deletion_of_node(self): """Validate deletion of a node which contains devices""" # Create Heketi volume to make sure we have devices with usages heketi_url = self.heketi_server_url vol = heketi_volume_create(self.heketi_client_node, heketi_url, 1, json=True) self.assertTrue(vol, "Failed to create heketi volume.") g.log.info("Heketi volume successfully created") volume_id = vol["bricks"][0]["volume"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) # Pick up suitable node node_ids = heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_ids) for node_id in node_ids: node_info = heketi_node_info(self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue for device in node_info['devices']: if device['state'].lower() != 'online': continue if device['storage']['used']: node_id = node_info['id'] break else: self.assertTrue( node_id, "Failed to find online node with online device which " "has some usages.") # Try to delete the node by its ID g.log.info("Trying to delete the node which contains devices in it. " "Expecting failure.") self.assertRaises(ExecutionError, heketi_node_delete, self.heketi_client_node, heketi_url, node_id) # Make sure our node hasn't been deleted g.log.info("Listing heketi node list") node_list = heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_list, ("Failed to list heketi nodes")) self.assertIn(node_id, node_list) node_info = heketi_node_info(self.heketi_client_node, heketi_url, node_id, json=True) self.assertEqual(node_info['state'].lower(), 'online')
def enable_disable_devices(self, additional_devices_attached, enable=True): """ Method to enable and disable devices """ op = 'enable' if enable else 'disable' for node_id in additional_devices_attached.keys(): node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) if not enable: self.assertNotEqual(node_info, False, "Node info for node %s failed" % node_id) for device in node_info["devices"]: if device["name"] == additional_devices_attached[node_id]: out = getattr(heketi_ops, 'heketi_device_%s' % op)( self.heketi_client_node, self.heketi_server_url, device["id"], json=True) if out is False: g.log.info("Device %s could not be %sd" % (device["id"], op)) else: g.log.info("Device %s %sd" % (device["id"], op))
def get_devices_summary_free_space(self): """ Calculates minimum free space per device and returns total free space across all devices """ free_spaces = [] heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) for node_id in heketi_node_id_list: node_info_dict = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) total_free_space = 0 for device in node_info_dict["devices"]: total_free_space += device["storage"]["free"] free_spaces.append(total_free_space) total_free_space = sum(free_spaces) / (1024**2) total_free_space = int(math.floor(total_free_space)) return total_free_space
def setUp(self): super(TestArbiterVolumeCreateExpandDelete, self).setUp() self.node = self.ocp_master_node[0] # Mark one of the Heketi nodes as arbiter-supported if none of # existent nodes or devices already enabled to support it. self.heketi_server_url = self.sc.get('resturl') arbiter_tags = ('required', 'supported') arbiter_already_supported = False self.node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) for node_id in self.node_id_list[::-1]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) if node_info.get('tags', {}).get('arbiter') in arbiter_tags: arbiter_already_supported = True break for device in node_info['devices'][::-1]: if device.get('tags', {}).get('arbiter') in arbiter_tags: arbiter_already_supported = True break else: continue break if not arbiter_already_supported: self._set_arbiter_tag_with_further_revert(self.heketi_client_node, self.heketi_server_url, 'node', self.node_id_list[0], 'supported')
def test_expand_arbiter_volume_according_to_avg_file_size( self, avg_file_size, expected_brick_size, vol_expand=True): """Validate expansion of arbiter volume with diff avg file size""" data_hosts = [] arbiter_hosts = [] # set tags arbiter:disabled, arbiter:required for i, node_id in enumerate(self.node_id_list): self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'disabled' if i < 2 else 'required') node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) (data_hosts.append(node_info['hostnames']['storage'][0]) if i < 2 else arbiter_hosts.append(node_info['hostnames']['storage'][0])) self.assertEqual(node_info['tags']['arbiter'], 'disabled' if i < 2 else 'required') # Create sc with gluster arbiter info self.create_storage_class(is_arbiter_vol=True, allow_volume_expansion=True, arbiter_avg_file_size=avg_file_size) # Create PVC and wait for it to be in 'Bound' state self.create_and_wait_for_pvc() vol_expanded = False for i in range(2): vol_info = get_gluster_vol_info_by_pvc_name( self.node, self.pvc_name) bricks = ( self.verify_amount_and_proportion_of_arbiter_and_data_bricks( vol_info, arbiter_bricks=(2 if vol_expanded else 1), data_bricks=(4 if vol_expanded else 2))) # verify arbiter bricks lies on arbiter hosts for brick in bricks['arbiter_list']: ip, brick_name = brick['name'].split(':') self.assertIn(ip, arbiter_hosts) # verify the size of arbiter brick cmd = "df -h %s --output=size | tail -1" % brick_name out = cmd_run_on_gluster_pod_or_node(self.node, cmd, ip) self.assertEqual(out, expected_brick_size) # verify that data bricks lies on data hosts for brick in bricks['data_list']: self.assertIn(brick['name'].split(':')[0], data_hosts) if vol_expanded or not vol_expand: break # Expand PVC and verify the size pvc_size = 2 resize_pvc(self.node, self.pvc_name, pvc_size) verify_pvc_size(self.node, self.pvc_name, pvc_size) vol_expanded = True
def get_node_info(self, node_id): """ Get node information from node_id. :param node_id: str node ID :return node_info: list node information """ node_info = heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) self.assertNotEqual(node_info, False, "Node info on %s failed" % node_id) return node_info
def _get_free_space(self): """Get free space in each heketi device""" free_spaces = [] heketi_node_id_list = heketi_node_list( self.heketi_client_node, self.heketi_server_url) for node_id in heketi_node_id_list: node_info_dict = heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) total_free_space = 0 for device in node_info_dict["devices"]: total_free_space += device["storage"]["free"] free_spaces.append(total_free_space) total_free_space = int(math.floor(sum(free_spaces) / (1024**2))) return total_free_space
def _get_vol_size(self): # Get available free space disabling redundant nodes min_free_space_gb = 5 heketi_url = self.heketi_server_url node_ids = heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_ids) nodes = {} min_free_space = min_free_space_gb * 1024**2 for node_id in node_ids: node_info = heketi_node_info( self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue if len(nodes) > 2: out = heketi_node_disable( self.heketi_client_node, heketi_url, node_id) self.assertTrue(out) self.addCleanup( heketi_node_enable, self.heketi_client_node, heketi_url, node_id) for device in node_info['devices']: if device['state'].lower() != 'online': continue free_space = device['storage']['free'] if free_space < min_free_space: out = heketi_device_disable( self.heketi_client_node, heketi_url, device['id']) self.assertTrue(out) self.addCleanup( heketi_device_enable, self.heketi_client_node, heketi_url, device['id']) continue if node_id not in nodes: nodes[node_id] = [] nodes[node_id].append(device['storage']['free']) # Skip test if nodes requirements are not met if (len(nodes) < 3 or not all(map((lambda _list: len(_list) > 1), nodes.values()))): raise self.skipTest( "Could not find 3 online nodes with, " "at least, 2 online devices having free space " "bigger than %dGb." % min_free_space_gb) # Calculate size of a potential distributed vol vol_size_gb = int(min(map(max, nodes.values())) / (1024 ** 2)) + 1 return vol_size_gb
def get_online_nodes_disable_redundant(self): """ Find online nodes and disable n-3 nodes and return list of online nodes """ node_list = heketi_node_list(self.heketi_client_node, self.heketi_server_url) self.assertTrue(node_list, "Failed to list heketi nodes") g.log.info("Successfully got the list of nodes") # Fetch online nodes from node list online_hosts = [] for node in node_list: node_info = heketi_node_info( self.heketi_client_node, self.heketi_server_url, node, json=True) if node_info["state"] == "online": online_hosts.append(node_info) # Skip test if online node count is less than 3i if len(online_hosts) < 3: raise self.skipTest( "This test can run only if online hosts are more than 2") # if we have n nodes, disable n-3 nodes for node_info in online_hosts[3:]: node_id = node_info["id"] g.log.info("going to disable node id %s", node_id) heketi_node_disable(self.heketi_client_node, self.heketi_server_url, node_id) self.addCleanup(heketi_node_enable, self.heketi_client_node, self.heketi_server_url, node_id) for host in online_hosts[1:3]: found_online = False for device in host["devices"]: if device["state"].strip().lower() == "online": found_online = True break if not found_online: self.skipTest(("no device online on node %s" % host["id"])) return online_hosts
def test_to_get_list_of_nodes(self): """ Listing all nodes and compare the node listed in previous step """ # List all list ip = [] g.log.info("Listing the node id") heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) g.log.info("Successfully listed the node") if (len(heketi_node_id_list) == 0): raise ExecutionError("Node list empty") for node_id in heketi_node_id_list: g.log.info("Retrieve the node info") node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) self.assertTrue(node_info, ("Failed to " "retrieve the node info")) g.log.info("Successfully retrieved the node info %s" % node_id) ip.append(node_info["hostnames"]["storage"]) # Compare the node listed in previous step hostname = [] g.log.info("Get the pool list") list_of_pools = get_pool_list('auto_get_gluster_endpoint') self.assertTrue(list_of_pools, ("Failed to get the " "pool list from gluster pods/nodes")) g.log.info("Successfully got the pool list from gluster pods/nodes") for pool in list_of_pools: hostname.append(pool["hostname"]) if (len(heketi_node_id_list) != len(list_of_pools)): raise ExecutionError("Heketi volume list %s is not equal " "to gluster volume list %s" % ((ip), (hostname))) g.log.info("The node IP's from node info and list" " is : %s/n and pool list from gluster" " pods/nodes is %s" % ((ip), (hostname)))
def get_free_space_summary_devices(self): """ Calculates free space across all devices """ heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) total_free_space = 0 for node_id in heketi_node_id_list: node_info_dict = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) for device in node_info_dict["devices"]: total_free_space += (device["storage"]["free"] / (1024**2)) return total_free_space
def test_to_retrieve_node_info(self): """ List and retrieve node related info """ # List all list g.log.info("Listing the node id") heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) self.assertTrue(heketi_node_id_list, ("Node Id list is empty.")) g.log.info("Successfully listed the node") for node_id in heketi_node_id_list: g.log.info("Retrieve the node info") node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) self.assertTrue(node_info, ("Failed to " "retrieve the node info")) g.log.info("Successfully retrieved the node info %s" % node_id)
def test_heketi_with_device_removal_insuff_space(self): """Validate heketi with device removal insufficient space""" # Disable 4+ nodes and 3+ devices on the first 3 nodes min_free_space_gb = 5 min_free_space = min_free_space_gb * 1024**2 heketi_url = self.heketi_server_url heketi_node = self.heketi_client_node nodes = {} node_ids = heketi_node_list(heketi_node, heketi_url) self.assertTrue(node_ids) for node_id in node_ids: node_info = heketi_node_info( heketi_node, heketi_url, node_id, json=True) if (node_info["state"].lower() != "online" or not node_info["devices"]): continue if len(nodes) > 2: heketi_node_disable(heketi_node, heketi_url, node_id) self.addCleanup( heketi_node_enable, heketi_node, heketi_url, node_id) continue for device in node_info["devices"]: if device["state"].lower() != "online": continue free_space = device["storage"]["free"] if node_id not in nodes: nodes[node_id] = [] if (free_space < min_free_space or len(nodes[node_id]) > 1): heketi_device_disable( heketi_node, heketi_url, device["id"]) self.addCleanup( heketi_device_enable, heketi_node, heketi_url, device["id"]) continue nodes[node_id].append({ "device_id": device["id"], "free": free_space}) # Skip test if nodes requirements are not met if (len(nodes) < 3 or not all(map((lambda _list: len(_list) > 1), nodes.values()))): raise self.skipTest( "Could not find 3 online nodes with 2 online devices " "having free space bigger than %dGb." % min_free_space_gb) # Calculate size of a potential distributed vol if nodes[node_ids[0]][0]["free"] > nodes[node_ids[0]][1]["free"]: index = 0 else: index = 1 vol_size_gb = int(nodes[node_ids[0]][index]["free"] / (1024 ** 2)) + 1 device_id = nodes[node_ids[0]][index]["device_id"] # Create volume with such size that we consume space more than # size of smaller disks try: heketi_vol = heketi_volume_create( heketi_node, heketi_url, vol_size_gb, json=True) except Exception as e: g.log.warning( "Got following error trying to create '%s'Gb vol: %s" % ( vol_size_gb, e)) vol_size_gb -= 1 heketi_vol = heketi_volume_create( heketi_node, heketi_url, vol_size_gb, json=True) self.addCleanup(self.delete_volumes, heketi_vol["bricks"][0]["volume"]) # Try to 'remove' bigger Heketi disk expecting error, # because there is no space on smaller disk to relocate bricks to heketi_device_disable(heketi_node, heketi_url, device_id) self.addCleanup( heketi_device_enable, heketi_node, heketi_url, device_id) try: self.assertRaises( ExecutionError, heketi_device_remove, heketi_node, heketi_url, device_id) except Exception: self.addCleanup( heketi_device_disable, heketi_node, heketi_url, device_id) raise
def test_device_remove_operation(self, delete_device): """Validate remove/delete device using heketi-cli""" gluster_server_0 = g.config["gluster_servers"].values()[0] try: device_name = gluster_server_0["additional_devices"][0] except (KeyError, IndexError): self.skipTest( "Additional disk is not specified for node with following " "hostnames and IP addresses: %s, %s." % ( gluster_server_0.get('manage', '?'), gluster_server_0.get('storage', '?'))) manage_hostname = gluster_server_0["manage"] # Get node ID of the Gluster hostname topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue( topo_info["clusters"][0]["nodes"], "Cluster info command returned empty list of nodes.") node_id = None for node in topo_info["clusters"][0]["nodes"]: if manage_hostname == node['hostnames']["manage"][0]: node_id = node["id"] break self.assertNotEqual( node_id, None, "No information about node_id for %s" % manage_hostname) # Iterate chosen node devices and pick the smallest online one. lowest_device_size = lowest_device_id = None online_hosts = self.get_online_nodes_disable_redundant() for host in online_hosts[0:3]: if node_id != host["id"]: continue for device in host["devices"]: if device["state"].strip().lower() != "online": continue if (lowest_device_size is None or device["storage"]["total"] < lowest_device_size): lowest_device_size = device["storage"]["total"] lowest_device_id = device["id"] lowest_device_name = device["name"] if lowest_device_id is None: self.skipTest( "Didn't find suitable device for disablement on '%s' node." % ( node_id)) # Create volume vol_size = 1 vol_info = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, vol_size, json=True) self.assertTrue(vol_info, ( "Failed to create heketi volume of size %d" % vol_size)) self.addCleanup(self.delete_volumes, vol_info['id']) # Add extra device, then remember it's ID and size heketi_device_add(self.heketi_client_node, self.heketi_server_url, device_name, node_id) node_info_after_addition = heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) for device in node_info_after_addition["devices"]: if device["name"] != device_name: continue device_id_new = device["id"] device_size_new = device["storage"]["total"] self.addCleanup(heketi_device_delete, self.heketi_client_node, self.heketi_server_url, device_id_new) self.addCleanup(heketi_device_remove, self.heketi_client_node, self.heketi_server_url, device_id_new) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, device_id_new) if lowest_device_size > device_size_new: skip_msg = ("Skip test case, because newly added disk %s is " "smaller than device which we want to remove %s." % ( device_size_new, lowest_device_size)) self.skipTest(skip_msg) g.log.info("Removing device id %s" % lowest_device_id) ret, out, err = heketi_device_remove( self.heketi_client_node, self.heketi_server_url, lowest_device_id, raw_cli_output=True) if ret == 0: self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.assertNotEqual(ret, 0, ( "Device removal did not fail. ret: %s, out: %s, err: %s." % ( ret, out, err))) g.log.info("Device removal failed as expected, err- %s", err) # Need to disable device before removing heketi_device_disable( self.heketi_client_node, self.heketi_server_url, lowest_device_id) if not delete_device: self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) # Remove device from Heketi try: heketi_device_remove( self.heketi_client_node, self.heketi_server_url, lowest_device_id) except Exception: if delete_device: self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) raise if not delete_device: self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) if delete_device: try: heketi_device_delete( self.heketi_client_node, self.heketi_server_url, lowest_device_id) except Exception: self.addCleanup(heketi_device_enable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) self.addCleanup(heketi_device_disable, self.heketi_client_node, self.heketi_server_url, lowest_device_id) raise self.addCleanup( heketi_device_add, self.heketi_client_node, self.heketi_server_url, lowest_device_name, node_id) # Create volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, vol_size, json=True) self.assertTrue(vol_info, ( "Failed to create heketi volume of size %d" % vol_size)) self.addCleanup(self.delete_volumes, vol_info['id']) if delete_device: return # Check that none of volume's bricks is present on the device present = self.check_any_of_bricks_present_in_device( vol_info['bricks'], lowest_device_id) self.assertFalse( present, "Some of the '%s' volume bricks is present of the removed " "'%s' device." % (vol_info['id'], lowest_device_id))
def test_volume_expansion_no_free_space(self): """Validate volume expansion when there is no free space""" vol_size, expand_size, additional_devices_attached = None, 10, {} h_node, h_server_url = self.heketi_client_node, self.heketi_server_url # Get nodes info heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url) if len(heketi_node_id_list) < 3: self.skipTest("3 Heketi nodes are required.") # Disable 4th and other nodes for node_id in heketi_node_id_list[3:]: heketi_ops.heketi_node_disable(h_node, h_server_url, node_id) self.addCleanup(heketi_ops.heketi_node_enable, h_node, h_server_url, node_id) # Prepare first 3 nodes smallest_size = None err_msg = '' for node_id in heketi_node_id_list[0:3]: node_info = heketi_ops.heketi_node_info(h_node, h_server_url, node_id, json=True) # Disable second and other devices devices = node_info["devices"] self.assertTrue(devices, "Node '%s' does not have devices." % node_id) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if (smallest_size is None or devices[0]["storage"]["free"] < smallest_size): smallest_size = devices[0]["storage"]["free"] for device in node_info["devices"][1:]: heketi_ops.heketi_device_disable(h_node, h_server_url, device["id"]) self.addCleanup(heketi_ops.heketi_device_enable, h_node, h_server_url, device["id"]) # Gather info about additional devices additional_device_name = None for gluster_server in self.gluster_servers: gluster_server_data = self.gluster_servers_info[gluster_server] g_manage = gluster_server_data["manage"] g_storage = gluster_server_data["storage"] if not (g_manage in node_info["hostnames"]["manage"] or g_storage in node_info["hostnames"]["storage"]): continue additional_device_name = (( gluster_server_data.get("additional_devices") or [''])[0]) break if not additional_device_name: err_msg += ( "No 'additional_devices' are configured for " "'%s' node, which has following hostnames and " "IP addresses: %s.\n" % (node_id, ', '.join(node_info["hostnames"]["manage"] + node_info["hostnames"]["storage"]))) continue heketi_ops.heketi_device_add(h_node, h_server_url, additional_device_name, node_id) additional_devices_attached.update( {node_id: additional_device_name}) # Schedule cleanup of the added devices for node_id in additional_devices_attached.keys(): node_info = heketi_ops.heketi_node_info(h_node, h_server_url, node_id, json=True) for device in node_info["devices"]: if device["name"] != additional_devices_attached[node_id]: continue self.addCleanup(self.detach_devices_attached, device["id"]) break else: self.fail("Could not find ID for added device on " "'%s' node." % node_id) if err_msg: self.skipTest(err_msg) # Temporary disable new devices self.disable_devices(additional_devices_attached) # Create volume and save info about it vol_size = int(smallest_size / (1024**2)) - 1 creation_info = heketi_ops.heketi_volume_create(h_node, h_server_url, vol_size, json=True) volume_name, volume_id = creation_info["name"], creation_info["id"] self.addCleanup(heketi_ops.heketi_volume_delete, h_node, h_server_url, volume_id, raise_on_error=False) volume_info_before_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) free_space_before_expansion = self.get_devices_summary_free_space() # Try to expand volume with not enough device space self.assertRaises(ExecutionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume self.enable_devices(additional_devices_attached) # Expand volume and validate results heketi_ops.heketi_volume_expand(h_node, h_server_url, volume_id, expand_size, json=True) free_space_after_expansion = self.get_devices_summary_free_space() self.assertGreater( free_space_before_expansion, free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) volume_info_after_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) self.assertGreater(volume_info_after_expansion["size"], volume_info_before_expansion["size"], "Size of %s not increased" % volume_id) self.assertGreater(num_of_bricks_after_expansion, num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0) # Delete volume and validate release of the used space heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertGreater( free_space_after_deletion, free_space_after_expansion, "Free space not reclaimed after deletion of volume %s" % volume_id)
def _pv_resize(self, exceed_free_space): dir_path = "/mnt" pvc_size_gb, min_free_space_gb = 1, 3 # Get available free space disabling redundant devices and nodes heketi_url = self.heketi_server_url node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_id_list) nodes = {} min_free_space = min_free_space_gb * 1024**2 for node_id in node_id_list: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue if len(nodes) > 2: out = heketi_ops.heketi_node_disable(self.heketi_client_node, heketi_url, node_id) self.assertTrue(out) self.addCleanup(heketi_ops.heketi_node_enable, self.heketi_client_node, heketi_url, node_id) for device in node_info['devices']: if device['state'].lower() != 'online': continue free_space = device['storage']['free'] if (node_id in nodes.keys() or free_space < min_free_space): out = heketi_ops.heketi_device_disable( self.heketi_client_node, heketi_url, device['id']) self.assertTrue(out) self.addCleanup(heketi_ops.heketi_device_enable, self.heketi_client_node, heketi_url, device['id']) continue nodes[node_id] = free_space if len(nodes) < 3: raise self.skipTest("Could not find 3 online nodes with, " "at least, 1 online device having free space " "bigger than %dGb." % min_free_space_gb) # Calculate maximum available size for PVC available_size_gb = int(min(nodes.values()) / (1024**2)) # Create PVC self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb) # Create DC with POD and attached PVC to it dc_name = oc_create_app_dc_with_io(self.node, pvc_name) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) if exceed_free_space: # Try to expand existing PVC exceeding free space resize_pvc(self.node, pvc_name, available_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeFailed') # Check that app POD is up and runnig then try to write data wait_for_pod_be_ready(self.node, pod_name) cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to write data after failed attempt to expand PVC.") else: # Expand existing PVC using all the available free space expand_size_gb = available_size_gb - pvc_size_gb resize_pvc(self.node, pvc_name, expand_size_gb) verify_pvc_size(self.node, pvc_name, expand_size_gb) pv_name = get_pv_name_from_pvc(self.node, pvc_name) verify_pv_size(self.node, pv_name, expand_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeSuccessful') # Recreate app POD oc_delete(self.node, 'pod', pod_name) wait_for_resource_absence(self.node, 'pod', pod_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Write data on the expanded PVC cmd = ("dd if=/dev/urandom of=%s/autotest " "bs=1M count=1025" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual(ret, 0, "Failed to write data on the expanded PVC")
def test_volume_creation_no_free_devices(self): """Validate heketi error is returned when no free devices available""" node, server_url = self.heketi_client_node, self.heketi_server_url # Get nodes info node_id_list = heketi_ops.heketi_node_list(node, server_url) node_info_list = [] for node_id in node_id_list[0:3]: node_info = heketi_ops.heketi_node_info(node, server_url, node_id, json=True) node_info_list.append(node_info) # Disable 4th and other nodes for node_id in node_id_list[3:]: heketi_ops.heketi_node_disable(node, server_url, node_id) self.addCleanup(heketi_ops.heketi_node_enable, node, server_url, node_id) # Disable second and other devices on the first 3 nodes for node_info in node_info_list[0:3]: devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_info["id"]) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if len(devices) < 2: continue for device in node_info["devices"][1:]: out = heketi_ops.heketi_device_disable(node, server_url, device["id"]) self.assertTrue( out, "Failed to disable the device %s" % device["id"]) self.addCleanup(heketi_ops.heketi_device_enable, node, server_url, device["id"]) # Calculate common available space available_spaces = [ int(node_info["devices"][0]["storage"]["free"]) for n in node_info_list[0:3] ] min_space_gb = int(min(available_spaces) / 1024**2) self.assertGreater(min_space_gb, 3, "Not enough available free space.") # Create first small volume vol = heketi_ops.heketi_volume_create(node, server_url, 1, json=True) self.addCleanup(heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol["id"]) # Try to create second volume getting "no free space" error try: vol_fail = heketi_ops.heketi_volume_create(node, server_url, min_space_gb, json=True) except exceptions.ExecutionError: g.log.info("Volume was not created as expected.") else: self.addCleanup(heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_fail["bricks"][0]["volume"]) self.assertFalse( vol_fail, "Volume should have not been created. Out: %s" % vol_fail)
def test_create_volumes_enabling_and_disabling_heketi_devices(self): """Validate enable/disable of heketi device""" # Get nodes info node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node, self.heketi_server_url) node_info_list = [] for node_id in node_id_list[0:3]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) node_info_list.append(node_info) # Disable 4th and other nodes if len(node_id_list) > 3: for node in node_id_list[3:]: heketi_ops.heketi_node_disable(self.heketi_client_node, self.heketi_server_url, node_id) self.addCleanup(heketi_ops.heketi_node_enable, self.heketi_client_node, self.heketi_server_url, node_id) # Disable second and other devices on the first 3 nodes for node_info in node_info_list[0:3]: devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_info["id"]) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if len(devices) < 2: continue for device in node_info["devices"][1:]: out = heketi_ops.heketi_device_disable(self.heketi_client_node, self.heketi_server_url, device["id"]) self.assertTrue( out, "Failed to disable the device %s" % device["id"]) self.addCleanup(heketi_ops.heketi_device_enable, self.heketi_client_node, self.heketi_server_url, device["id"]) # Create heketi volume out = heketi_ops.heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(out, "Failed to create heketi volume of size 1") g.log.info("Successfully created heketi volume of size 1") device_id = out["bricks"][0]["device"] self.addCleanup(heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, out["bricks"][0]["volume"]) # Disable device g.log.info("Disabling '%s' device" % device_id) out = heketi_ops.heketi_device_disable(self.heketi_client_node, self.heketi_server_url, device_id) self.assertTrue(out, "Failed to disable the device %s" % device_id) g.log.info("Successfully disabled device %s" % device_id) try: # Get device info g.log.info("Retrieving '%s' device info" % device_id) out = heketi_ops.heketi_device_info(self.heketi_client_node, self.heketi_server_url, device_id, json=True) self.assertTrue(out, "Failed to get device info %s" % device_id) g.log.info("Successfully retrieved device info %s" % device_id) name = out["name"] if out["state"].lower().strip() != "offline": raise exceptions.ExecutionError( "Device %s is not in offline state." % name) g.log.info("Device %s is now offine" % name) # Try to create heketi volume g.log.info("Creating heketi volume: Expected to fail.") try: out = heketi_ops.heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) except exceptions.ExecutionError: g.log.info("Volume was not created as expected.") else: self.addCleanup(heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, out["bricks"][0]["volume"]) msg = "Volume unexpectedly created. Out: %s" % out assert False, msg finally: # Enable the device back g.log.info("Enable '%s' device back." % device_id) out = heketi_ops.heketi_device_enable(self.heketi_client_node, self.heketi_server_url, device_id) self.assertTrue(out, "Failed to enable the device %s" % device_id) g.log.info("Successfully enabled device %s" % device_id) # Get device info out = heketi_ops.heketi_device_info(self.heketi_client_node, self.heketi_server_url, device_id, json=True) self.assertTrue(out, ("Failed to get device info %s" % device_id)) g.log.info("Successfully retrieved device info %s" % device_id) name = out["name"] if out["state"] != "online": raise exceptions.ExecutionError( "Device %s is not in online state." % name) # Create heketi volume of size out = heketi_ops.heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(out, "Failed to create volume of size 1") self.addCleanup(heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, out["bricks"][0]["volume"]) g.log.info("Successfully created volume of size 1") name = out["name"] # Get gluster volume info vol_info = get_volume_info('auto_get_gluster_endpoint', volname=name) self.assertTrue(vol_info, "Failed to get '%s' volume info." % name) g.log.info("Successfully got the '%s' volume info." % name)
def test_create_arbiter_vol_with_more_than_one_brick_set(self): """Validate volume creation using heketi for more than six brick set""" # Set arbiter:disabled tag to the data devices and get their info data_nodes = [] for node_id in self.node_id_list[0:2]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) if len(node_info['devices']) < 2: self.skipTest("Nodes are expected to have at least 2 devices") if not all([ int(d['storage']['free']) > (3 * 1024**2) for d in node_info['devices'][0:2] ]): self.skipTest( "Devices are expected to have more than 3Gb of free space") for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'disabled', device.get('tags', {}).get('arbiter')) self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'disabled', node_info.get('tags', {}).get('arbiter')) data_nodes.append(node_info) # Set arbiter:required tag to all other nodes and their devices for node_id in self.node_id_list[2:]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'required', node_info.get('tags', {}).get('arbiter')) for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'required', device.get('tags', {}).get('arbiter')) # Get second big volume between 2 data nodes and use it # for target vol calculation. for i, node_info in enumerate(data_nodes): biggest_disk_free_space = 0 for device in node_info['devices'][0:2]: free = int(device['storage']['free']) if free > biggest_disk_free_space: biggest_disk_free_space = free data_nodes[i]['biggest_free_space'] = biggest_disk_free_space target_vol_size_kb = 1 + min( [n['biggest_free_space'] for n in data_nodes]) # Check that all the data devices have, at least, half of required size all_big_enough = True for node_info in data_nodes: for device in node_info['devices'][0:2]: if float(device['storage']['free']) < (target_vol_size_kb / 2): all_big_enough = False break # Create sc with gluster arbiter info self.create_storage_class(is_arbiter_vol=True) # Create helper arbiter vol if not all the data devices have # half of required free space. if not all_big_enough: helper_vol_size_kb, target_vol_size_kb = 0, 0 smaller_device_id = None for node_info in data_nodes: devices = node_info['devices'] if ((devices[0]['storage']['free']) > (devices[1]['storage']['free'])): smaller_device_id = devices[1]['id'] smaller_device = devices[1]['storage']['free'] bigger_device = devices[0]['storage']['free'] else: smaller_device_id = devices[0]['id'] smaller_device = devices[0]['storage']['free'] bigger_device = devices[1]['storage']['free'] diff = bigger_device - (2 * smaller_device) + 1 if diff > helper_vol_size_kb: helper_vol_size_kb = diff target_vol_size_kb = bigger_device - diff # Disable smaller device and create helper vol on bigger one # to reduce its size, then enable smaller device back. try: out = heketi_ops.heketi_device_disable(self.heketi_client_node, self.heketi_server_url, smaller_device_id) self.assertTrue(out) self.create_and_wait_for_pvc( int(helper_vol_size_kb / 1024.0**2) + 1) finally: out = heketi_ops.heketi_device_enable(self.heketi_client_node, self.heketi_server_url, smaller_device_id) self.assertTrue(out) # Create target arbiter volume self.create_and_wait_for_pvc(int(target_vol_size_kb / 1024.0**2)) # Get gluster volume info vol_info = get_gluster_vol_info_by_pvc_name(self.node, self.pvc_name) self.verify_amount_and_proportion_of_arbiter_and_data_bricks( vol_info, arbiter_bricks=2, data_bricks=4)
def test_expand_arbiter_volume_setting_tags_on_nodes_or_devices( self, node_tags): """Validate exapnsion of arbiter volume with defferent tags This test case is going to run two tests: 1. If value is True it is going to set tags on nodes and run test 2. If value is False it is going to set tags on devices and run test """ data_nodes = [] arbiter_nodes = [] # set tags arbiter:disabled, arbiter:required for i, node_id in enumerate(self.node_id_list): if node_tags: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'disabled' if i < 2 else 'required') node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) if not node_tags: for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'disabled' if i < 2 else 'required') device_info = heketi_ops.heketi_device_info( self.heketi_client_node, self.heketi_server_url, device['id'], json=True) self.assertEqual(device_info['tags']['arbiter'], 'disabled' if i < 2 else 'required') node = { 'id': node_id, 'host': node_info['hostnames']['storage'][0] } if node_tags: self.assertEqual(node_info['tags']['arbiter'], 'disabled' if i < 2 else 'required') data_nodes.append(node) if i < 2 else arbiter_nodes.append(node) # Create sc with gluster arbiter info self.create_storage_class(is_arbiter_vol=True, allow_volume_expansion=True) # Create PVC and wait for it to be in 'Bound' state self.create_and_wait_for_pvc() vol_info = get_gluster_vol_info_by_pvc_name(self.node, self.pvc_name) bricks = self.verify_amount_and_proportion_of_arbiter_and_data_bricks( vol_info) arbiter_hosts = [obj['host'] for obj in arbiter_nodes] data_hosts = [obj['host'] for obj in data_nodes] for brick in bricks['arbiter_list']: self.assertIn(brick['name'].split(':')[0], arbiter_hosts) for brick in bricks['data_list']: self.assertIn(brick['name'].split(':')[0], data_hosts) # Expand PVC and verify the size pvc_size = 2 resize_pvc(self.node, self.pvc_name, pvc_size) verify_pvc_size(self.node, self.pvc_name, pvc_size) vol_info = get_gluster_vol_info_by_pvc_name(self.node, self.pvc_name) bricks = self.verify_amount_and_proportion_of_arbiter_and_data_bricks( vol_info, arbiter_bricks=2, data_bricks=4) for brick in bricks['arbiter_list']: self.assertIn(brick['name'].split(':')[0], arbiter_hosts) for brick in bricks['data_list']: self.assertIn(brick['name'].split(':')[0], data_hosts)
def test_create_delete_pvcs_to_make_gluster_reuse_released_space(self): """Validate reuse of volume space after deletion of PVCs""" min_storage_gb = 10 # Set arbiter:disabled tags to the first 2 nodes data_nodes = [] biggest_disks = [] self.assertGreater(len(self.node_id_list), 2) for node_id in self.node_id_list[0:2]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) biggest_disk_free_space = 0 for device in node_info['devices']: disk_free_space = int(device['storage']['free']) if disk_free_space < (min_storage_gb * 1024**2): self.skipTest("Devices are expected to have more than " "%sGb of free space" % min_storage_gb) if disk_free_space > biggest_disk_free_space: biggest_disk_free_space = disk_free_space self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'disabled', revert_to=device.get('tags', {}).get('arbiter')) biggest_disks.append(biggest_disk_free_space) self._set_arbiter_tag_with_further_revert(self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'disabled', revert_to=node_info.get( 'tags', {}).get('arbiter')) data_nodes.append(node_info) # Set arbiter:required tag to all other nodes and their devices arbiter_nodes = [] for node_id in self.node_id_list[2:]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'required', revert_to=device.get('tags', {}).get('arbiter')) self._set_arbiter_tag_with_further_revert(self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'required', revert_to=node_info.get( 'tags', {}).get('arbiter')) arbiter_nodes.append(node_info) # Calculate size and amount of volumes to be created pvc_size = int(min(biggest_disks) / 1024**2) pvc_amount = max([len(n['devices']) for n in data_nodes]) + 1 # Create sc with gluster arbiter info self.create_storage_class(is_arbiter_vol=True) # Create and delete 3 small volumes concurrently pvc_names = [] for i in range(3): pvc_name = oc_create_pvc(self.node, self.sc_name, pvc_name_prefix='arbiter-pvc', pvc_size=int(pvc_size / 3)) pvc_names.append(pvc_name) exception_exists = False for pvc_name in pvc_names: try: verify_pvc_status_is_bound(self.node, pvc_name) except Exception: for pvc_name in pvc_names: self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name) for pvc_name in pvc_names: self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) exception_exists = True if exception_exists: raise for pvc_name in pvc_names: oc_delete(self.node, 'pvc', pvc_name) for pvc_name in pvc_names: wait_for_resource_absence(self.node, 'pvc', pvc_name) # Create and delete big volumes in a loop for i in range(pvc_amount): pvc_name = oc_create_pvc(self.node, self.sc_name, pvc_name_prefix='arbiter-pvc', pvc_size=pvc_size) try: verify_pvc_status_is_bound(self.node, pvc_name) except Exception: self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) raise oc_delete(self.node, 'pvc', pvc_name) wait_for_resource_absence(self.node, 'pvc', pvc_name)
def test_aribiter_required_tag_on_node_or_devices_other_disabled( self, node_with_tag): """Validate arbiter vol creation with required node or device tag""" pvc_amount = 3 # Get Heketi nodes info node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node, self.heketi_server_url) # Set arbiter:required tags arbiter_node = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id_list[0], json=True) arbiter_nodes_ip_addresses = arbiter_node['hostnames']['storage'] self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id_list[0], ('required' if node_with_tag else None), revert_to=arbiter_node.get('tags', {}).get('arbiter')) for device in arbiter_node['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], (None if node_with_tag else 'required'), revert_to=device.get('tags', {}).get('arbiter')) # Set arbiter:disabled tags data_nodes, data_nodes_ip_addresses = [], [] for node_id in node_id_list[1:]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) if not any([ int(d['storage']['free']) > (pvc_amount * 1024**2) for d in node_info['devices'] ]): self.skipTest("Devices are expected to have more than " "%sGb of free space" % pvc_amount) data_nodes_ip_addresses.extend(node_info['hostnames']['storage']) for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], (None if node_with_tag else 'disabled'), revert_to=device.get('tags', {}).get('arbiter')) self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id, ('disabled' if node_with_tag else None), revert_to=node_info.get('tags', {}).get('arbiter')) data_nodes.append(node_info) # Create PVCs and check that their bricks are correctly located self.create_storage_class(is_arbiter_vol=True) for i in range(pvc_amount): self.create_and_wait_for_pvc(1) # Get gluster volume info vol_info = get_gluster_vol_info_by_pvc_name( self.node, self.pvc_name) arbiter_bricks, data_bricks = [], [] for brick in vol_info['bricks']['brick']: if int(brick["isArbiter"]) == 1: arbiter_bricks.append(brick["name"]) else: data_bricks.append(brick["name"]) # Verify that all the arbiter bricks are located on # arbiter:required node and data bricks on all other nodes only. for arbiter_brick in arbiter_bricks: self.assertIn( arbiter_brick.split(':')[0], arbiter_nodes_ip_addresses) for data_brick in data_bricks: self.assertIn( data_brick.split(':')[0], data_nodes_ip_addresses)