def test_to_check_deletion_of_node(self): """Validate deletion of a node which contains devices""" # Create Heketi volume to make sure we have devices with usages heketi_url = self.heketi_server_url vol = heketi_volume_create(self.heketi_client_node, heketi_url, 1, json=True) self.assertTrue(vol, "Failed to create heketi volume.") g.log.info("Heketi volume successfully created") volume_id = vol["bricks"][0]["volume"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) # Pick up suitable node node_ids = heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_ids) for node_id in node_ids: node_info = heketi_node_info(self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue for device in node_info['devices']: if device['state'].lower() != 'online': continue if device['storage']['used']: node_id = node_info['id'] break else: self.assertTrue( node_id, "Failed to find online node with online device which " "has some usages.") # Try to delete the node by its ID g.log.info("Trying to delete the node which contains devices in it. " "Expecting failure.") self.assertRaises(AssertionError, heketi_node_delete, self.heketi_client_node, heketi_url, node_id) # Make sure our node hasn't been deleted g.log.info("Listing heketi node list") node_list = heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_list, ("Failed to list heketi nodes")) self.assertIn(node_id, node_list) node_info = heketi_node_info(self.heketi_client_node, heketi_url, node_id, json=True) self.assertEqual(node_info['state'].lower(), 'online')
def test_to_check_deletion_of_node(self): """Validate deletion of a node which contains devices""" # Create Heketi volume to make sure we have devices with usages heketi_url = self.heketi_server_url vol = heketi_volume_create( self.heketi_client_node, heketi_url, 1, json=True) self.assertTrue(vol, "Failed to create heketi volume.") g.log.info("Heketi volume successfully created") volume_id = vol["bricks"][0]["volume"] self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) # Pick up suitable node node_ids = heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_ids) for node_id in node_ids: node_info = heketi_node_info( self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue for device in node_info['devices']: if device['state'].lower() != 'online': continue if device['storage']['used']: node_id = node_info['id'] break else: self.assertTrue( node_id, "Failed to find online node with online device which " "has some usages.") # Try to delete the node by its ID g.log.info("Trying to delete the node which contains devices in it. " "Expecting failure.") self.assertRaises( ExecutionError, heketi_node_delete, self.heketi_client_node, heketi_url, node_id) # Make sure our node hasn't been deleted g.log.info("Listing heketi node list") node_list = heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_list, ("Failed to list heketi nodes")) self.assertIn(node_id, node_list) node_info = heketi_node_info( self.heketi_client_node, heketi_url, node_id, json=True) self.assertEqual(node_info['state'].lower(), 'online')
def get_devices_summary_free_space(self): """ Calculates minimum free space per device and returns total free space across all devices """ free_spaces = [] heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) for node_id in heketi_node_id_list: node_info_dict = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) total_free_space = 0 for device in node_info_dict["devices"]: total_free_space += device["storage"]["free"] free_spaces.append(total_free_space) total_free_space = sum(free_spaces) / (1024**2) total_free_space = int(math.floor(total_free_space)) return total_free_space
def test_heketi_node_list(self): """Test node list operation """ h_client, h_server = self.heketi_client_node, self.heketi_server_url # List heketi nodes node_ips = [] heketi_node_id_list = heketi_ops.heketi_node_list(h_client, h_server) for node_id in heketi_node_id_list: node_info = heketi_ops.heketi_node_info( h_client, h_server, node_id, json=True) node_ips.append(node_info["hostnames"]["storage"]) # Compare the node listed in previous step hostnames = [] list_of_pools = peer_ops.get_pool_list('auto_get_gluster_endpoint') self.assertTrue( list_of_pools, "Failed to get the pool list from gluster pods/nodes") for pool in list_of_pools: hostnames.append(pool["hostname"]) self.assertEqual( len(heketi_node_id_list), len(list_of_pools), "Heketi volume list %s is not equal to gluster volume list %s" % (node_ips, hostnames))
def test_brick_evict_on_three_node_with_one_down(self): """Test brick evict basic functionality and verify brick evict will fail after node down if nodes are three""" h_node, h_server = self.heketi_client_node, self.heketi_server_url # Disable node if more than 3 node_list = heketi_ops.heketi_node_list(h_node, h_server) if len(node_list) > 3: for node_id in node_list[3:]: heketi_ops.heketi_node_disable(h_node, h_server, node_id) self.addCleanup(heketi_ops.heketi_node_enable, h_node, h_server, node_id) # Create heketi volume vol_info = heketi_ops.heketi_volume_create(h_node, h_server, 1, json=True) self.addCleanup(heketi_ops.heketi_volume_delete, h_node, h_server, vol_info.get('id')) # Get node on which heketi pod is scheduled heketi_pod = openshift_ops.get_pod_name_from_dc( self.ocp_client, self.heketi_dc_name) heketi_node = openshift_ops.oc_get_custom_resource( self.ocp_client, 'pod', '.:spec.nodeName', heketi_pod)[0] # Get list of hostname from node id host_list = [] for node_id in node_list[3:]: node_info = heketi_ops.heketi_node_info(h_node, h_server, node_id, json=True) host_list.append(node_info.get('hostnames').get('manage')[0]) # Get brick id and glusterfs node which is not heketi node for node in vol_info.get('bricks', {}): node_info = heketi_ops.heketi_node_info(h_node, h_server, node.get('node'), json=True) hostname = node_info.get('hostnames').get('manage')[0] if (hostname != heketi_node) and (hostname not in host_list): brick_id = node.get('id') break self._power_off_node_and_wait_node_to_be_not_ready(hostname) # Perform brick evict operation try: heketi_ops.heketi_brick_evict(h_node, h_server, brick_id) except AssertionError as e: if ('No Replacement was found' not in six.text_type(e)): raise
def get_ready_for_node_add(self, hostname): self.configure_node_to_run_gluster(hostname) h_nodes = heketi_ops.heketi_node_list(self.h_node, self.h_url) # Disable nodes except first two nodes for node_id in h_nodes[2:]: heketi_ops.heketi_node_disable(self.h_node, self.h_url, node_id) self.addCleanup(heketi_ops.heketi_node_enable, self.h_node, self.h_url, node_id)
def test_heketi_node_states_enable_disable(self): """Test node enable and disable functionality """ h_client, h_server = self.heketi_client_node, self.heketi_server_url node_list = heketi_ops.heketi_node_list(h_client, h_server) online_hosts = [] for node_id in node_list: node_info = heketi_ops.heketi_node_info(h_client, h_server, node_id, json=True) if node_info["state"] == "online": online_hosts.append(node_info) if len(online_hosts) < 3: raise self.skipTest( "This test can run only if online hosts are more than 2") # Disable n-3 nodes, in case we have n nodes for node_info in online_hosts[3:]: node_id = node_info["id"] heketi_ops.heketi_node_disable(h_client, h_server, node_id) self.addCleanup(heketi_ops.heketi_node_enable, h_client, h_server, node_id) # Create volume when 3 nodes are online vol_size = 1 vol_info = heketi_ops.heketi_volume_create(h_client, h_server, vol_size, json=True) self.addCleanup(heketi_ops.heketi_volume_delete, h_client, h_server, vol_info['id']) node_id = online_hosts[0]['id'] try: heketi_ops.heketi_node_disable(h_client, h_server, node_id) # Try to create a volume, volume creation should fail with self.assertRaises(AssertionError): heketi_volume = heketi_ops.heketi_volume_create( h_client, h_server, vol_size) self.addCleanup(heketi_ops.heketi_volume_delete, h_client, h_server, heketi_volume["id"]) finally: # Enable heketi node heketi_ops.heketi_node_enable(h_client, h_server, node_id) # Create volume when heketi node is enabled vol_info = heketi_ops.heketi_volume_create(h_client, h_server, vol_size, json=True) heketi_ops.heketi_volume_delete(h_client, h_server, vol_info['id'])
def _get_online_nodes(self): node_ids = heketi_ops.heketi_node_list(self.h_client, self.h_server) online_nodes = [] for node_id in node_ids: node_info = heketi_ops.heketi_node_info(self.h_client, self.h_server, node_id, json=True) if (node_info["state"] == "online" and node_info['cluster'] == self.cluster_id): online_nodes.append( (node_info["zone"], node_info['hostnames']['storage'])) return online_nodes
def test_brick_multiplex_pids_with_diff_vol_option_values(self): """Test Brick Pid's should be same when values of vol options are diff """ h_client, h_url = self.heketi_client_node, self.heketi_server_url # Disable heketi nodes except first three nodes h_nodes_list = heketi_node_list(h_client, h_url) for node_id in h_nodes_list[3:]: heketi_node_disable(h_client, h_url, node_id) self.addCleanup(heketi_node_enable, h_client, h_url, node_id) # Create storage class with diff volumeoptions sc1 = self.create_storage_class(volumeoptions='user.heketi.abc 1') sc2 = self.create_storage_class(volumeoptions='user.heketi.abc 2') # Create PVC's with above SC pvc1 = self.create_and_wait_for_pvcs(sc_name=sc1) pvc2 = self.create_and_wait_for_pvcs(sc_name=sc2) # Get vol info and status vol_info1 = get_gluster_vol_info_by_pvc_name(self.node, pvc1[0]) vol_info2 = get_gluster_vol_info_by_pvc_name(self.node, pvc2[0]) vol_status1 = get_gluster_vol_status(vol_info1['gluster_vol_id']) vol_status2 = get_gluster_vol_status(vol_info2['gluster_vol_id']) # Verify vol options err_msg = ('Volume option "user.heketi.abc %s" did not got match for ' 'volume %s in gluster vol info') self.assertEqual( vol_info1['options']['user.heketi.abc'], '1', err_msg % (1, vol_info1['gluster_vol_id'])) self.assertEqual( vol_info2['options']['user.heketi.abc'], '2', err_msg % (2, vol_info2['gluster_vol_id'])) # Get the PID's and match them pids1 = set() for brick in vol_info1['bricks']['brick']: host, bname = brick['name'].split(":") pids1.add(vol_status1[host][bname]['pid']) pids2 = set() for brick in vol_info2['bricks']['brick']: host, bname = brick['name'].split(":") pids2.add(vol_status2[host][bname]['pid']) err_msg = ('Pids of both the volumes %s and %s are expected to be' 'same. But got the different Pids "%s" and "%s".' % (vol_info1['gluster_vol_id'], vol_info2['gluster_vol_id'], pids1, pids2)) self.assertEqual(pids1, pids2, err_msg)
def _get_vol_size(self): # Get available free space disabling redundant nodes min_free_space_gb = 5 heketi_url = self.heketi_server_url node_ids = heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_ids) nodes = {} min_free_space = min_free_space_gb * 1024**2 for node_id in node_ids: node_info = heketi_node_info(self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue if len(nodes) > 2: out = heketi_node_disable(self.heketi_client_node, heketi_url, node_id) self.assertTrue(out) self.addCleanup(heketi_node_enable, self.heketi_client_node, heketi_url, node_id) for device in node_info['devices']: if device['state'].lower() != 'online': continue free_space = device['storage']['free'] if free_space < min_free_space: out = heketi_device_disable(self.heketi_client_node, heketi_url, device['id']) self.assertTrue(out) self.addCleanup(heketi_device_enable, self.heketi_client_node, heketi_url, device['id']) continue if node_id not in nodes: nodes[node_id] = [] nodes[node_id].append(device['storage']['free']) # Skip test if nodes requirements are not met if (len(nodes) < 3 or not all(map( (lambda _list: len(_list) > 1), nodes.values()))): raise self.skipTest("Could not find 3 online nodes with, " "at least, 2 online devices having free space " "bigger than %dGb." % min_free_space_gb) # Calculate size of a potential distributed vol vol_size_gb = int(min(map(max, nodes.values())) / (1024**2)) + 1 return vol_size_gb
def _get_free_space(self): """Get free space in each heketi device""" free_spaces = [] heketi_node_id_list = heketi_node_list( self.heketi_client_node, self.heketi_server_url) for node_id in heketi_node_id_list: node_info_dict = heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) total_free_space = 0 for device in node_info_dict["devices"]: total_free_space += device["storage"]["free"] free_spaces.append(total_free_space) total_free_space = int(math.floor(sum(free_spaces) / (1024**2))) return total_free_space
def test_compare_node_count_with_db_check_info(self): """Validate nodes count using heketi db check""" # Check heketi db db_result = heketi_ops.heketi_db_check(self.heketi_client_node, self.heketi_server_url) db_nodes_count = db_result["nodes"]["total"] nodes_list = heketi_ops.heketi_node_list(self.heketi_client_node, self.heketi_server_url, json=True) calculated_nodes_count = len(nodes_list) self.assertEqual( calculated_nodes_count, db_nodes_count, "Nodes count from 'DB check' (%s) doesn't match calculated nodes " "count (%s)." % (db_nodes_count, calculated_nodes_count))
def test_heketi_node_info(self): """Test heketi node info operation """ h_client, h_server = self.heketi_client_node, self.heketi_server_url # List heketi node heketi_node_id_list = heketi_ops.heketi_node_list(h_client, h_server) self.assertTrue(heketi_node_id_list, "Node Id list is empty.") for node_id in heketi_node_id_list: node_info = heketi_ops.heketi_node_info( h_client, h_server, node_id, json=True) self.assertTrue(node_info, "Failed to retrieve the node info") self.assertEqual( node_info["id"], node_id, "Failed to match node ID. Exp: %s, Act: %s" % ( node_id, node_info["id"]))
def get_online_nodes_disable_redundant(self): """ Find online nodes and disable n-3 nodes and return list of online nodes """ node_list = heketi_node_list(self.heketi_client_node, self.heketi_server_url) self.assertTrue(node_list, "Failed to list heketi nodes") g.log.info("Successfully got the list of nodes") # Fetch online nodes from node list online_hosts = [] for node in node_list: node_info = heketi_node_info( self.heketi_client_node, self.heketi_server_url, node, json=True) if node_info["state"] == "online": online_hosts.append(node_info) # Skip test if online node count is less than 3i if len(online_hosts) < 3: raise self.skipTest( "This test can run only if online hosts are more than 2") # if we have n nodes, disable n-3 nodes for node_info in online_hosts[3:]: node_id = node_info["id"] g.log.info("going to disable node id %s", node_id) heketi_node_disable(self.heketi_client_node, self.heketi_server_url, node_id) self.addCleanup(heketi_node_enable, self.heketi_client_node, self.heketi_server_url, node_id) for host in online_hosts[1:3]: found_online = False for device in host["devices"]: if device["state"].strip().lower() == "online": found_online = True break if not found_online: self.skipTest(("no device online on node %s" % host["id"])) return online_hosts
def _available_disk_free_space(self): min_free_space_gb = 3 # Get available free space disabling redundant devices and nodes heketi_url = self.heketi_server_url node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_id_list) nodes = {} min_free_space = min_free_space_gb * 1024**2 for node_id in node_id_list: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue if len(nodes) > 2: self.addCleanup(heketi_ops.heketi_node_enable, self.heketi_client_node, heketi_url, node_id) out = heketi_ops.heketi_node_disable(self.heketi_client_node, heketi_url, node_id) self.assertTrue(out) for device in node_info['devices']: if device['state'].lower() != 'online': continue free_space = device['storage']['free'] if (node_id in nodes.keys() or free_space < min_free_space): out = heketi_ops.heketi_device_disable( self.heketi_client_node, heketi_url, device['id']) self.assertTrue(out) self.addCleanup(heketi_ops.heketi_device_enable, self.heketi_client_node, heketi_url, device['id']) continue nodes[node_id] = free_space if len(nodes) < 3: raise self.skipTest("Could not find 3 online nodes with, " "at least, 1 online device having free space " "bigger than %dGb." % min_free_space_gb) # Calculate maximum available size for PVC available_size_gb = int(min(nodes.values()) / (1024**2)) return available_size_gb
def test_to_get_list_of_nodes(self): """ Listing all nodes and compare the node listed in previous step """ # List all list ip = [] g.log.info("Listing the node id") heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) g.log.info("Successfully listed the node") if (len(heketi_node_id_list) == 0): raise ExecutionError("Node list empty") for node_id in heketi_node_id_list: g.log.info("Retrieve the node info") node_info = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) self.assertTrue(node_info, ("Failed to " "retrieve the node info")) g.log.info("Successfully retrieved the node info %s" % node_id) ip.append(node_info["hostnames"]["storage"]) # Compare the node listed in previous step hostname = [] g.log.info("Get the pool list") list_of_pools = get_pool_list('auto_get_gluster_endpoint') self.assertTrue(list_of_pools, ("Failed to get the " "pool list from gluster pods/nodes")) g.log.info("Successfully got the pool list from gluster pods/nodes") for pool in list_of_pools: hostname.append(pool["hostname"]) if (len(heketi_node_id_list) != len(list_of_pools)): raise ExecutionError( "Heketi volume list %s is not equal " "to gluster volume list %s" % ((ip), (hostname))) g.log.info("The node IP's from node info and list" " is : %s/n and pool list from gluster" " pods/nodes is %s" % ((ip), (hostname)))
def test_to_get_list_of_nodes(self): """ Listing all nodes and compare the node listed in previous step """ # List all list ip = [] g.log.info("Listing the node id") heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) g.log.info("Successfully listed the node") if (len(heketi_node_id_list) == 0): raise ExecutionError("Node list empty") for node_id in heketi_node_id_list: g.log.info("Retrieve the node info") node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) self.assertTrue(node_info, ("Failed to " "retrieve the node info")) g.log.info("Successfully retrieved the node info %s" % node_id) ip.append(node_info["hostnames"]["storage"]) # Compare the node listed in previous step hostname = [] g.log.info("Get the pool list") list_of_pools = get_pool_list('auto_get_gluster_endpoint') self.assertTrue(list_of_pools, ("Failed to get the " "pool list from gluster pods/nodes")) g.log.info("Successfully got the pool list from gluster pods/nodes") for pool in list_of_pools: hostname.append(pool["hostname"]) if (len(heketi_node_id_list) != len(list_of_pools)): raise ExecutionError("Heketi volume list %s is not equal " "to gluster volume list %s" % ((ip), (hostname))) g.log.info("The node IP's from node info and list" " is : %s/n and pool list from gluster" " pods/nodes is %s" % ((ip), (hostname)))
def get_free_space_summary_devices(self): """ Calculates free space across all devices """ heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) total_free_space = 0 for node_id in heketi_node_id_list: node_info_dict = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) for device in node_info_dict["devices"]: total_free_space += (device["storage"] ["free"] / (1024 ** 2)) return total_free_space
def _get_bricks_counts_and_device_name(self): """Fetch bricks count and device name from all the nodes""" h_client, h_url = self.heketi_client_node, self.heketi_server_url # Fetch bricks on the devices h_nodes = heketi_ops.heketi_node_list(h_client, h_url) node_details = {} for h_node in h_nodes: h_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) node_details[h_node] = [[], []] for device in h_node_info['devices']: node_details[h_node][0].append(len(device['bricks'])) node_details[h_node][1].append(device['id']) return node_details
def setUp(self): super(TestArbiterVolumeCreateExpandDelete, self).setUp() self.node = self.ocp_master_node[0] if openshift_version.get_openshift_version() < "3.9": self.skipTest("Arbiter feature cannot be used on OCP older " "than 3.9, because 'volumeoptions' for Heketi " "is not supported there.") version = heketi_version.get_heketi_version(self.heketi_client_node) if version < '6.0.0-11': self.skipTest("heketi-client package %s does not support arbiter " "functionality" % version.v_str) # Mark one of the Heketi nodes as arbiter-supported if none of # existent nodes or devices already enabled to support it. self.heketi_server_url = self.sc.get('resturl') arbiter_tags = ('required', 'supported') arbiter_already_supported = False self.node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) for node_id in self.node_id_list[::-1]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) if node_info.get('tags', {}).get('arbiter') in arbiter_tags: arbiter_already_supported = True break for device in node_info['devices'][::-1]: if device.get('tags', {}).get('arbiter') in arbiter_tags: arbiter_already_supported = True break else: continue break if not arbiter_already_supported: self._set_arbiter_tag_with_further_revert(self.heketi_client_node, self.heketi_server_url, 'node', self.node_id_list[0], 'supported')
def test_to_retrieve_node_info(self): """ List and retrieve node related info """ # List all list g.log.info("Listing the node id") heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) self.assertTrue(heketi_node_id_list, ("Node Id list is empty.")) g.log.info("Successfully listed the node") for node_id in heketi_node_id_list: g.log.info("Retrieve the node info") node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) self.assertTrue(node_info, ("Failed to " "retrieve the node info")) g.log.info("Successfully retrieved the node info %s" % node_id)
def test_to_retrieve_node_info(self): """ List and retrieve node related info """ # List all list g.log.info("Listing the node id") heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) self.assertTrue(heketi_node_id_list, ("Node Id list is empty.")) g.log.info("Successfully listed the node") for node_id in heketi_node_id_list: g.log.info("Retrieve the node info") node_info = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) self.assertTrue(node_info, ("Failed to " "retrieve the node info")) g.log.info("Successfully retrieved the node info %s" % node_id)
def setUp(self): super(TestDevPathMapping, self).setUp() self.node = self.ocp_master_node[0] self.h_node, self.h_server = (self.heketi_client_node, self.heketi_server_url) h_nodes_list = heketi_ops.heketi_node_list(self.h_node, self.h_server) h_node_count = len(h_nodes_list) if h_node_count < 3: self.skipTest( "At least 3 nodes are required, found {}".format(h_node_count)) # Disable 4th and other nodes for node_id in h_nodes_list[3:]: self.addCleanup(heketi_ops.heketi_node_enable, self.h_node, self.h_server, node_id) heketi_ops.heketi_node_disable(self.h_node, self.h_server, node_id) h_info = heketi_ops.heketi_node_info(self.h_node, self.h_server, h_nodes_list[0], json=True) self.assertTrue( h_info, "Failed to get the heketi node info for node id" " {}".format(h_nodes_list[0])) self.node_ip = h_info['hostnames']['storage'][0] self.node_hostname = h_info["hostnames"]["manage"][0] self.vm_name = node_ops.find_vm_name_by_ip_or_hostname( self.node_hostname) self.devices_list = [device['name'] for device in h_info["devices"]] # Get list of additional devices for one of the Gluster nodes for gluster_server in list(g.config["gluster_servers"].values()): if gluster_server['storage'] == self.node_ip: additional_device = gluster_server.get("additional_devices") if additional_device: self.devices_list.extend(additional_device) # sort the devices list self.devices_list.sort()
def setUp(self): super(TestHeketiBrickEvict, self).setUp() version = heketi_version.get_heketi_version(self.heketi_client_node) if version < '9.0.0-14': self.skipTest( "heketi-client package {} does not support brick evict".format( version.v_str)) node_list = heketi_ops.heketi_node_list(self.heketi_client_node, self.heketi_server_url) if len(node_list) > 3: return for node_id in node_list: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) if len(node_info["devices"]) < 2: self.skipTest("does not have extra device/node to evict brick")
def setUp(self): super(TestArbiterVolumeCreateExpandDelete, self).setUp() self.node = self.ocp_master_node[0] if get_openshift_version() < "3.9": self.skipTest("Arbiter feature cannot be used on OCP older " "than 3.9, because 'volumeoptions' for Heketi " "is not supported there.") version = heketi_version.get_heketi_version(self.heketi_client_node) if version < '6.0.0-11': self.skipTest("heketi-client package %s does not support arbiter " "functionality" % version.v_str) # Mark one of the Heketi nodes as arbiter-supported if none of # existent nodes or devices already enabled to support it. self.heketi_server_url = self.sc.get('resturl') arbiter_tags = ('required', 'supported') arbiter_already_supported = False self.node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) for node_id in self.node_id_list[::-1]: node_info = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) if node_info.get('tags', {}).get('arbiter') in arbiter_tags: arbiter_already_supported = True break for device in node_info['devices'][::-1]: if device.get('tags', {}).get('arbiter') in arbiter_tags: arbiter_already_supported = True break else: continue break if not arbiter_already_supported: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', self.node_id_list[0], 'supported')
def test_volume_create_as_tag_maching_rule(self): """Validate settags operation only on one device in the cluster""" h_node, h_server = self.heketi_client_node, self.heketi_server_url # Set tag on any one device in cluster node_list = heketi_node_list(h_node, h_server, json=True) node_info = heketi_node_info(h_node, h_server, node_list[0], json=True) device_id = node_info.get('devices', {})[0].get('id') set_tags(h_node, h_server, 'device', device_id, "tier:it") self.addCleanup(rm_tags, h_node, h_server, 'device', device_id, 'tier') # Volume creation should fail try: heketi_volume_create( h_node, h_server, 2, gluster_volume_options="user.heketi.device-tag-match tier=it") except AssertionError as e: if ("Failed to allocate new volume" not in six.text_type(e)): raise
def verify_free_space(self, free_space): # verify free space on nodes otherwise skip test case node_list = heketi_node_list(self.heketi_client_node, self.heketi_server_url) self.assertTrue(node_list) free_nodes = 0 for node in node_list: node_info = heketi_node_info(self.heketi_client_node, self.heketi_server_url, node, json=True) if node_info['state'] != 'online': continue free_size = 0 self.assertTrue(node_info['devices']) for device in node_info['devices']: if device['state'] != 'online': continue # convert size kb into gb device_f_size = device['storage']['free'] / 1048576 free_size += device_f_size if free_size > free_space: free_nodes += 1 break if free_nodes >= 3: break if free_nodes < 3: self.skipTest("skip test case because required free space is " "not available for creating BHV of size %s /n" "only %s free space is available" % (free_space, free_size))
def verify_free_space(self, free_space): # verify free space on nodes otherwise skip test case node_list = heketi_node_list( self.heketi_client_node, self.heketi_server_url) self.assertTrue(node_list) free_nodes = 0 for node in node_list: node_info = heketi_node_info( self.heketi_client_node, self.heketi_server_url, node, json=True) if node_info['state'] != 'online': continue free_size = 0 self.assertTrue(node_info['devices']) for device in node_info['devices']: if device['state'] != 'online': continue # convert size kb into gb device_f_size = device['storage']['free'] / 1048576 free_size += device_f_size if free_size > free_space: free_nodes += 1 break if free_nodes >= 3: break if free_nodes < 3: self.skipTest("skip test case because required free space is " "not available for creating BHV of size %s /n" "only %s free space is available" % (free_space, free_size))
def get_devices_summary_free_space(self): """ Calculates minimum free space per device and returns total free space across all devices """ free_spaces = [] heketi_node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) for node_id in heketi_node_id_list: node_info_dict = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) total_free_space = 0 for device in node_info_dict["devices"]: total_free_space += device["storage"]["free"] free_spaces.append(total_free_space) total_free_space = sum(free_spaces)/(1024 ** 2) total_free_space = int(math.floor(total_free_space)) return total_free_space
def _get_bricks_and_device_details(self): """Fetch bricks count and device id list from the node where dev path operation is performed """ h_client, h_url = self.heketi_client_node, self.heketi_server_url h_node_details = [] # Fetch bricks on the devices h_nodes = heketi_ops.heketi_node_list(h_client, h_url) for h_node in h_nodes: h_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) h_node_hostname = h_node_info.get("hostnames").get("manage")[0] # Fetch bricks count and device list if h_node_hostname == self.node_hostname: h_node_details = [[ node_info['id'], len(node_info['bricks']), node_info['name'] ] for node_info in h_node_info['devices']] return h_node_details, h_node
def test_arbiter_required_tag_on_node_or_devices_other_disabled( self, r_node_tag, d_node_tag, r_device_tag, d_device_tag): """Validate arbiter vol creation with node or device tag""" pvc_amount = 3 # Get Heketi nodes info node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node, self.heketi_server_url) # Disable n-3 nodes for node_id in node_id_list[3:]: heketi_ops.heketi_node_disable(self.heketi_client_node, self.heketi_server_url, node_id) self.addCleanup(heketi_ops.heketi_node_enable, self.heketi_client_node, self.heketi_server_url, node_id) # Set arbiter:required tags arbiter_node = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id_list[0], json=True) arbiter_nodes_ip_addresses = arbiter_node['hostnames']['storage'] self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id_list[0], ('required' if r_node_tag else None), revert_to=arbiter_node.get('tags', {}).get('arbiter')) for device in arbiter_node['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], ('required' if r_device_tag else None), revert_to=device.get('tags', {}).get('arbiter')) # Set arbiter:disabled tags data_nodes_ip_addresses = [] for node_id in node_id_list[1:]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) if not any([ int(d['storage']['free']) > (pvc_amount * 1024**2) for d in node_info['devices'] ]): self.skipTest("Devices are expected to have more than " "%sGb of free space" % pvc_amount) data_nodes_ip_addresses.extend(node_info['hostnames']['storage']) for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], ('disabled' if d_device_tag else None), revert_to=device.get('tags', {}).get('arbiter')) self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id, ('disabled' if d_node_tag else None), revert_to=node_info.get('tags', {}).get('arbiter')) # Create PVCs and check that their bricks are correctly located self.create_storage_class(is_arbiter_vol=True) for i in range(pvc_amount): self.create_and_wait_for_pvc(1) # Get gluster volume info vol_info = openshift_ops.get_gluster_vol_info_by_pvc_name( self.node, self.pvc_name) arbiter_bricks, data_bricks = [], [] for brick in vol_info['bricks']['brick']: if int(brick["isArbiter"]) == 1: arbiter_bricks.append(brick["name"]) else: data_bricks.append(brick["name"]) # Verify that all the arbiter bricks are located on # arbiter:required node and data bricks on all other nodes only. for arbiter_brick in arbiter_bricks: self.assertIn( arbiter_brick.split(':')[0], arbiter_nodes_ip_addresses) for data_brick in data_bricks: self.assertIn( data_brick.split(':')[0], data_nodes_ip_addresses)
def test_aribiter_required_tag_on_node_or_devices_other_disabled( self, node_with_tag): """Validate arbiter vol creation with required node or device tag""" pvc_amount = 3 # Get Heketi nodes info node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) # Set arbiter:required tags arbiter_node = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id_list[0], json=True) arbiter_nodes_ip_addresses = arbiter_node['hostnames']['storage'] self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id_list[0], ('required' if node_with_tag else None), revert_to=arbiter_node.get('tags', {}).get('arbiter')) for device in arbiter_node['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], (None if node_with_tag else 'required'), revert_to=device.get('tags', {}).get('arbiter')) # Set arbiter:disabled tags data_nodes, data_nodes_ip_addresses = [], [] for node_id in node_id_list[1:]: node_info = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) if not any([int(d['storage']['free']) > (pvc_amount * 1024**2) for d in node_info['devices']]): self.skipTest( "Devices are expected to have more than " "%sGb of free space" % pvc_amount) data_nodes_ip_addresses.extend(node_info['hostnames']['storage']) for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], (None if node_with_tag else 'disabled'), revert_to=device.get('tags', {}).get('arbiter')) self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id, ('disabled' if node_with_tag else None), revert_to=node_info.get('tags', {}).get('arbiter')) data_nodes.append(node_info) # Create PVCs and check that their bricks are correctly located self.create_storage_class(is_arbiter_vol=True) for i in range(pvc_amount): self.create_and_wait_for_pvc(1) # Get gluster volume info vol_info = get_gluster_vol_info_by_pvc_name( self.node, self.pvc_name) arbiter_bricks, data_bricks = [], [] for brick in vol_info['bricks']['brick']: if int(brick["isArbiter"]) == 1: arbiter_bricks.append(brick["name"]) else: data_bricks.append(brick["name"]) # Verify that all the arbiter bricks are located on # arbiter:required node and data bricks on all other nodes only. for arbiter_brick in arbiter_bricks: self.assertIn( arbiter_brick.split(':')[0], arbiter_nodes_ip_addresses) for data_brick in data_bricks: self.assertIn( data_brick.split(':')[0], data_nodes_ip_addresses)
def test_volume_expansion_no_free_space(self): """Validate volume expansion when there is no free space""" vol_size, expand_size, additional_devices_attached = None, 10, {} h_node, h_server_url = self.heketi_client_node, self.heketi_server_url # Get nodes info heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url) if len(heketi_node_id_list) < 3: self.skipTest("3 Heketi nodes are required.") # Disable 4th and other nodes for node_id in heketi_node_id_list[3:]: heketi_ops.heketi_node_disable(h_node, h_server_url, node_id) self.addCleanup( heketi_ops.heketi_node_enable, h_node, h_server_url, node_id) # Prepare first 3 nodes smallest_size = None err_msg = '' for node_id in heketi_node_id_list[0:3]: node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) # Disable second and other devices devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_id) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if (smallest_size is None or devices[0]["storage"]["free"] < smallest_size): smallest_size = devices[0]["storage"]["free"] for device in node_info["devices"][1:]: heketi_ops.heketi_device_disable( h_node, h_server_url, device["id"]) self.addCleanup( heketi_ops.heketi_device_enable, h_node, h_server_url, device["id"]) # Gather info about additional devices additional_device_name = None for gluster_server in self.gluster_servers: gluster_server_data = self.gluster_servers_info[gluster_server] g_manage = gluster_server_data["manage"] g_storage = gluster_server_data["storage"] if not (g_manage in node_info["hostnames"]["manage"] or g_storage in node_info["hostnames"]["storage"]): continue additional_device_name = (( gluster_server_data.get("additional_devices") or [''])[0]) break if not additional_device_name: err_msg += ("No 'additional_devices' are configured for " "'%s' node, which has following hostnames and " "IP addresses: %s.\n" % ( node_id, ', '.join(node_info["hostnames"]["manage"] + node_info["hostnames"]["storage"]))) continue heketi_ops.heketi_device_add( h_node, h_server_url, additional_device_name, node_id) additional_devices_attached.update( {node_id: additional_device_name}) # Schedule cleanup of the added devices for node_id in additional_devices_attached.keys(): node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) for device in node_info["devices"]: if device["name"] != additional_devices_attached[node_id]: continue self.addCleanup(self.detach_devices_attached, device["id"]) break else: self.fail("Could not find ID for added device on " "'%s' node." % node_id) if err_msg: self.skipTest(err_msg) # Temporary disable new devices self.disable_devices(additional_devices_attached) # Create volume and save info about it vol_size = int(smallest_size / (1024**2)) - 1 creation_info = heketi_ops.heketi_volume_create( h_node, h_server_url, vol_size, json=True) volume_name, volume_id = creation_info["name"], creation_info["id"] self.addCleanup( heketi_ops.heketi_volume_delete, h_node, h_server_url, volume_id, raise_on_error=False) volume_info_before_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) free_space_before_expansion = self.get_devices_summary_free_space() # Try to expand volume with not enough device space self.assertRaises( ExecutionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume self.enable_devices(additional_devices_attached) # Expand volume and validate results heketi_ops.heketi_volume_expand( h_node, h_server_url, volume_id, expand_size, json=True) free_space_after_expansion = self.get_devices_summary_free_space() self.assertGreater( free_space_before_expansion, free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) volume_info_after_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) self.assertGreater( volume_info_after_expansion["size"], volume_info_before_expansion["size"], "Size of %s not increased" % volume_id) self.assertGreater( num_of_bricks_after_expansion, num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0) # Delete volume and validate release of the used space heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertGreater( free_space_after_deletion, free_space_after_expansion, "Free space not reclaimed after deletion of volume %s" % volume_id)
def test_create_volumes_enabling_and_disabling_heketi_devices(self): """Validate enable/disable of heketi device""" # Get nodes info node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node, self.heketi_server_url) node_info_list = [] for node_id in node_id_list[0:3]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) node_info_list.append(node_info) # Disable 4th and other nodes if len(node_id_list) > 3: for node_id in node_id_list[3:]: heketi_ops.heketi_node_disable(self.heketi_client_node, self.heketi_server_url, node_id) self.addCleanup(heketi_ops.heketi_node_enable, self.heketi_client_node, self.heketi_server_url, node_id) # Disable second and other devices on the first 3 nodes for node_info in node_info_list[0:3]: devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_info["id"]) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if len(devices) < 2: continue for device in node_info["devices"][1:]: out = heketi_ops.heketi_device_disable(self.heketi_client_node, self.heketi_server_url, device["id"]) self.assertTrue( out, "Failed to disable the device %s" % device["id"]) self.addCleanup(heketi_ops.heketi_device_enable, self.heketi_client_node, self.heketi_server_url, device["id"]) # Create heketi volume out = heketi_ops.heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(out, "Failed to create heketi volume of size 1") g.log.info("Successfully created heketi volume of size 1") device_id = out["bricks"][0]["device"] self.addCleanup(heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, out["bricks"][0]["volume"]) # Disable device g.log.info("Disabling '%s' device" % device_id) out = heketi_ops.heketi_device_disable(self.heketi_client_node, self.heketi_server_url, device_id) self.assertTrue(out, "Failed to disable the device %s" % device_id) g.log.info("Successfully disabled device %s" % device_id) try: # Get device info g.log.info("Retrieving '%s' device info" % device_id) out = heketi_ops.heketi_device_info(self.heketi_client_node, self.heketi_server_url, device_id, json=True) self.assertTrue(out, "Failed to get device info %s" % device_id) g.log.info("Successfully retrieved device info %s" % device_id) name = out["name"] self.assertEqual(out["state"].lower().strip(), "offline", "Device %s is not in offline state." % name) g.log.info("Device %s is now offine" % name) # Try to create heketi volume g.log.info("Creating heketi volume: Expected to fail.") try: out = heketi_ops.heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) except AssertionError: g.log.info("Volume was not created as expected.") else: self.addCleanup(heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, out["bricks"][0]["volume"]) msg = "Volume unexpectedly created. Out: %s" % out assert False, msg finally: # Enable the device back g.log.info("Enable '%s' device back." % device_id) out = heketi_ops.heketi_device_enable(self.heketi_client_node, self.heketi_server_url, device_id) self.assertTrue(out, "Failed to enable the device %s" % device_id) g.log.info("Successfully enabled device %s" % device_id) # Get device info out = heketi_ops.heketi_device_info(self.heketi_client_node, self.heketi_server_url, device_id, json=True) self.assertTrue(out, ("Failed to get device info %s" % device_id)) g.log.info("Successfully retrieved device info %s" % device_id) name = out["name"] self.assertEqual(out["state"], "online", "Device %s is not in online state." % name) # Create heketi volume of size out = heketi_ops.heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(out, "Failed to create volume of size 1") self.addCleanup(heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, out["bricks"][0]["volume"]) g.log.info("Successfully created volume of size 1") name = out["name"] # Get gluster volume info vol_info = get_volume_info('auto_get_gluster_endpoint', volname=name) self.assertTrue(vol_info, "Failed to get '%s' volume info." % name) g.log.info("Successfully got the '%s' volume info." % name)
def test_dynamic_provisioning_glusterfile_gluster_pod_or_node_failure( self): """Create glusterblock PVC when gluster pod or node is down.""" mount_path = "/mnt" datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id()) # Create secret and storage class self.create_storage_class() # Create PVC pvc_name = self.create_and_wait_for_pvc() # Create app POD with attached volume pod_name = oc_create_tiny_pod_with_volume( self.node, pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path, image=self.io_container_image_cirros) self.addCleanup(wait_for_resource_absence, self.node, 'pod', pod_name) self.addCleanup(oc_delete, self.node, 'pod', pod_name) # Wait for app POD be up and running wait_for_pod_be_ready(self.node, pod_name, timeout=60, wait_step=2) # Run IO in background io_cmd = "oc rsh %s dd if=/dev/urandom of=%s bs=1000K count=900" % ( pod_name, datafile_path) async_io = g.run_async(self.node, io_cmd, "root") # Check for containerized Gluster if self.is_containerized_gluster(): # Pick up one of the hosts which stores PV brick (4+ nodes case) gluster_pod_data = get_gluster_pod_names_by_pvc_name( self.node, pvc_name)[0] # Delete glusterfs POD from chosen host and wait for # spawn of new one oc_delete(self.node, 'pod', gluster_pod_data["pod_name"]) cmd = ("oc get pods -o wide | grep glusterfs | grep %s | " "grep -v Terminating | awk '{print $1}'") % ( gluster_pod_data["pod_hostname"]) for w in Waiter(600, 15): new_gluster_pod_name = self.cmd_run(cmd) if new_gluster_pod_name: break if w.expired: error_msg = "exceeded timeout, new gluster pod not created" g.log.error(error_msg) raise AssertionError(error_msg) g.log.info("new gluster pod name is %s" % new_gluster_pod_name) wait_for_pod_be_ready(self.node, new_gluster_pod_name) else: pvc_hosting_node_ip = get_gluster_host_ips_by_pvc_name( self.node, pvc_name)[0] heketi_nodes = heketi_node_list(self.heketi_client_node, self.heketi_server_url) node_ip_for_reboot = None for heketi_node in heketi_nodes: heketi_node_ip = heketi_node_info( self.heketi_client_node, self.heketi_server_url, heketi_node, json=True)["hostnames"]["storage"][0] if heketi_node_ip == pvc_hosting_node_ip: node_ip_for_reboot = heketi_node_ip break if not node_ip_for_reboot: raise AssertionError( "Gluster node IP %s not matched with heketi node %s" % (pvc_hosting_node_ip, heketi_node_ip)) node_reboot_by_command(node_ip_for_reboot) # Check that async IO was not interrupted ret, out, err = async_io.async_communicate() self.assertEqual(ret, 0, "IO %s failed on %s" % (io_cmd, self.node))
def test_pv_resize_device_disabled(self): """Validate resize after disabling all devices except one""" h_node, h_url = self.heketi_client_node, self.heketi_server_url # expand volume size and path volume is mounted expand_size, dir_path = 7, "/mnt" # Get nodes info heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_url) if len(heketi_node_id_list) < 3: self.skipTest( "At-least 3 gluster nodes are required to execute test case") self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc(pvc_size=2) vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) dc_name, pod_name = self.create_dc_with_pvc(pvc_name) self._write_file(pod_name, "file1", "1G", dir_path) with self.assertRaises(AssertionError): self._write_file(pod_name, "file2", "3G", dir_path) # Prepare first 3 nodes and then disable other devices. for node_id in heketi_node_id_list[:3]: node_info = heketi_ops.heketi_node_info(h_node, h_url, node_id, json=True) self.assertTrue(node_info, "Failed to get node info") devices = node_info.get("devices", None) self.assertTrue(devices, "Node {} does not have devices".format(node_id)) if devices[0]["state"].strip().lower() != "online": self.skipTest("Skipping test as it expects to first device to" " be enabled") for device in devices[1:]: heketi_ops.heketi_device_disable(h_node, h_url, device["id"]) self.addCleanup(heketi_ops.heketi_device_enable, h_node, h_url, device["id"]) usedsize_before_resize = self._get_mount_size(pod_name, dir_path) # Resize pvc resize_pvc(self.node, pvc_name, expand_size) verify_pvc_size(self.node, pvc_name, expand_size) vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) self.assertFalse(len(vol_info['bricks']['brick']) % 3) for node_id in heketi_node_id_list[:3]: for device in devices[1:]: heketi_ops.heketi_device_enable(h_node, h_url, device["id"]) self._write_file(pod_name, "file3", "3G", dir_path) usedsize_after_resize = self._get_mount_size(pod_name, dir_path) self.assertGreater( int(usedsize_before_resize.strip('%')), int(usedsize_after_resize.strip('%')), "Mount size {} should be greater than {}".format( usedsize_before_resize, usedsize_after_resize)) self._write_file(pod_name, "file4", "1024", dir_path) # Validate dist-rep volume with 6 bricks after pv resize vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) self.assertEqual( 6, len(vol_info['bricks']['brick']), "Expected bricks count is 6, but actual brick count is {}".format( len(vol_info['bricks']['brick'])))
def test_node_state(self): """ Test node enable and disable functionality. If we have 4 gluster servers, if we disable 1/4 nodes from heketi and create a volume, the volume creation should be successful. If we disable 2/4 nodes from heketi-cli and create a volume the volume creation should fail. If we enable back one gluster server and create a volume the volume creation should be successful. """ g.log.info("Disable node in heketi") node_list = heketi_node_list(self.heketi_client_node, self.heketi_server_url) self.assertTrue(node_list, "Failed to list heketi nodes") g.log.info("Successfully got the list of nodes") online_hosts = self.get_online_nodes(node_list) if len(online_hosts) < 3: raise self.skipTest( "This test can run only if online hosts are more " "than 2") # if we have n nodes, disable n-3 nodes for node_info in online_hosts[3:]: node_id = node_info["id"] g.log.info("going to disable node id %s", node_id) self.disable_node(node_id) self.addCleanup(self.enable_node, node_id) vol_size = 1 # create volume when 3 nodes are online vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, vol_size, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) node_id = online_hosts[0]['id'] g.log.info("going to disable node id %s", node_id) self.disable_node(node_id) self.addCleanup(self.enable_node, node_id) # try to create a volume, volume creation should fail ret, out, err = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, vol_size, raw_cli_output=True) if ret == 0: out_json = json.loads(out) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, out_json["id"]) self.assertNotEqual(ret, 0, ("Volume creation did not fail ret- %s " "out- %s err- %s" % (ret, out, err))) g.log.info("Volume creation failed as expected, err- %s", err) # enable node self.enable_node(node_id) # create volume when node is enabled vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, vol_size, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def _pv_resize(self, exceed_free_space): dir_path = "/mnt" pvc_size_gb, min_free_space_gb = 1, 3 # Get available free space disabling redundant devices and nodes heketi_url = self.heketi_server_url node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, heketi_url) self.assertTrue(node_id_list) nodes = {} min_free_space = min_free_space_gb * 1024**2 for node_id in node_id_list: node_info = heketi_ops.heketi_node_info( self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue if len(nodes) > 2: out = heketi_ops.heketi_node_disable( self.heketi_client_node, heketi_url, node_id) self.assertTrue(out) self.addCleanup( heketi_ops.heketi_node_enable, self.heketi_client_node, heketi_url, node_id) for device in node_info['devices']: if device['state'].lower() != 'online': continue free_space = device['storage']['free'] if (node_id in nodes.keys() or free_space < min_free_space): out = heketi_ops.heketi_device_disable( self.heketi_client_node, heketi_url, device['id']) self.assertTrue(out) self.addCleanup( heketi_ops.heketi_device_enable, self.heketi_client_node, heketi_url, device['id']) continue nodes[node_id] = free_space if len(nodes) < 3: raise self.skipTest( "Could not find 3 online nodes with, " "at least, 1 online device having free space " "bigger than %dGb." % min_free_space_gb) # Calculate maximum available size for PVC available_size_gb = int(min(nodes.values()) / (1024**2)) # Create PVC self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb) # Create DC with POD and attached PVC to it dc_name = oc_create_app_dc_with_io(self.node, pvc_name) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) if exceed_free_space: # Try to expand existing PVC exceeding free space resize_pvc(self.node, pvc_name, available_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeFailed') # Check that app POD is up and runnig then try to write data wait_for_pod_be_ready(self.node, pod_name) cmd = ( "dd if=/dev/urandom of=%s/autotest bs=100K count=1" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to write data after failed attempt to expand PVC.") else: # Expand existing PVC using all the available free space expand_size_gb = available_size_gb - pvc_size_gb resize_pvc(self.node, pvc_name, expand_size_gb) verify_pvc_size(self.node, pvc_name, expand_size_gb) pv_name = get_pv_name_from_pvc(self.node, pvc_name) verify_pv_size(self.node, pv_name, expand_size_gb) wait_for_events( self.node, obj_name=pvc_name, event_reason='VolumeResizeSuccessful') # Recreate app POD oc_delete(self.node, 'pod', pod_name) wait_for_resource_absence(self.node, 'pod', pod_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Write data on the expanded PVC cmd = ("dd if=/dev/urandom of=%s/autotest " "bs=1M count=1025" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to write data on the expanded PVC")
def _pv_resize(self, exceed_free_space): dir_path = "/mnt" pvc_size_gb, min_free_space_gb = 1, 3 # Get available free space disabling redundant devices and nodes heketi_url = self.heketi_server_url node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_id_list) nodes = {} min_free_space = min_free_space_gb * 1024**2 for node_id in node_id_list: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue if len(nodes) > 2: out = heketi_ops.heketi_node_disable(self.heketi_client_node, heketi_url, node_id) self.assertTrue(out) self.addCleanup(heketi_ops.heketi_node_enable, self.heketi_client_node, heketi_url, node_id) for device in node_info['devices']: if device['state'].lower() != 'online': continue free_space = device['storage']['free'] if (node_id in nodes.keys() or free_space < min_free_space): out = heketi_ops.heketi_device_disable( self.heketi_client_node, heketi_url, device['id']) self.assertTrue(out) self.addCleanup(heketi_ops.heketi_device_enable, self.heketi_client_node, heketi_url, device['id']) continue nodes[node_id] = free_space if len(nodes) < 3: raise self.skipTest("Could not find 3 online nodes with, " "at least, 1 online device having free space " "bigger than %dGb." % min_free_space_gb) # Calculate maximum available size for PVC available_size_gb = int(min(nodes.values()) / (1024**2)) # Create PVC self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb) # Create DC with POD and attached PVC to it dc_name = oc_create_app_dc_with_io(self.node, pvc_name) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) if exceed_free_space: # Try to expand existing PVC exceeding free space resize_pvc(self.node, pvc_name, available_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeFailed') # Check that app POD is up and runnig then try to write data wait_for_pod_be_ready(self.node, pod_name) cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to write data after failed attempt to expand PVC.") else: # Expand existing PVC using all the available free space expand_size_gb = available_size_gb - pvc_size_gb resize_pvc(self.node, pvc_name, expand_size_gb) verify_pvc_size(self.node, pvc_name, expand_size_gb) pv_name = get_pv_name_from_pvc(self.node, pvc_name) verify_pv_size(self.node, pv_name, expand_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeSuccessful') # Recreate app POD oc_delete(self.node, 'pod', pod_name) wait_for_resource_absence(self.node, 'pod', pod_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Write data on the expanded PVC cmd = ("dd if=/dev/urandom of=%s/autotest " "bs=1M count=1025" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual(ret, 0, "Failed to write data on the expanded PVC")
def test_heketi_device_removal_with_insuff_space(self): """Validate heketi with device removal insufficient space""" # Disable 4+ nodes and 3+ devices on the first 3 nodes min_free_space_gb = 5 min_free_space = min_free_space_gb * 1024**2 heketi_url = self.heketi_server_url heketi_node = self.heketi_client_node nodes = {} node_ids = heketi_node_list(heketi_node, heketi_url) self.assertTrue(node_ids) for node_id in node_ids: node_info = heketi_node_info(heketi_node, heketi_url, node_id, json=True) if (node_info["state"].lower() != "online" or not node_info["devices"]): continue if len(nodes) > 2: heketi_node_disable(heketi_node, heketi_url, node_id) self.addCleanup(heketi_node_enable, heketi_node, heketi_url, node_id) continue for device in node_info["devices"]: if device["state"].lower() != "online": continue free_space = device["storage"]["free"] if node_id not in nodes: nodes[node_id] = [] if (free_space < min_free_space or len(nodes[node_id]) > 1): heketi_device_disable(heketi_node, heketi_url, device["id"]) self.addCleanup(heketi_device_enable, heketi_node, heketi_url, device["id"]) continue nodes[node_id].append({ "device_id": device["id"], "free": free_space }) # Skip test if nodes requirements are not met if (len(nodes) < 3 or not all(map((lambda _l: len(_l) > 1), nodes.values()))): raise self.skipTest( "Could not find 3 online nodes with 2 online devices " "having free space bigger than %dGb." % min_free_space_gb) # Calculate size of a potential distributed vol if nodes[node_ids[0]][0]["free"] > nodes[node_ids[0]][1]["free"]: index = 0 else: index = 1 vol_size_gb = int(nodes[node_ids[0]][index]["free"] / (1024**2)) + 1 device_id = nodes[node_ids[0]][index]["device_id"] # Create volume with such size that we consume space more than # size of smaller disks h_volume_name = "autotests-heketi-volume-%s" % utils.get_random_str() try: self.create_heketi_volume_with_name_and_wait(h_volume_name, vol_size_gb, json=True) except Exception as e: # NOTE: rare situation when we need to decrease size of a volume. g.log.info("Failed to create '%s'Gb volume. " "Trying to create another one, smaller for 1Gb.") if not ('more required' in str(e) and ('Insufficient suitable allocatable extents for ' 'logical volume' in str(e))): raise vol_size_gb -= 1 self.create_heketi_volume_with_name_and_wait(h_volume_name, vol_size_gb, json=True) # Try to 'remove' bigger Heketi disk expecting error, # because there is no space on smaller disk to relocate bricks to heketi_device_disable(heketi_node, heketi_url, device_id) self.addCleanup(heketi_device_enable, heketi_node, heketi_url, device_id) try: self.assertRaises(AssertionError, heketi_device_remove, heketi_node, heketi_url, device_id) except Exception: self.addCleanup(heketi_device_disable, heketi_node, heketi_url, device_id) raise
def test_create_volumes_enabling_and_disabling_heketi_devices(self): """Validate enable/disable of heketi device""" # Get nodes info node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, self.heketi_server_url) node_info_list = [] for node_id in node_id_list[0:3]: node_info = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) node_info_list.append(node_info) # Disable 4th and other nodes if len(node_id_list) > 3: for node in node_id_list[3:]: heketi_ops.heketi_node_disable( self.heketi_client_node, self.heketi_server_url, node_id) self.addCleanup( heketi_ops.heketi_node_enable, self.heketi_client_node, self.heketi_server_url, node_id) # Disable second and other devices on the first 3 nodes for node_info in node_info_list[0:3]: devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_info["id"]) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if len(devices) < 2: continue for device in node_info["devices"][1:]: out = heketi_ops.heketi_device_disable( self.heketi_client_node, self.heketi_server_url, device["id"]) self.assertTrue( out, "Failed to disable the device %s" % device["id"]) self.addCleanup( heketi_ops.heketi_device_enable, self.heketi_client_node, self.heketi_server_url, device["id"]) # Create heketi volume out = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(out, "Failed to create heketi volume of size 1") g.log.info("Successfully created heketi volume of size 1") device_id = out["bricks"][0]["device"] self.addCleanup( heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, out["bricks"][0]["volume"]) # Disable device g.log.info("Disabling '%s' device" % device_id) out = heketi_ops.heketi_device_disable( self.heketi_client_node, self.heketi_server_url, device_id) self.assertTrue(out, "Failed to disable the device %s" % device_id) g.log.info("Successfully disabled device %s" % device_id) try: # Get device info g.log.info("Retrieving '%s' device info" % device_id) out = heketi_ops.heketi_device_info( self.heketi_client_node, self.heketi_server_url, device_id, json=True) self.assertTrue(out, "Failed to get device info %s" % device_id) g.log.info("Successfully retrieved device info %s" % device_id) name = out["name"] if out["state"].lower().strip() != "offline": raise exceptions.ExecutionError( "Device %s is not in offline state." % name) g.log.info("Device %s is now offine" % name) # Try to create heketi volume g.log.info("Creating heketi volume: Expected to fail.") try: out = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) except exceptions.ExecutionError: g.log.info("Volume was not created as expected.") else: self.addCleanup( heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, out["bricks"][0]["volume"]) msg = "Volume unexpectedly created. Out: %s" % out assert False, msg finally: # Enable the device back g.log.info("Enable '%s' device back." % device_id) out = heketi_ops.heketi_device_enable( self.heketi_client_node, self.heketi_server_url, device_id) self.assertTrue(out, "Failed to enable the device %s" % device_id) g.log.info("Successfully enabled device %s" % device_id) # Get device info out = heketi_ops.heketi_device_info( self.heketi_client_node, self.heketi_server_url, device_id, json=True) self.assertTrue(out, ("Failed to get device info %s" % device_id)) g.log.info("Successfully retrieved device info %s" % device_id) name = out["name"] if out["state"] != "online": raise exceptions.ExecutionError( "Device %s is not in online state." % name) # Create heketi volume of size out = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(out, "Failed to create volume of size 1") self.addCleanup( heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, out["bricks"][0]["volume"]) g.log.info("Successfully created volume of size 1") name = out["name"] # Get gluster volume info vol_info = get_volume_info('auto_get_gluster_endpoint', volname=name) self.assertTrue(vol_info, "Failed to get '%s' volume info." % name) g.log.info("Successfully got the '%s' volume info." % name)