def test_heketi_metrics_validating_vol_count_on_vol_deletion(self): """Validate heketi metrics VolumeCount after volume deletion""" vol_list = [] for i in range(3): # Create volume vol = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(vol) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol['id'], raise_on_error=False) volume_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertIn(vol['id'], volume_list) vol_list.append(vol) for vol in vol_list: # delete volume heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol['id']) volume_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertNotIn(vol['id'], volume_list) self.verify_volume_count()
def test_restart_heketi_pod(self): """Validate restarting heketi pod""" # create heketi volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size=1, json=True) self.assertTrue(vol_info, "Failed to create heketi volume of size 1") self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id'], raise_on_error=False) topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) # get heketi-pod name heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0], self.heketi_dc_name) # delete heketi-pod (it restarts the pod) oc_delete(self.ocp_master_node[0], 'pod', heketi_pod_name) wait_for_resource_absence(self.ocp_master_node[0], 'pod', heketi_pod_name) # get new heketi-pod name heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0], self.heketi_dc_name) wait_for_pod_be_ready(self.ocp_master_node[0], heketi_pod_name) # check heketi server is running self.assertTrue( hello_heketi(self.heketi_client_node, self.heketi_server_url), "Heketi server %s is not alive" % self.heketi_server_url ) # compare the topology new_topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) self.assertEqual(new_topo_info, topo_info, "topology info is not same," " difference - %s" % diff(topo_info, new_topo_info)) # create new volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size=2, json=True) self.assertTrue(vol_info, "Failed to create heketi volume of size 20") heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self): """Validate retain policy for glusterfs after deletion of pvc""" self.create_storage_class(reclaim_policy='Retain') self.create_and_wait_for_pvc() # get the name of the volume pv_name = get_pv_name_from_pvc(self.node, self.pvc_name) custom = [ r':.metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi\-volume\-id"', r':.spec.persistentVolumeReclaimPolicy' ] vol_id, reclaim_policy = oc_get_custom_resource( self.node, 'pv', custom, pv_name) self.assertEqual(reclaim_policy, 'Retain') # Create DC with POD and attached PVC to it. try: dc_name = oc_create_app_dc_with_io(self.node, self.pvc_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) finally: scale_dc_pod_amount_and_wait(self.node, dc_name, 0) oc_delete(self.node, 'dc', dc_name) wait_for_resource_absence(self.node, 'pod', pod_name) oc_delete(self.node, 'pvc', self.pvc_name) with self.assertRaises(ExecutionError): wait_for_resource_absence(self.node, 'pvc', self.pvc_name, interval=3, timeout=30) heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol_id) vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertNotIn(vol_id, vol_list) oc_delete(self.node, 'pv', pv_name) wait_for_resource_absence(self.node, 'pv', pv_name)
def test_delete_heketi_volume(self): """ Method to test heketi volume deletion and whether it frees up used space after deletion """ creation_output_dict = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.assertNotEqual(creation_output_dict, False, "Volume creation failed") volume_id = creation_output_dict["name"].strip().split("_")[1] free_space_after_creation = self.get_free_space_summary_devices() deletion_output = heketi_ops.heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, volume_id) self.assertNotEqual(deletion_output, False, "Deletion of volume failed, id: %s" % volume_id) free_space_after_deletion = self.get_free_space_summary_devices() self.assertTrue( free_space_after_deletion > free_space_after_creation, "Free space is not reclaimed after deletion of %s" % volume_id)
def test_to_check_entry_in_fstab_file(self): """Validate /etc/fstab entries after creation/deletion of volume""" # Create heketi volume vol = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size=1, json=True) self.assertTrue(vol, "Failed to create 1Gb heketi volume") vol_id = vol["bricks"][0]["volume"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_id, raise_on_error=False) # Gather brick paths brick_paths = [p['path'].rstrip("/brick") for p in vol["bricks"]] # Make sure that volume's brick paths exist in the fstab files self._find_bricks_in_fstab_files(brick_paths, present=True) # Delete heketi volume out = heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol_id) self.assertTrue(out, "Failed to delete heketi volume %s" % vol_id) # Make sure that volume's brick paths are absent in the fstab file self._find_bricks_in_fstab_files(brick_paths, present=False)
def test_validate_brick_paths_on_gluster_pods_or_nodes(self): """Validate brick paths after creation and deletion of a volume.""" # Create heketi volume vol = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size=1, json=True) self.assertTrue(vol, "Failed to create 1Gb heketi volume") vol_id = vol["bricks"][0]["volume"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_id, raise_on_error=False) # Gather brick paths brick_paths = [p['path'] for p in vol["bricks"]] # Make sure that volume's brick paths exist in the fstab files self._find_bricks(brick_paths, present=True) # Delete heketi volume out = heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol_id) self.assertTrue(out, "Failed to delete heketi volume %s" % vol_id) # Make sure that volume's brick paths are absent in the fstab file self._find_bricks(brick_paths, present=False)
def test_block_host_volume_delete_without_block_volumes(self): """Validate deletion of empty block hosting volume""" block_host_create_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True, block=True) block_hosting_vol_id = block_host_create_info["id"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, block_hosting_vol_id, raise_on_error=False) heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, block_hosting_vol_id, json=True)
def test_delete_heketidb_volume(self): """ Method to test heketidb volume deletion via heketi-cli """ volume_id_list = [] heketidbexists = False msg = "Error: Cannot delete volume containing the Heketi database" for i in range(0, 2): volume_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.assertNotEqual(volume_info, False, "Volume creation failed") volume_id_list.append(volume_info["id"]) self.addCleanup(self.delete_volumes, volume_id_list) volume_list_info = heketi_ops.heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertNotEqual(volume_list_info, False, "Heketi volume list command failed") if volume_list_info["volumes"] == []: raise ExecutionError("Heketi volume list empty") for volume_id in volume_list_info["volumes"]: volume_info = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) if volume_info["name"] == "heketidbstorage": heketidbexists = True delete_ret, delete_output, delete_error = ( heketi_ops.heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, volume_id, raw_cli_output=True)) self.assertNotEqual(delete_ret, 0, "Return code not 0") self.assertEqual( delete_error.strip(), msg, "Invalid reason for heketidb deletion failure") if not heketidbexists: raise ExecutionError( "Warning: heketidbstorage doesn't exist in list of volumes")
def delete_volumes(self, volume_ids): """ Delete volumes by their IDs and raise error with list of failures Input: (volume_ids) It can be a single volume ID or a list of volume IDs """ errored_ids = [] if not isinstance(volume_ids, (list, set, tuple)): volume_ids = [volume_ids] for volume_id in volume_ids: out = heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, volume_id) output_str = 'Volume %s deleted' % volume_id if output_str not in out: errored_ids.append(volume_id) if errored_ids: raise ExecutionError("Failed to delete following heketi volumes: " "%s" % ',\n'.join(errored_ids))
def test_block_host_volume_delete_without_block_volumes(self): """Validate deletion of empty block hosting volume""" block_host_create_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True, block=True) self.assertNotEqual(block_host_create_info, False, "Block host volume creation failed") block_hosting_vol_id = block_host_create_info["id"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, block_hosting_vol_id, raise_on_error=False) block_host_delete_output = heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, block_hosting_vol_id, json=True) self.assertNotEqual( block_host_delete_output, False, "Block host volume delete failed, ID: %s" % block_hosting_vol_id)
def test_volume_expansion_rebalance_brick(self): """Validate volume expansion with brick and check rebalance""" creation_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.assertNotEqual(creation_info, False, "Volume creation failed") volume_name = creation_info["name"] volume_id = creation_info["id"] free_space_after_creation = self.get_devices_summary_free_space() volume_info_before_expansion = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(volume_info_before_expansion, False, "Volume info for %s failed" % volume_id) heketi_vol_info_size_before_expansion = ( volume_info_before_expansion["size"]) self.get_brick_and_volume_status(volume_name) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) expansion_info = heketi_ops.heketi_volume_expand( self.heketi_client_node, self.heketi_server_url, volume_id, 5) self.assertNotEqual(expansion_info, False, "Volume expansion of %s failed" % volume_id) free_space_after_expansion = self.get_devices_summary_free_space() self.assertTrue( free_space_after_creation > free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) volume_info_after_expansion = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(volume_info_after_expansion, False, "Volume info failed for %s" % volume_id) heketi_vol_info_size_after_expansion = ( volume_info_after_expansion["size"]) difference_size = (heketi_vol_info_size_after_expansion - heketi_vol_info_size_before_expansion) self.assertTrue(difference_size > 0, "Size not increased after expansion of %s" % volume_id) self.get_brick_and_volume_status(volume_name) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) num_of_bricks_added = (num_of_bricks_after_expansion - num_of_bricks_before_expansion) self.assertEqual(num_of_bricks_added, 3, "Number of bricks added is not 3 for %s" % volume_id) self.get_rebalance_status(volume_name) deletion_info = heketi_ops.heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(deletion_info, False, "Deletion of volume %s failed" % volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertTrue( free_space_after_deletion > free_space_after_expansion, "Free space is not reclaimed after volume deletion of %s" % volume_id)
def test_volume_expansion_no_free_space(self): """Validate volume expansion when there is no free space""" vol_size, expand_size, additional_devices_attached = None, 10, {} h_node, h_server_url = self.heketi_client_node, self.heketi_server_url # Get nodes info heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url) if len(heketi_node_id_list) < 3: self.skipTest("3 Heketi nodes are required.") # Disable 4th and other nodes for node_id in heketi_node_id_list[3:]: heketi_ops.heketi_node_disable(h_node, h_server_url, node_id) self.addCleanup(heketi_ops.heketi_node_enable, h_node, h_server_url, node_id) # Prepare first 3 nodes smallest_size = None err_msg = '' for node_id in heketi_node_id_list[0:3]: node_info = heketi_ops.heketi_node_info(h_node, h_server_url, node_id, json=True) # Disable second and other devices devices = node_info["devices"] self.assertTrue(devices, "Node '%s' does not have devices." % node_id) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if (smallest_size is None or devices[0]["storage"]["free"] < smallest_size): smallest_size = devices[0]["storage"]["free"] for device in node_info["devices"][1:]: heketi_ops.heketi_device_disable(h_node, h_server_url, device["id"]) self.addCleanup(heketi_ops.heketi_device_enable, h_node, h_server_url, device["id"]) # Gather info about additional devices additional_device_name = None for gluster_server in self.gluster_servers: gluster_server_data = self.gluster_servers_info[gluster_server] g_manage = gluster_server_data["manage"] g_storage = gluster_server_data["storage"] if not (g_manage in node_info["hostnames"]["manage"] or g_storage in node_info["hostnames"]["storage"]): continue additional_device_name = (( gluster_server_data.get("additional_devices") or [''])[0]) break if not additional_device_name: err_msg += ( "No 'additional_devices' are configured for " "'%s' node, which has following hostnames and " "IP addresses: %s.\n" % (node_id, ', '.join(node_info["hostnames"]["manage"] + node_info["hostnames"]["storage"]))) continue heketi_ops.heketi_device_add(h_node, h_server_url, additional_device_name, node_id) additional_devices_attached.update( {node_id: additional_device_name}) # Schedule cleanup of the added devices for node_id in additional_devices_attached.keys(): node_info = heketi_ops.heketi_node_info(h_node, h_server_url, node_id, json=True) for device in node_info["devices"]: if device["name"] != additional_devices_attached[node_id]: continue self.addCleanup(self.detach_devices_attached, device["id"]) break else: self.fail("Could not find ID for added device on " "'%s' node." % node_id) if err_msg: self.skipTest(err_msg) # Temporary disable new devices self.disable_devices(additional_devices_attached) # Create volume and save info about it vol_size = int(smallest_size / (1024**2)) - 1 creation_info = heketi_ops.heketi_volume_create(h_node, h_server_url, vol_size, json=True) volume_name, volume_id = creation_info["name"], creation_info["id"] self.addCleanup(heketi_ops.heketi_volume_delete, h_node, h_server_url, volume_id, raise_on_error=False) volume_info_before_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) free_space_before_expansion = self.get_devices_summary_free_space() # Try to expand volume with not enough device space self.assertRaises(ExecutionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume self.enable_devices(additional_devices_attached) # Expand volume and validate results heketi_ops.heketi_volume_expand(h_node, h_server_url, volume_id, expand_size, json=True) free_space_after_expansion = self.get_devices_summary_free_space() self.assertGreater( free_space_before_expansion, free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) volume_info_after_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) self.assertGreater(volume_info_after_expansion["size"], volume_info_before_expansion["size"], "Size of %s not increased" % volume_id) self.assertGreater(num_of_bricks_after_expansion, num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0) # Delete volume and validate release of the used space heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertGreater( free_space_after_deletion, free_space_after_expansion, "Free space not reclaimed after deletion of volume %s" % volume_id)
def _create_distributed_replica_vol(self, validate_cleanup): # Create distributed vol vol_size_gb = self._get_vol_size() heketi_url = self.heketi_server_url try: g.log.info( "Trying to create distributed '%s'Gb volume." % vol_size_gb) heketi_vol = heketi_volume_create( self.heketi_client_node, heketi_url, vol_size_gb, json=True) except exceptions.ExecutionError as e: # NOTE: rare situation when we need to decrease size of a volume. # and we expect this vol to be distributed. g.log.info("Failed to create distributed '%s'Gb volume. " "Trying to create another one, smaller for 1Gb.") if ('more required' in str(e) and ('Insufficient suitable allocatable extents for ' 'logical volume' in str(e))): vol_size_gb -= 1 heketi_vol = heketi_volume_create( self.heketi_client_node, heketi_url, vol_size_gb, json=True) else: raise g.log.info("Successfully created distributed volume.") vol_name = heketi_vol['name'] vol_id = heketi_vol["bricks"][0]["volume"] self.addCleanup( heketi_volume_delete, self.heketi_client_node, heketi_url, vol_id, raise_on_error=(not validate_cleanup)) # Get gluster volume info g.log.info("Get gluster volume '%s' info" % vol_name) gluster_vol = get_volume_info( 'auto_get_gluster_endpoint', volname=vol_name) self.assertTrue( gluster_vol, "Failed to get volume '%s' info" % vol_name) g.log.info("Successfully got volume '%s' info" % vol_name) gluster_vol = gluster_vol[vol_name] self.assertEqual( gluster_vol["typeStr"], "Distributed-Replicate", "'%s' gluster vol isn't a Distributed-Replicate volume" % vol_name) # Check amount of bricks brick_amount = len(gluster_vol['bricks']['brick']) self.assertEqual(brick_amount % 3, 0, "Brick amount is expected to be divisible by 3. " "Actual amount is '%s'" % brick_amount) self.assertGreater(brick_amount, 3, "Brick amount is expected to be bigger than 3. " "Actual amount is '%s'." % brick_amount) # Run unique actions to Validate whether deleting a dist-rep # volume is handled by heketi else return if not validate_cleanup: return # Get the free space after creating heketi volume free_space_after_creating_vol = self._get_free_space() # Delete heketi volume g.log.info("Deleting heketi volume '%s'" % vol_id) volume_deleted = heketi_volume_delete( self.heketi_client_node, heketi_url, vol_id) self.assertTrue( volume_deleted, "Failed to delete heketi volume '%s'" % vol_id) g.log.info("Heketi volume '%s' has successfully been deleted" % vol_id) # Check the heketi volume list g.log.info("List heketi volumes") heketi_volumes = heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(heketi_volumes, "Failed to list heketi volumes") g.log.info("Heketi volumes have successfully been listed") heketi_volumes = heketi_volumes.get('volumes', heketi_volumes) self.assertNotIn(vol_id, heketi_volumes) self.assertNotIn(vol_name, heketi_volumes) # Check the gluster volume list g.log.info("Get the gluster volume list") gluster_volumes = get_volume_list('auto_get_gluster_endpoint') self.assertTrue(gluster_volumes, "Unable to get Gluster volume list") g.log.info("Successfully got Gluster volume list" % gluster_volumes) self.assertNotIn(vol_id, gluster_volumes) self.assertNotIn(vol_name, gluster_volumes) # Get the used space after deleting heketi volume free_space_after_deleting_vol = self._get_free_space() # Compare the free space before and after deleting the volume g.log.info("Comparing the free space before and after deleting volume") self.assertLessEqual( free_space_after_creating_vol + (3 * vol_size_gb), free_space_after_deleting_vol) g.log.info("Volume successfully deleted and space is reallocated. " "Free space after creating volume %s. " "Free space after deleting volume %s." % ( free_space_after_creating_vol, free_space_after_deleting_vol))