def test_heketi_metrics_validating_vol_count_on_vol_deletion(self): """Validate heketi metrics VolumeCount after volume deletion""" vol_list = [] for i in range(3): # Create volume vol = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(vol) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol['id'], raise_on_error=False) volume_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertIn(vol['id'], volume_list) vol_list.append(vol) for vol in vol_list: # delete volume heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol['id']) volume_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertNotIn(vol['id'], volume_list) self.verify_volume_count()
def test_volume_create_and_list_volume(self): """Validate heketi and gluster volume list""" g.log.info("List gluster volumes before Heketi volume creation") existing_g_vol_list = get_volume_list('auto_get_gluster_endpoint') self.assertTrue(existing_g_vol_list, ("Unable to get volumes list")) g.log.info("List heketi volumes before volume creation") existing_h_vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url, json=True)["volumes"] g.log.info("Heketi volumes successfully listed") g.log.info("Create a heketi volume") out = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) g.log.info("Heketi volume successfully created" % out) volume_id = out["bricks"][0]["volume"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) g.log.info("List heketi volumes after volume creation") h_vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url, json=True)["volumes"] g.log.info("Heketi volumes successfully listed") g.log.info("List gluster volumes after Heketi volume creation") g_vol_list = get_volume_list('auto_get_gluster_endpoint') self.assertTrue(g_vol_list, ("Unable to get volumes list")) g.log.info("Successfully got the volumes list") # Perform checks self.assertEqual( len(existing_g_vol_list) + 1, len(g_vol_list), "Expected creation of only one volume in Gluster creating " "Heketi volume. Here is lists before and after volume creation: " "%s \n%s" % (existing_g_vol_list, g_vol_list)) self.assertEqual( len(existing_h_vol_list) + 1, len(h_vol_list), "Expected creation of only one volume in Heketi. Here is lists " "of Heketi volumes before and after volume creation: %s\n%s" % (existing_h_vol_list, h_vol_list))
def test_to_check_deletion_of_cluster(self): """Validate deletion of cluster with volumes""" # List heketi volumes g.log.info("List heketi volumes") volumes = heketi_volume_list(self.heketi_client_node, self.heketi_server_url, json=True) if (len(volumes["volumes"]) == 0): g.log.info("Creating heketi volume") out = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) self.assertTrue(out, ("Failed to create heketi " "volume of size %s" % self.volume_size)) g.log.info("Heketi volume successfully created" % out) volume_id = out["bricks"][0]["volume"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) # List heketi cluster's g.log.info("Listing heketi cluster list") out = heketi_cluster_list(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(out, ("Failed to list heketi cluster")) g.log.info("All heketi cluster successfully listed") cluster_id = out["clusters"][0] # Deleting a heketi cluster g.log.info("Trying to delete a heketi cluster" " which contains volumes and/or nodes:" " Expected to fail") self.assertRaises( ExecutionError, heketi_cluster_delete, self.heketi_client_node, self.heketi_server_url, cluster_id, ) g.log.info("Expected result: Unable to delete cluster %s" " because it contains volumes " " and/or nodes" % cluster_id) # To confirm deletion failed, check heketi cluster list g.log.info("Listing heketi cluster list") out = heketi_cluster_list(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(out, ("Failed to list heketi cluster")) g.log.info("All heketi cluster successfully listed")
def test_delete_heketidb_volume(self): """ Method to test heketidb volume deletion via heketi-cli """ volume_id_list = [] heketidbexists = False msg = "Error: Cannot delete volume containing the Heketi database" for i in range(0, 2): volume_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.assertNotEqual(volume_info, False, "Volume creation failed") volume_id_list.append(volume_info["id"]) self.addCleanup(self.delete_volumes, volume_id_list) volume_list_info = heketi_ops.heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertNotEqual(volume_list_info, False, "Heketi volume list command failed") if volume_list_info["volumes"] == []: raise ExecutionError("Heketi volume list empty") for volume_id in volume_list_info["volumes"]: volume_info = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) if volume_info["name"] == "heketidbstorage": heketidbexists = True delete_ret, delete_output, delete_error = ( heketi_ops.heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, volume_id, raw_cli_output=True)) self.assertNotEqual(delete_ret, 0, "Return code not 0") self.assertEqual( delete_error.strip(), msg, "Invalid reason for heketidb deletion failure") if not heketidbexists: raise ExecutionError( "Warning: heketidbstorage doesn't exist in list of volumes")
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self): """Validate retain policy for glusterfs after deletion of pvc""" self.create_storage_class(reclaim_policy='Retain') self.create_and_wait_for_pvc() # get the name of the volume pv_name = get_pv_name_from_pvc(self.node, self.pvc_name) custom = [ r':.metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi\-volume\-id"', r':.spec.persistentVolumeReclaimPolicy' ] vol_id, reclaim_policy = oc_get_custom_resource( self.node, 'pv', custom, pv_name) self.assertEqual(reclaim_policy, 'Retain') # Create DC with POD and attached PVC to it. try: dc_name = oc_create_app_dc_with_io(self.node, self.pvc_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) finally: scale_dc_pod_amount_and_wait(self.node, dc_name, 0) oc_delete(self.node, 'dc', dc_name) wait_for_resource_absence(self.node, 'pod', pod_name) oc_delete(self.node, 'pvc', self.pvc_name) with self.assertRaises(ExecutionError): wait_for_resource_absence(self.node, 'pvc', self.pvc_name, interval=3, timeout=30) heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol_id) vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertNotIn(vol_id, vol_list) oc_delete(self.node, 'pv', pv_name) wait_for_resource_absence(self.node, 'pv', pv_name)
def test_heketi_metrics_heketipod_failure(self): """Validate heketi metrics after heketi pod failure""" scale_dc_pod_amount_and_wait(self.ocp_master_node, self.heketi_dc_name, pod_amount=0) self.addCleanup(scale_dc_pod_amount_and_wait, self.ocp_master_node, self.heketi_dc_name, pod_amount=1) # verify that metrics is not accessable when heketi pod is down with self.assertRaises(exceptions.ExecutionError): get_heketi_metrics(self.heketi_client_node, self.heketi_server_url, prometheus_format=True) scale_dc_pod_amount_and_wait(self.ocp_master_node, self.heketi_dc_name, pod_amount=1) pod_name = get_pod_name_from_dc(self.ocp_master_node, self.heketi_dc_name, self.heketi_dc_name) wait_for_pod_be_ready(self.ocp_master_node, pod_name, wait_step=5) for i in range(3): vol = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(vol) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol['id'], raise_on_error=False) vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertIn(vol['id'], vol_list) self.verify_heketi_metrics_with_topology_info()
def _heketi_vols(ocp_node, url): # Unfortunately, getting json from heketi-cli only gets the ids # To get a mapping of ids & volume names without a lot of # back and forth between the test and the ocp_node we end up having # to scrape the output of 'volume list' # TODO: This probably should be made into a utility function out = heketi_volume_list(ocp_node, url, json=False) res = [] for line in out.splitlines(): if not line.startswith('Id:'): continue row = {} for section in line.split(): if ':' in section: key, value = section.split(':', 1) row[key.lower()] = value.strip() res.append(row) return res
def test_blockvolume_create_no_free_space(self): """Validate error is returned when free capacity is exhausted""" # Create first small blockvolume blockvol1 = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(blockvol1, "Failed to create block volume.") self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol1['id']) # Get info about block hosting volumes file_volumes = heketi_volume_list(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(file_volumes) self.assertIn("volumes", file_volumes) self.assertTrue(file_volumes["volumes"]) max_block_hosting_vol_size, file_volumes_debug_info = 0, [] for vol_id in file_volumes["volumes"]: vol = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, vol_id, json=True) current_block_hosting_vol_size = vol.get('size', 0) if current_block_hosting_vol_size > max_block_hosting_vol_size: max_block_hosting_vol_size = current_block_hosting_vol_size if current_block_hosting_vol_size: file_volumes_debug_info.append( six.text_type({ 'id': vol.get('id', '?'), 'name': vol.get('name', '?'), 'size': current_block_hosting_vol_size, 'blockinfo': vol.get('blockinfo', '?'), })) self.assertGreater(max_block_hosting_vol_size, 0) # Try to create blockvolume with size bigger than available too_big_vol_size = max_block_hosting_vol_size + 1 try: blockvol2 = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, too_big_vol_size, json=True) except ExecutionError: return if blockvol2 and blockvol2.get('id'): self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol2['id']) block_hosting_vol = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, blockvol2.get('blockhostingvolume'), json=True) self.assertGreater( block_hosting_vol.get('size', -2), blockvol2.get('size', -1), ("Block volume unexpectedly was created. " "Calculated 'max free size' is '%s'.\nBlock volume info is: %s \n" "File volume info, which hosts block volume: \n%s," "Block hosting volumes which were considered: \n%s" % (max_block_hosting_vol_size, blockvol2, block_hosting_vol, '\n'.join(file_volumes_debug_info))))
def _create_distributed_replica_vol(self, validate_cleanup): # Create distributed vol vol_size_gb = self._get_vol_size() heketi_url = self.heketi_server_url try: g.log.info( "Trying to create distributed '%s'Gb volume." % vol_size_gb) heketi_vol = heketi_volume_create( self.heketi_client_node, heketi_url, vol_size_gb, json=True) except exceptions.ExecutionError as e: # NOTE: rare situation when we need to decrease size of a volume. # and we expect this vol to be distributed. g.log.info("Failed to create distributed '%s'Gb volume. " "Trying to create another one, smaller for 1Gb.") if ('more required' in str(e) and ('Insufficient suitable allocatable extents for ' 'logical volume' in str(e))): vol_size_gb -= 1 heketi_vol = heketi_volume_create( self.heketi_client_node, heketi_url, vol_size_gb, json=True) else: raise g.log.info("Successfully created distributed volume.") vol_name = heketi_vol['name'] vol_id = heketi_vol["bricks"][0]["volume"] self.addCleanup( heketi_volume_delete, self.heketi_client_node, heketi_url, vol_id, raise_on_error=(not validate_cleanup)) # Get gluster volume info g.log.info("Get gluster volume '%s' info" % vol_name) gluster_vol = get_volume_info( 'auto_get_gluster_endpoint', volname=vol_name) self.assertTrue( gluster_vol, "Failed to get volume '%s' info" % vol_name) g.log.info("Successfully got volume '%s' info" % vol_name) gluster_vol = gluster_vol[vol_name] self.assertEqual( gluster_vol["typeStr"], "Distributed-Replicate", "'%s' gluster vol isn't a Distributed-Replicate volume" % vol_name) # Check amount of bricks brick_amount = len(gluster_vol['bricks']['brick']) self.assertEqual(brick_amount % 3, 0, "Brick amount is expected to be divisible by 3. " "Actual amount is '%s'" % brick_amount) self.assertGreater(brick_amount, 3, "Brick amount is expected to be bigger than 3. " "Actual amount is '%s'." % brick_amount) # Run unique actions to Validate whether deleting a dist-rep # volume is handled by heketi else return if not validate_cleanup: return # Get the free space after creating heketi volume free_space_after_creating_vol = self._get_free_space() # Delete heketi volume g.log.info("Deleting heketi volume '%s'" % vol_id) volume_deleted = heketi_volume_delete( self.heketi_client_node, heketi_url, vol_id) self.assertTrue( volume_deleted, "Failed to delete heketi volume '%s'" % vol_id) g.log.info("Heketi volume '%s' has successfully been deleted" % vol_id) # Check the heketi volume list g.log.info("List heketi volumes") heketi_volumes = heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(heketi_volumes, "Failed to list heketi volumes") g.log.info("Heketi volumes have successfully been listed") heketi_volumes = heketi_volumes.get('volumes', heketi_volumes) self.assertNotIn(vol_id, heketi_volumes) self.assertNotIn(vol_name, heketi_volumes) # Check the gluster volume list g.log.info("Get the gluster volume list") gluster_volumes = get_volume_list('auto_get_gluster_endpoint') self.assertTrue(gluster_volumes, "Unable to get Gluster volume list") g.log.info("Successfully got Gluster volume list" % gluster_volumes) self.assertNotIn(vol_id, gluster_volumes) self.assertNotIn(vol_name, gluster_volumes) # Get the used space after deleting heketi volume free_space_after_deleting_vol = self._get_free_space() # Compare the free space before and after deleting the volume g.log.info("Comparing the free space before and after deleting volume") self.assertLessEqual( free_space_after_creating_vol + (3 * vol_size_gb), free_space_after_deleting_vol) g.log.info("Volume successfully deleted and space is reallocated. " "Free space after creating volume %s. " "Free space after deleting volume %s." % ( free_space_after_creating_vol, free_space_after_deleting_vol))