def test_try_to_create_sc_with_duplicated_name(self): """Verify SC creation fails with duplicate name""" sc_name = "test-sc-duplicated-name-" + utils.get_random_str() self.create_storage_class(sc_name=sc_name) self.create_and_wait_for_pvc(sc_name=sc_name) with self.assertRaises(AssertionError): self.create_storage_class(sc_name=sc_name)
def setUp(self): """Deploys, Verifies and adds resources required for testcases in cleanup method """ super(GlusterStabilityTestSetup, self).setUp() self.oc_node = self.ocp_master_node[0] self.prefix = "autotest-%s" % utils.get_random_str()
def test_100gb_block_pvc_create_and_delete_twice(self): """Validate creation and deletion of blockvoume of size 100GB""" # Define required space, bhv size required for on 100GB block PVC size, bhv_size, required_space = 100, 103, 309 h_node, h_url = self.heketi_client_node, self.heketi_server_url prefix = 'autotest-pvc-{}'.format(utils.get_random_str(size=5)) # Skip test if required free space is not available free_space = get_total_free_space(self.heketi_client_node, self.heketi_server_url)[0] if free_space < required_space: self.skipTest("Available free space {} is less than the required " "free space {}".format(free_space, required_space)) # Create block hosting volume of 103GB required for 100GB block PVC bhv = heketi_volume_create(h_node, h_url, bhv_size, block=True, json=True)['id'] self.addCleanup(heketi_volume_delete, h_node, h_url, bhv) for _ in range(2): # Create PVC of size 100GB pvc_name = self.create_and_wait_for_pvc(pvc_size=size, pvc_name_prefix=prefix) match_pvc_and_pv(self.node, prefix) # Delete the PVC oc_delete(self.node, 'pvc', pvc_name) wait_for_resource_absence(self.node, 'pvc', pvc_name)
def test_arbiter_volume_delete_using_pvc(self): """Test Arbiter volume delete using pvc when volume is not mounted on app pod """ prefix = "autotest-%s" % utils.get_random_str() # Create sc with gluster arbiter info sc_name = self.create_storage_class(vol_name_prefix=prefix, is_arbiter_vol=True) # Create PVC and wait for it to be in 'Bound' state pvc_name = self.create_and_wait_for_pvc(pvc_name_prefix=prefix, sc_name=sc_name) # Get vol info gluster_vol_info = openshift_ops.get_gluster_vol_info_by_pvc_name( self.node, pvc_name) # Verify arbiter volume properties self.verify_amount_and_proportion_of_arbiter_and_data_bricks( gluster_vol_info) # Get volume ID gluster_vol_id = gluster_vol_info["gluster_vol_id"] # Delete the pvc openshift_ops.oc_delete(self.node, 'pvc', pvc_name) openshift_ops.wait_for_resource_absence(self.node, 'pvc', pvc_name) # Check the heketi volume list if pvc is deleted g.log.info("List heketi volumes") heketi_volumes = heketi_ops.heketi_volume_list(self.heketi_client_node, self.heketi_server_url) err_msg = "Failed to delete heketi volume by prefix %s" % prefix self.assertNotIn(prefix, heketi_volumes, err_msg) # Check presence for the gluster volume get_gluster_vol_info = volume_ops.get_volume_info( "auto_get_gluster_endpoint", gluster_vol_id) err_msg = "Failed to delete gluster volume %s" % gluster_vol_id self.assertFalse(get_gluster_vol_info, err_msg) # Check presence of bricks and lvs for brick in gluster_vol_info['bricks']['brick']: gluster_node_ip, brick_name = brick["name"].split(":") with self.assertRaises(exceptions.ExecutionError): cmd = "df %s" % brick_name openshift_ops.cmd_run_on_gluster_pod_or_node( self.node, cmd, gluster_node_ip) with self.assertRaises(exceptions.ExecutionError): lv_match = re.search(BRICK_REGEX, brick["name"]) if lv_match: cmd = "lvs %s" % lv_match.group(2).strip() openshift_ops.cmd_run_on_gluster_pod_or_node( self.node, cmd, gluster_node_ip)
def setUp(self): """Deploys, Verifies and adds resources required for testcases in cleanup method """ self.oc_node = self.ocp_master_node[0] self.prefix = "autotest-%s" % utils.get_random_str() _storage_class = self.storage_classes.get( 'storage_class2', self.storage_classes.get('block_storage_class')) self.provisioner = _storage_class["provisioner"] self.restsecretnamespace = _storage_class["restsecretnamespace"] self.restuser = _storage_class["restuser"] self.resturl = _storage_class["resturl"] # using pvc size count as 1 by default self.pvcsize = 1 # using pvc count as 10 by default self.pvccount = 10 # create gluster block storage class, PVC and user app pod self.sc_name, self.pvc_name, self.dc_name, self.secret_name = ( self.deploy_resouces()) # verify storage class oc_get_yaml(self.oc_node, "sc", self.sc_name) # verify pod creation, it's state and get the pod name self.pod_name = get_pod_name_from_dc(self.oc_node, self.dc_name, timeout=180, wait_step=3) wait_for_pod_be_ready(self.oc_node, self.pod_name, timeout=180, wait_step=3) verify_pvc_status_is_bound(self.oc_node, self.pvc_name) # create pvc's to test self.pvc_list = [] for pvc in range(self.pvccount): test_pvc_name = oc_create_pvc(self.oc_node, self.sc_name, pvc_name_prefix=self.prefix, pvc_size=self.pvcsize) self.pvc_list.append(test_pvc_name) self.addCleanup(wait_for_resource_absence, self.oc_node, "pvc", test_pvc_name, timeout=600, interval=10) for pvc_name in self.pvc_list: self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
def test_dynamic_provisioning_block_vol_with_custom_prefix(self): """Verify creation of block volume with custom prefix """ node = self.ocp_master_node[0] prefix = "autotest-{}".format(utils.get_random_str()) # cmd to get available space cmd_get_free_space = "df -h | grep '/mnt'| awk '{{print $4}}'" # cmd to create a 100M file cmd_run_io = 'dd if=/dev/zero of=/mnt/testfile bs=1024 count=102400' # Create sc with prefix sc_name = self.create_storage_class(sc_name_prefix=prefix, create_vol_name_prefix=True, vol_name_prefix=prefix) # Create pvc and wait for it to be in bound state pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name, pvc_size=1) # Verify blockvolume list with prefix h_block_vol = heketi_blockvolume_list_by_name_prefix( self.heketi_client_node, self.heketi_server_url, prefix) self.assertIsNotNone( h_block_vol, "Failed to find blockvolume with prefix {}".format(prefix)) self.assertTrue( h_block_vol[0][2].startswith(prefix), "Failed to create blockvolume with the prefix {}".format(prefix)) # Create app pod dc_name, pod_name = self.create_dc_with_pvc(pvc_name) err_msg = ("Failed to get the free space for the mount point of the " "app pod {} with error {}") # Get free space of app pod before IO run _, free_space_before, err = oc_rsh(node, pod_name, cmd_get_free_space) self.assertTrue(free_space_before, err_msg.format(pod_name, err)) # Running IO on the app pod ret, _, err = oc_rsh(node, pod_name, cmd_run_io) self.assertFalse( ret, "Failed to run the Io with the error msg {}".format(err)) # Get free space of app pod after IO run _, free_space_after, err = oc_rsh(node, pod_name, cmd_get_free_space) self.assertTrue(free_space_after, err_msg.format(pod_name, err)) self.assertGreaterEqual( free_space_before, free_space_after, "Expecting free space in app pod before {} should be greater than" " {} as 100M file is created".format(free_space_before, free_space_after))
def setUp(self): """Deploys, Verifies and adds resources required for testcases in cleanup method """ self.oc_node = self.ocp_master_node[0] self.prefix = "autotest-%s" % utils.get_random_str() _storage_class = self.storage_classes.get( 'storage_class2', self.storage_classes.get('block_storage_class')) self.provisioner = _storage_class["provisioner"] self.restsecretnamespace = _storage_class["restsecretnamespace"] self.restuser = _storage_class["restuser"] self.resturl = _storage_class["resturl"] # using pvc size count as 1 by default self.pvcsize = 1 # using pvc count as 10 by default self.pvccount = 10 # create gluster block storage class, PVC and user app pod self.sc_name, self.pvc_name, self.dc_name, self.secret_name = ( self.deploy_resouces() ) # verify storage class oc_get_yaml(self.oc_node, "sc", self.sc_name) # verify pod creation, it's state and get the pod name self.pod_name = get_pod_name_from_dc( self.oc_node, self.dc_name, timeout=180, wait_step=3 ) wait_for_pod_be_ready( self.oc_node, self.pod_name, timeout=180, wait_step=3 ) verify_pvc_status_is_bound(self.oc_node, self.pvc_name) # create pvc's to test self.pvc_list = [] for pvc in range(self.pvccount): test_pvc_name = oc_create_pvc( self.oc_node, self.sc_name, pvc_name_prefix=self.prefix, pvc_size=self.pvcsize ) self.pvc_list.append(test_pvc_name) self.addCleanup( wait_for_resource_absence, self.oc_node, "pvc", test_pvc_name, timeout=600, interval=10 ) for pvc_name in self.pvc_list: self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
def test_heketi_node_add_with_invalid_cluster(self): """Test heketi node add operation with invalid cluster id""" storage_hostname, cluster_id = None, utils.get_random_str(size=33) try: storage_hostname, _ = self.add_heketi_node_to_cluster(cluster_id) except AssertionError as e: err_msg = ("validation failed: cluster: %s is not a valid UUID" % cluster_id) if err_msg not in six.text_type(e): raise err_msg = ("Unexpectedly node %s got added to cluster %s" % ( storage_hostname, cluster_id)) self.assertFalse(storage_hostname, err_msg)
def test_block_provisioner_on_multiple_clusters(self): """Check block provisioner and verify on multiple clusters """ # Skip test if registry project is not present self.registry_sc = self.storage_classes.get( 'registry_block_storage_class') if not self.registry_sc: self.skipTest("Config file doesn't have key " "openshift.dynamic_provisioning.storage_classes") self._registry_heketi_server_url = self.registry_sc.get('resturl') self._registry_project_name = self.registry_sc.get( 'restsecretnamespace') if not (self._registry_heketi_server_url and self._registry_project_name): self.skipTest( "Config file doesn't have key" "'storage_classes.registry_block_storage_class.resturl' or " "'storage_classes.registry_block_storage_class" ".restsecretnamespace'") size = 1 prefix = 'autotest-pvc-{}'.format(utils.get_random_str(size=5)) # Create PVC in default namespace and verify multipath pvc_name = self.create_and_wait_for_pvc(pvc_size=size, pvc_name_prefix=prefix) _, pod_name = self.create_dc_with_pvc(pvc_name) match_pvc_and_pv(self.node, prefix) self.verify_iscsi_sessions_and_multipath(pvc_name, pod_name, rtype='pod') # Change project namespace self.addCleanup(switch_oc_project, self.node, self.storage_project_name) switch_oc_project(self.node, self._registry_project_name) # Create PVC in registry namespace and verify multipath self.sc_name = self.create_storage_class(glusterfs_registry=True) pvc_name = self.create_and_wait_for_pvc(sc_name=self.sc_name, pvc_size=size, pvc_name_prefix=prefix) _, pod_name = self.create_dc_with_pvc(pvc_name) self.verify_iscsi_sessions_and_multipath( pvc_name, pod_name, rtype='pod', heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True)
def test_heketi_volume_mount(self): self.node = self.ocp_master_node[0] try: cmd_run('rpm -q glusterfs-fuse', self.node) except AssertionError: self.skipTest("gluster-fuse package is not present on Node %s" % self.node) h_node, h_url = self.heketi_client_node, self.heketi_server_url # Create volume vol_info = heketi_volume_create(h_node, h_url, 2, json=True) self.addCleanup(heketi_volume_delete, h_node, h_url, vol_info['id']) mount_point = vol_info['mount']['glusterfs']['device'] mount_dir = '/mnt/dir-%s' % utils.get_random_str() mount_cmd = 'mount -t glusterfs %s %s' % (mount_point, mount_dir) # Create directory to mount volume cmd_run('mkdir %s' % mount_dir, self.node) self.addCleanup(cmd_run, 'rm -rf %s' % mount_dir, self.node) # Mount volume cmd_run(mount_cmd, self.node) self.addCleanup(cmd_run, 'umount %s' % mount_dir, self.node) # Run I/O to make sure Mount point works _file = 'file' run_io_cmd = ('dd if=/dev/urandom of=%s/%s bs=4k count=1000' % (mount_dir, _file)) # Verify size of volume cmd_run(run_io_cmd, self.node) size = cmd_run('df -kh --output=size %s | tail -1' % mount_dir, self.node).strip() self.assertEqual('2.0G', size) # Verify file on gluster vol bricks for brick in vol_info['bricks']: node_id = brick['node'] node_info = heketi_node_info(h_node, h_url, node_id, json=True) brick_host = node_info['hostnames']['storage'][0] cmd_run_on_gluster_pod_or_node(self.node, 'ls %s/%s' % (brick['path'], _file), brick_host)
def test_heketi_volume_create_mutiple_sizes(self): """Validate creation of heketi volume with differnt sizes""" prefix = "autotest-{}".format(utils.get_random_str()) sizes, required_space = [15, 50, 100], 495 h_node, h_url = self.heketi_client_node, self.heketi_server_url # Skip test if space is not available available_space = get_total_free_space(h_node, h_url)[0] if required_space > available_space: self.skipTest("Required space {} greater than the available space " "{}".format(required_space, available_space)) # Create volume 3 times, each time different size for size in sizes: self.create_heketi_volume_with_name_and_wait("{}-{}".format( prefix, size), size, json=True)
def test_block_host_volume_delete_block_volume_delete(self): """Validate block volume and BHV removal using heketi""" free_space, nodenum = get_total_free_space(self.heketi_client_node, self.heketi_server_url) if nodenum < 3: self.skipTest("Skipping the test since number of nodes" "online are less than 3") free_space_available = int(free_space / nodenum) default_bhv_size = get_default_block_hosting_volume_size( self.heketi_client_node, self.heketi_dc_name) if free_space_available < default_bhv_size: self.skipTest("Skipping the test since free_space_available %s" "is less than the default_bhv_size %s" % (free_space_available, default_bhv_size)) h_volume_name = ("autotests-heketi-volume-%s" % utils.get_random_str()) block_host_create_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, default_bhv_size, json=True, block=True) block_vol_size = block_host_create_info["blockinfo"]["freesize"] block_hosting_vol_id = block_host_create_info["id"] block_vol_info = {"blockhostingvolume": "init_value"} while (block_vol_info['blockhostingvolume'] != block_hosting_vol_id): block_vol = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, block_vol_size, json=True, ha=3, auth=True) self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, block_vol["id"], raise_on_error=True) block_vol_info = heketi_blockvolume_info(self.heketi_client_node, self.heketi_server_url, block_vol["id"], json=True) bhv_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, block_hosting_vol_id, json=True) self.assertIn(block_vol_info["id"], bhv_info["blockinfo"]["blockvolume"])
def test_block_volume_create_with_name(self): """Validate creation of block volume with name""" vol_name = "autotests-heketi-volume-%s" % utils.get_random_str() block_vol = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, 1, name=vol_name, json=True) self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, block_vol["id"]) # check volume name through heketi-cli block_vol_info = heketi_blockvolume_info(self.heketi_client_node, self.heketi_server_url, block_vol["id"], json=True) self.assertEqual(block_vol_info["name"], vol_name, ("Block volume Names are not same %s as %s", (block_vol_info["name"], vol_name)))
def test_replica_volume_expand(self): """ Test expansion of a replica volume """ h_node, h_server = self.heketi_client_node, self.heketi_server_url volume_name = ("autotests-heketi-volume-{}".format( utils.get_random_str())) volume_size = 10 creation_info = self.create_heketi_volume_with_name_and_wait( volume_name, volume_size, json=True, raise_on_cleanup_error=False) volume_id = creation_info["id"] volume_info = heketi_ops.heketi_volume_info(h_node, h_server, volume_id, json=True) # Get gluster volume info gluster_vol = volume_ops.get_volume_info('auto_get_gluster_endpoint', volname=volume_name) self.assertTrue(gluster_vol, "Failed to get volume {} info".format(volume_name)) vol_name = gluster_vol[volume_name] self.assertEqual( vol_name['replicaCount'], "3", "Replica count is different for volume {} Actual:{} " "Expected : 3".format(vol_name, vol_name['replicaCount'])) expand_size = 5 heketi_ops.heketi_volume_expand(h_node, h_server, volume_id, expand_size) volume_info = heketi_ops.heketi_volume_info(h_node, h_server, volume_id, json=True) expected_size = volume_size + expand_size self.assertEqual( volume_info['size'], expected_size, "Volume Expansion failed, Expected Size: {}, Actual " "Size: {}".format(str(expected_size), str(volume_info['size']))) self.get_brick_and_volume_status(volume_name) self.get_rebalance_status(volume_name)
def test_create_volume_with_same_name(self): """Test create two volumes with the same name and verify that 2nd one is failing with the appropriate error. """ h_node, h_url = self.heketi_client_node, self.heketi_server_url vol_name = "autovol-%s" % utils.get_random_str() # Create volume for the first time vol_info = heketi_ops.heketi_volume_create(h_node, h_url, size=1, name=vol_name, json=True) self.addCleanup(heketi_ops.heketi_volume_delete, h_node, h_url, vol_info['id']) vol_info_new = None try: # Try to create volume for 2nd time vol_info_new = heketi_ops.heketi_volume_create(h_node, h_url, size=1, name=vol_name, json=True) self.addCleanup(heketi_ops.heketi_volume_delete, h_node, h_url, vol_info_new['id']) except AssertionError as err: # Verify msg in error msg = "Volume name '%s' already in use" % vol_name if msg not in six.text_type(err): raise # Raise exception if there is no error raised by heketi msg = ('Volume %s and %s got created two times with the same name ' 'unexpectedly.' % (vol_info, vol_info_new)) self.assertFalse(vol_info_new, msg)
def test_heketi_manual_cleanup_operation_in_bhv(self): """Validate heketi db cleanup will resolve the mismatch in the free size of the block hosting volume with failed block device create operations. """ bhv_size_before, bhv_size_after, vol_count = [], [], 5 ocp_node, g_node = self.ocp_master_node[0], self.gluster_servers[0] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Get nodes id list node_id_list = heketi_node_list(h_node, h_url) # Disable 4th and other nodes for node_id in node_id_list[3:]: heketi_node_disable(h_node, h_url, node_id) self.addCleanup(heketi_node_enable, h_node, h_url, node_id) # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 50: h_volume_size = 50 else: h_volume_size, vol_count = 1, free_space_available # Create BHV in case blockvolume size is greater than default BHV size default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Get BHV list h_bhv_list = get_block_hosting_volume_list(h_node, h_url).keys() self.assertTrue(h_bhv_list, "Failed to get the BHV list") # Get BHV size for bhv in h_bhv_list: vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bhv_vol_size_before = vol_info.get("freesize") bhv_size_before.append(bhv_vol_size_before) # Kill Tcmu-runner service services = ("tcmu-runner", "gluster-block-target", "gluster-blockd") kill_service_on_gluster_pod_or_node(ocp_node, "tcmu-runner", g_node) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') self.addCleanup(wait_for_service_status_on_gluster_pod_or_node, ocp_node, service, 'active', state, g_node) self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node, service, g_node) def run_async(cmd, hostname, raise_on_error=True): return g.run_async(host=hostname, command=cmd) # Create stale block volumes in async for count in range(vol_count): with mock.patch.object(json, 'loads', side_effect=(lambda j: j)): with mock.patch.object(command, 'cmd_run', side_effect=run_async): heketi_blockvolume_create(h_node, h_url, h_volume_size, json=True) # Wait for pending operation to get generated self._check_for_pending_operations(h_node, h_url) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') restart_service_on_gluster_pod_or_node(ocp_node, service, g_node) wait_for_service_status_on_gluster_pod_or_node( ocp_node, service, 'active', state, g_node) # Cleanup pending operation heketi_server_operation_cleanup(h_node, h_url) # wait for pending operation to get cleaned up for w in waiter.Waiter(timeout=120, interval=10): # Get BHV size for bhv in h_bhv_list: vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bhv_vol_size_after = vol_info.get("freesize") bhv_size_after.append(bhv_vol_size_after) if (set(bhv_size_before) == set(bhv_size_after)): break if w.expired: raise exceptions.ExecutionError( "Failed to Validate volume size Actual:{}," " Expected:{}".format(set(bhv_size_before), set(bhv_size_after)))
def test_verify_create_heketi_volumes_pending_entries_in_db( self, vol_type): """Verify pending entries of file/block volumes in db during volumes creation from heketi side """ # Create large volumes to observe the pending operations vol_count, h_vol_creation_async_op = 3, [] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Verify file/block volumes pending operation before creation, h_db_check_before = heketi_db_check(h_node, h_url) h_db_check_vol_before = (h_db_check_before.get( "{}volumes".format(vol_type))) # Delete heketi pod to clean db operations if (h_db_check_vol_before.get("pending") or h_db_check_before.get("bricks").get("pending")): self._respin_heketi_pod() # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 30: h_volume_size = 30 else: h_volume_size, vol_count = 1, free_space_available # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add cleanup function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Create BHV in case blockvolume size is greater than default BHV size if vol_type: default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Temporary replace g.run with g.async_run in heketi_blockvolume_create # func to be able to run it in background.Also, avoid parsing the # output as it won't be json at that moment. Parse it after reading # the async operation results. def run_async(cmd, hostname, raise_on_error=True): return g.run_async(host=hostname, command=cmd) for count in range(vol_count): with mock.patch.object(json, 'loads', side_effect=(lambda j: j)): with mock.patch.object(command, 'cmd_run', side_effect=run_async): h_vol_creation_async_op.append( eval("heketi_{}volume_create".format(vol_type))( h_node, h_url, h_volume_size, json=True)) # Check for pending operations for w in waiter.Waiter(timeout=120, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if h_db_check_vol.get("pending"): h_db_check_bricks = h_db_check.get("bricks") break if w.expired: raise exceptions.ExecutionError( "No any pending operations found during {}volumes creation " "{}".format(vol_type, h_db_check_vol.get("pending"))) # Verify bricks pending operation during creation if not vol_type: self.assertTrue(h_db_check_bricks.get("pending"), "Expecting at least one bricks pending count") self.assertFalse( h_db_check_bricks.get("pending") % 3, "Expecting bricks pending count to be multiple of 3 but " "found {}".format(h_db_check_bricks.get("pending"))) # Wait for all counts of pending operations to be zero for w in waiter.Waiter(timeout=300, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if not h_db_check_vol.get("pending"): break if w.expired: raise exceptions.ExecutionError( "Expecting no pending operations after 300 sec but " "found {} operation".format(h_db_check_vol.get("pending"))) # Get heketi server DB details h_db_check_after = heketi_db_check(h_node, h_url) h_db_check_vol_after = (h_db_check_after.get( "{}volumes".format(vol_type))) h_db_check_bricks_after = h_db_check_after.get("bricks") # Verify if initial and final file/block volumes are same act_vol_count = h_db_check_vol_after.get("total") exp_vol_count = h_db_check_vol_before.get("total") + vol_count err_msg = ( "Actual {} and expected {} {}volume counts are not matched".format( act_vol_count, exp_vol_count, vol_type)) self.assertEqual(act_vol_count, exp_vol_count, err_msg) # Verify if initial and final bricks are same for file volume volumes = heketi_volume_list(h_node, h_url, json=True).get("volumes") new_volumes = list(set(volumes) - set(existing_volumes)) exp_brick_count = 0 for volume in new_volumes: vol_info = heketi_volume_info(h_node, h_url, volume, json=True) exp_brick_count += len(vol_info.get("bricks")) err_msg = "Actual {} and expected {} bricks counts are not matched" act_brick_count = h_db_check_bricks_after.get("total") self.assertEqual(act_brick_count, exp_brick_count, err_msg.format(act_brick_count, exp_brick_count))
def test_verify_delete_heketi_volumes_pending_entries_in_db( self, vol_type): """Verify pending entries of blockvolumes/volumes and bricks in heketi db during blockvolume/volume delete operation. """ # Create a large volumes to observe the pending operation vol_count, volume_ids, async_obj = 10, [], [] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Verify file/block volumes pending operation before creation, h_db_check_before = heketi_db_check(h_node, h_url) h_db_check_bricks_before = h_db_check_before.get("bricks") h_db_check_vol_before = (h_db_check_before.get( "{}volumes".format(vol_type))) # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add cleanup function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Delete heketi pod to clean db operations if (h_db_check_bricks_before.get("pending") or h_db_check_vol_before.get("pending")): self._respin_heketi_pod() # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 50: h_volume_size = 50 else: h_volume_size, vol_count = 1, free_space_available # Create BHV in case blockvolume size is greater than default BHV size if vol_type: default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Create file/block volumes for _ in range(vol_count): vol_id = eval("heketi_{}volume_create".format(vol_type))( h_node, h_url, h_volume_size, json=True).get("id") volume_ids.append(vol_id) self.addCleanup(eval("heketi_{}volume_delete".format(vol_type)), h_node, h_url, vol_id, raise_on_error=False) def run_async(cmd, hostname, raise_on_error=True): async_op = g.run_async(host=hostname, command=cmd) async_obj.append(async_op) return async_op bhv_list = [] for vol_id in volume_ids: # Get BHV ids to delete in case of block volumes if vol_type: vol_info = (heketi_blockvolume_info(h_node, h_url, vol_id, json=True)) if not vol_info.get("blockhostingvolume") in bhv_list: bhv_list.append(vol_info.get("blockhostingvolume")) # Temporary replace g.run with g.async_run in heketi_volume_delete # and heketi_blockvolume_delete func to be able to run it in # background. with mock.patch.object(command, 'cmd_run', side_effect=run_async): eval("heketi_{}volume_delete".format(vol_type))(h_node, h_url, vol_id) # Wait for pending operations to get generate for w in waiter.Waiter(timeout=30, interval=3): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if h_db_check_vol.get("pending"): h_db_check_bricks = h_db_check.get("bricks") break if w.expired: raise exceptions.ExecutionError( "No any pending operations found during {}volumes deletion " "{}".format(vol_type, h_db_check_vol.get("pending"))) # Verify bricks pending operation during creation if not vol_type: self.assertTrue(h_db_check_bricks.get("pending"), "Expecting at least one bricks pending count") self.assertFalse( h_db_check_bricks.get("pending") % 3, "Expecting bricks pending count to be multiple of 3 but " "found {}".format(h_db_check_bricks.get("pending"))) # Verify file/block volume pending operation during delete for w in waiter.Waiter(timeout=120, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) h_db_check_bricks = h_db_check.get("bricks") if ((not h_db_check_bricks.get("pending")) and (not h_db_check_vol.get("pending"))): break if w.expired: raise AssertionError( "Failed to delete {}volumes after 120 secs".format(vol_type)) # Check that all background processes got exited for obj in async_obj: ret, out, err = obj.async_communicate() self.assertFalse( ret, "Failed to delete {}volume due to error: {}".format( vol_type, err)) # Delete BHV created during block volume creation if vol_type: for bhv_id in bhv_list: heketi_volume_delete(h_node, h_url, bhv_id) # Verify bricks and volume pending operations h_db_check_after = heketi_db_check(h_node, h_url) h_db_check_bricks_after = h_db_check_after.get("bricks") h_db_check_vol_after = (h_db_check_after.get( "{}volumes".format(vol_type))) act_brick_count = h_db_check_bricks_after.get("pending") act_vol_count = h_db_check_vol_after.get("pending") # Verify bricks pending operation after delete err_msg = "{} operations are pending for {} after {}volume deletion" if not vol_type: self.assertFalse( act_brick_count, err_msg.format(act_brick_count, "brick", vol_type)) # Verify file/bock volumes pending operation after delete self.assertFalse(act_vol_count, err_msg.format(act_vol_count, "volume", vol_type)) act_brick_count = h_db_check_bricks_after.get("total") act_vol_count = h_db_check_vol_after.get("total") exp_brick_count = h_db_check_bricks_before.get("total") exp_vol_count = h_db_check_vol_before.get("total") err_msg = "Actual {} and expected {} {} counts are not matched" # Verify if initial and final file/block volumes are same self.assertEqual( act_vol_count, exp_vol_count, err_msg.format(act_vol_count, exp_vol_count, "volume")) # Verify if initial and final bricks are same self.assertEqual( act_brick_count, exp_brick_count, err_msg.format(act_brick_count, exp_brick_count, "brick"))
def setUp(self): super(TestHeketiZones, self).setUp() self.node = self.ocp_master_node[0] self.h_client = self.heketi_client_node self.h_server = self.heketi_server_url self.prefix = "autotests-{}".format(utils.get_random_str())
def _create_distributed_replica_vol(self, validate_cleanup, block=False): # Create distributed vol vol_size_gb = self._get_vol_size() heketi_url = self.heketi_server_url h_volume_name = "autotests-heketi-volume-%s" % utils.get_random_str() try: heketi_vol = self.create_heketi_volume_with_name_and_wait( h_volume_name, vol_size_gb, json=True, raise_on_cleanup_error=False, block=block) except AssertionError as e: # NOTE: rare situation when we need to decrease size of a volume. # and we expect this vol to be distributed. g.log.info("Failed to create distributed '%s'Gb volume. " "Trying to create another one, smaller for 1Gb.") if not ('more required' in str(e) and ('Insufficient suitable allocatable extents for ' 'logical volume' in str(e))): raise vol_size_gb -= 1 heketi_vol = self.create_heketi_volume_with_name_and_wait( h_volume_name, vol_size_gb, json=True, raise_on_cleanup_error=False, block=block) g.log.info("Successfully created distributed volume.") vol_name = heketi_vol['name'] vol_id = heketi_vol["bricks"][0]["volume"] # Get gluster volume info g.log.info("Get gluster volume '%s' info" % vol_name) gluster_vol = get_volume_info('auto_get_gluster_endpoint', volname=vol_name) self.assertTrue(gluster_vol, "Failed to get volume '%s' info" % vol_name) g.log.info("Successfully got volume '%s' info" % vol_name) gluster_vol = gluster_vol[vol_name] self.assertEqual( gluster_vol["typeStr"], "Distributed-Replicate", "'%s' gluster vol isn't a Distributed-Replicate volume" % vol_name) # Check amount of bricks brick_amount = len(gluster_vol['bricks']['brick']) self.assertEqual( brick_amount % 3, 0, "Brick amount is expected to be divisible by 3. " "Actual amount is '%s'" % brick_amount) self.assertGreater( brick_amount, 3, "Brick amount is expected to be bigger than 3. " "Actual amount is '%s'." % brick_amount) # Run unique actions to Validate whether deleting a dist-rep # volume is handled by heketi else return if not validate_cleanup: return vol_id # Get the free space after creating heketi volume free_space_after_creating_vol = self._get_free_space() # Delete heketi volume g.log.info("Deleting heketi volume '%s'" % vol_id) volume_deleted = heketi_volume_delete(self.heketi_client_node, heketi_url, vol_id) self.assertTrue(volume_deleted, "Failed to delete heketi volume '%s'" % vol_id) g.log.info("Heketi volume '%s' has successfully been deleted" % vol_id) # Check the heketi volume list g.log.info("List heketi volumes") heketi_volumes = heketi_volume_list(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(heketi_volumes, "Failed to list heketi volumes") g.log.info("Heketi volumes have successfully been listed") heketi_volumes = heketi_volumes.get('volumes', heketi_volumes) self.assertNotIn(vol_id, heketi_volumes) self.assertNotIn(vol_name, heketi_volumes) # Check the gluster volume list g.log.info("Get the gluster volume list") gluster_volumes = get_volume_list('auto_get_gluster_endpoint') self.assertTrue(gluster_volumes, "Unable to get Gluster volume list") g.log.info("Successfully got Gluster volume list" % gluster_volumes) self.assertNotIn(vol_id, gluster_volumes) self.assertNotIn(vol_name, gluster_volumes) # Get the used space after deleting heketi volume free_space_after_deleting_vol = self._get_free_space() # Compare the free space before and after deleting the volume g.log.info("Comparing the free space before and after deleting volume") self.assertLessEqual(free_space_after_creating_vol + (3 * vol_size_gb), free_space_after_deleting_vol) g.log.info( "Volume successfully deleted and space is reallocated. " "Free space after creating volume %s. " "Free space after deleting volume %s." % (free_space_after_creating_vol, free_space_after_deleting_vol))
def test_create_max_num_blockhostingvolumes(self): """Create blockvolumes to test max no of blockvolumes in blockhosting volume """ prefix = "autotest-{}".format(utils.get_random_str()) num_of_bv = 10 new_bhv_list, bv_list, g_nodes = [], [], [] free_space, nodenum = get_total_free_space(self.heketi_client_node, self.heketi_server_url) if nodenum < 3: self.skipTest("Skip the test case since number of" "online nodes is less than 3.") free_space_available = int(free_space / nodenum) default_bhv_size = get_default_block_hosting_volume_size( self.heketi_client_node, self.heketi_dc_name) # Get existing list of BHV's existing_bhv_list = get_block_hosting_volume_list( self.heketi_client_node, self.heketi_server_url) # Skip the test if available space is less than default_bhv_size if free_space_available < default_bhv_size: self.skipTest("Skip the test case since free_space_available %s" "is less than space_required_for_bhv %s ." % (free_space_available, default_bhv_size)) # Create BHV's count = 1 while free_space_available > default_bhv_size: bhv_name = "{}-bhv-{}".format(prefix, count) bhv_info = self.create_heketi_volume_with_name_and_wait( bhv_name, default_bhv_size, raise_on_cleanup_error=False, json=True, block=True) if bhv_info["id"] not in existing_bhv_list.keys(): new_bhv_list.append(bhv_info["id"]) free_size = bhv_info["blockinfo"]["freesize"] if free_size > num_of_bv: block_vol_size = int(free_size / num_of_bv) else: block_vol_size, num_of_bv = 1, free_size # Create specified number of BV's in BHV's created for i in range(0, num_of_bv): block_vol = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, block_vol_size, json=True, ha=3, auth=True) self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, block_vol["id"], raise_on_error=False) bv_list.append(block_vol["id"]) free_space_available = int(free_space_available - default_bhv_size) count += 1 # Get gluster node ips h_nodes_ids = heketi_node_list(self.heketi_client_node, self.heketi_server_url) for h_node in h_nodes_ids[:2]: g_node = heketi_node_info(self.heketi_client_node, self.heketi_server_url, h_node, json=True) g_nodes.append(g_node['hostnames']['manage'][0]) # Check if there is no crash in gluster related services & heketi services = (("glusterd", "running"), ("gluster-blockd", "running"), ("tcmu-runner", "running"), ("gluster-block-target", "exited")) for g_node in g_nodes: for service, state in services: wait_for_service_status_on_gluster_pod_or_node( self.ocp_client[0], service, 'active', state, g_node, raise_on_error=False) out = hello_heketi(self.heketi_client_node, self.heketi_server_url) self.assertTrue( out, "Heketi server %s is not alive" % self.heketi_server_url) # Delete all the BHV's and BV's created for bv_volume in bv_list: heketi_blockvolume_delete(self.heketi_client_node, self.heketi_server_url, bv_volume) # Check if any blockvolume exist in heketi & gluster for bhv_volume in new_bhv_list[:]: heketi_vol_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, bhv_volume, json=True) self.assertNotIn("blockvolume", heketi_vol_info["blockinfo"].keys()) gluster_vol_info = get_block_list('auto_get_gluster_endpoint', volname="vol_%s" % bhv_volume) self.assertIsNotNone(gluster_vol_info, "Failed to get volume info %s" % bhv_volume) new_bhv_list.remove(bhv_volume) for blockvol in gluster_vol_info: self.assertNotIn("blockvol_", blockvol) heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, bhv_volume) # Check if all blockhosting volumes are deleted from heketi self.assertFalse(new_bhv_list)
def test_prometheus_volume_metrics_on_node_reboot(self): """Validate volume metrics using prometheus before and after node reboot""" # Pod name for the entire test prefix = "autotest-{}".format(utils.get_random_str()) # Create I/O pod with PVC pvc_name = self.create_and_wait_for_pvc() pod_name = openshift_ops.oc_create_tiny_pod_with_volume( self._master, pvc_name, prefix, image=self.io_container_image_cirros) self.addCleanup(openshift_ops.oc_delete, self._master, 'pod', pod_name, raise_on_absence=False) openshift_ops.wait_for_pod_be_ready( self._master, pod_name, timeout=60, wait_step=5) # Write data on the volume and wait for 2 mins and sleep is must for # prometheus to get the exact values of the metrics ret, _, err = openshift_ops.oc_rsh( self._master, pod_name, "touch /mnt/file{1..1000}") self.assertEqual( ret, 0, "Failed to create files in the app pod " "with {}".format(err)) time.sleep(120) # Fetch the metrics and store in initial_metrics as dictionary initial_metrics = self._get_and_manipulate_metric_data( self.metrics, pvc_name) openshift_ops.switch_oc_project( self._master, self.storage_project_name) # Get the hostname to reboot where the pod is running pod_info = openshift_ops.oc_get_pods(self._master, name=pod_name) node_for_reboot = pod_info[pod_name]['node'] # Get the vm name by the hostname vm_name = node_ops.find_vm_name_by_ip_or_hostname(node_for_reboot) # power off and on the vm, based on the vm type(either gluster or not) if node_for_reboot in self.gluster_servers: self.power_off_gluster_node_vm(vm_name, node_for_reboot) self.power_on_gluster_node_vm(vm_name, node_for_reboot) else: self.power_off_vm(vm_name) self.power_on_vm(vm_name) openshift_ops.wait_for_ocp_node_be_ready( self._master, node_for_reboot) # Create the new pod and validate the prometheus metrics pod_name = openshift_ops.oc_create_tiny_pod_with_volume( self._master, pvc_name, prefix) self.addCleanup(openshift_ops.oc_delete, self._master, 'pod', pod_name) # Wait for POD be up and running and prometheus to refresh the data openshift_ops.wait_for_pod_be_ready( self._master, pod_name, timeout=60, wait_step=5) time.sleep(120) # Fetching the metrics and storing in final_metrics as dictionary and # validating with initial_metrics final_metrics = self._get_and_manipulate_metric_data( self.metrics, pvc_name) self.assertEqual(dict(initial_metrics), dict(final_metrics), "Metrics are different post node reboot")
def test_heketi_device_removal_with_insuff_space(self): """Validate heketi with device removal insufficient space""" # Disable 4+ nodes and 3+ devices on the first 3 nodes min_free_space_gb = 5 min_free_space = min_free_space_gb * 1024**2 heketi_url = self.heketi_server_url heketi_node = self.heketi_client_node nodes = {} node_ids = heketi_node_list(heketi_node, heketi_url) self.assertTrue(node_ids) for node_id in node_ids: node_info = heketi_node_info(heketi_node, heketi_url, node_id, json=True) if (node_info["state"].lower() != "online" or not node_info["devices"]): continue if len(nodes) > 2: heketi_node_disable(heketi_node, heketi_url, node_id) self.addCleanup(heketi_node_enable, heketi_node, heketi_url, node_id) continue for device in node_info["devices"]: if device["state"].lower() != "online": continue free_space = device["storage"]["free"] if node_id not in nodes: nodes[node_id] = [] if (free_space < min_free_space or len(nodes[node_id]) > 1): heketi_device_disable(heketi_node, heketi_url, device["id"]) self.addCleanup(heketi_device_enable, heketi_node, heketi_url, device["id"]) continue nodes[node_id].append({ "device_id": device["id"], "free": free_space }) # Skip test if nodes requirements are not met if (len(nodes) < 3 or not all(map((lambda _l: len(_l) > 1), nodes.values()))): raise self.skipTest( "Could not find 3 online nodes with 2 online devices " "having free space bigger than %dGb." % min_free_space_gb) # Calculate size of a potential distributed vol if nodes[node_ids[0]][0]["free"] > nodes[node_ids[0]][1]["free"]: index = 0 else: index = 1 vol_size_gb = int(nodes[node_ids[0]][index]["free"] / (1024**2)) + 1 device_id = nodes[node_ids[0]][index]["device_id"] # Create volume with such size that we consume space more than # size of smaller disks h_volume_name = "autotests-heketi-volume-%s" % utils.get_random_str() try: self.create_heketi_volume_with_name_and_wait(h_volume_name, vol_size_gb, json=True) except Exception as e: # NOTE: rare situation when we need to decrease size of a volume. g.log.info("Failed to create '%s'Gb volume. " "Trying to create another one, smaller for 1Gb.") if not ('more required' in str(e) and ('Insufficient suitable allocatable extents for ' 'logical volume' in str(e))): raise vol_size_gb -= 1 self.create_heketi_volume_with_name_and_wait(h_volume_name, vol_size_gb, json=True) # Try to 'remove' bigger Heketi disk expecting error, # because there is no space on smaller disk to relocate bricks to heketi_device_disable(heketi_node, heketi_url, device_id) self.addCleanup(heketi_device_enable, heketi_node, heketi_url, device_id) try: self.assertRaises(AssertionError, heketi_device_remove, heketi_node, heketi_url, device_id) except Exception: self.addCleanup(heketi_device_disable, heketi_node, heketi_url, device_id) raise
def test_metrics_workload_on_prometheus(self): """Validate metrics workload on prometheus""" # Skip test if the prometheus pods are not present openshift_ops.switch_oc_project( self._master, self._prometheus_project_name) prometheus_pods = openshift_ops.oc_get_pods( self._master, selector=self._prometheus_resources_selector) if not prometheus_pods: self.skipTest( prometheus_pods, "Skipping test as prometheus" " pod is not present") if not self.registry_sc: self.skipTest( prometheus_pods, "Skipping test as registry " " storage details are not provided") self._registry_project = self.registry_sc.get( 'restsecretnamespace') self.prefix = "autotest-{}".format(utils.get_random_str()) # Get one of the prometheus pod name and respective pvc name prometheus_pod = list(prometheus_pods.keys())[0] pvc_custom = ":.spec.volumes[*].persistentVolumeClaim.claimName" pvc_name = openshift_ops.oc_get_custom_resource( self._master, "pod", pvc_custom, prometheus_pod)[0] self.assertTrue( pvc_name, "Failed to get PVC name for prometheus" " pod {}".format(prometheus_pod)) self.verify_iscsi_sessions_and_multipath( pvc_name, prometheus_pod, rtype='pod', heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True) # Try to fetch metric from the prometheus pod self._fetch_metric_from_promtheus_pod( metric='kube_persistentvolumeclaim_info') # Create storage class openshift_ops.switch_oc_project( self._master, self._registry_project) self.sc_name = self.create_storage_class( vol_name_prefix=self.prefix, glusterfs_registry=True) self.addCleanup(openshift_ops.switch_oc_project, self._master, self._registry_project) # Create PVCs and app pods pvc_size, pvc_count, batch_count = 1, 5, 5 for _ in range(batch_count): test_pvc_names = self.create_and_wait_for_pvcs( pvc_size, pvc_name_prefix=self.prefix, pvc_amount=pvc_count, sc_name=self.sc_name, timeout=600, wait_step=10) self.create_dcs_with_pvc( test_pvc_names, timeout=600, wait_step=5, dc_name_prefix="autotests-dc-with-app-io", space_to_use=1048576) # Check from the prometheus pod for the PVC space usage openshift_ops.switch_oc_project( self._master, self._prometheus_project_name) mount_path = "/prometheus" cmd = "oc exec {0} -- df -PT {1} | grep {1}".format( prometheus_pod, mount_path) out = self.cmd_run(cmd) self.assertTrue(out, "Failed to get info about mounted volume. " "Output is empty.") # Try to fetch metric from prometheus pod self._fetch_metric_from_promtheus_pod( metric='kube_persistentvolumeclaim_info') self._fetch_metric_from_promtheus_pod( metric='kube_pod_spec_volumes_persistentvolumeclaims_info') self.addCleanup(openshift_ops.switch_oc_project, self._master, self._registry_project)