def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_neg( self): """Verify that block volume creation fails when we create block volume of size greater than the default size of BHV. Verify that block volume creation succeed when we create BHV of size greater than the default size of BHV. """ default_bhv_size = get_default_block_hosting_volume_size( self.node, self.heketi_dc_name) reserve_size = default_bhv_size * 0.02 reserve_size = int(math.ceil(reserve_size)) self.verify_free_space(default_bhv_size + reserve_size + 2) with self.assertRaises(ExecutionError): # create a block vol greater than default BHV size bvol_info = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + 1), json=True) self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, bvol_info['id']) sc_name = self.create_storage_class() # create a block pvc greater than default BHV size pvc_name = oc_create_pvc( self.node, sc_name, pvc_size=(default_bhv_size + 1)) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) wait_for_events( self.node, pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='ProvisioningFailed') # create block hosting volume greater than default BHV size vol_info = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + reserve_size + 2), block=True, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # Cleanup PVC before block hosting volume to avoid failures self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) verify_pvc_status_is_bound(self.node, pvc_name)
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_neg( self): """Verify that block volume creation fails when we create block volume of size greater than the default size of BHV. Verify that block volume creation succeed when we create BHV of size greater than the default size of BHV. """ default_bhv_size = get_default_block_hosting_volume_size( self.node, self.heketi_dc_name) reserve_size = default_bhv_size * 0.02 reserve_size = int(math.ceil(reserve_size)) self.verify_free_space(default_bhv_size + reserve_size + 2) with self.assertRaises(AssertionError): # create a block vol greater than default BHV size bvol_info = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + 1), json=True) self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, bvol_info['id']) sc_name = self.create_storage_class() # create a block pvc greater than default BHV size pvc_name = oc_create_pvc( self.node, sc_name, pvc_size=(default_bhv_size + 1)) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) wait_for_events( self.node, pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='ProvisioningFailed') # create block hosting volume greater than default BHV size vol_info = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + reserve_size + 2), block=True, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # Cleanup PVC before block hosting volume to avoid failures self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) verify_pvc_status_is_bound(self.node, pvc_name)
def test_validate_gluster_voloptions_blockhostvolume(self): """Validate gluster volume options which are set for block hosting volume""" options_to_validate = ( ('performance.quick-read', 'off'), ('performance.read-ahead', 'off'), ('performance.io-cache', 'off'), ('performance.stat-prefetch', 'off'), ('performance.open-behind', 'off'), ('performance.readdir-ahead', 'off'), ('performance.strict-o-direct', 'on'), ('network.remote-dio', 'disable'), ('cluster.eager-lock', 'enable'), ('cluster.quorum-type', 'auto'), ('cluster.data-self-heal-algorithm', 'full'), ('cluster.locking-scheme', 'granular'), ('cluster.shd-max-threads', '8'), ('cluster.shd-wait-qlength', '10000'), ('features.shard', 'on'), ('features.shard-block-size', '64MB'), ('user.cifs', 'off'), ('server.allow-insecure', 'on'), ) free_space, nodenum = get_total_free_space(self.heketi_client_node, self.heketi_server_url) if nodenum < 3: self.skipTest("Skip the test case since number of" "online nodes is less than 3.") free_space_available = int(free_space / nodenum) default_bhv_size = get_default_block_hosting_volume_size( self.heketi_client_node, self.heketi_dc_name) if free_space_available < default_bhv_size: self.skipTest("Skip the test case since free_space_available %s" "is less than the default_bhv_size %s ." % (free_space_available, default_bhv_size)) block_host_create_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, default_bhv_size, json=True, block=True) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, block_host_create_info["id"], raise_on_error=True) bhv_name = block_host_create_info["name"] vol_info = get_volume_info('auto_get_gluster_endpoint', volname=bhv_name) self.assertTrue(vol_info, "Failed to get volume info %s" % bhv_name) self.assertIn("options", vol_info[bhv_name].keys()) for k, v in options_to_validate: self.assertIn(k, vol_info[bhv_name]["options"].keys()) self.assertEqual(v, vol_info[bhv_name]["options"][k])
def test_block_host_volume_delete_block_volume_delete(self): """Validate block volume and BHV removal using heketi""" free_space, nodenum = get_total_free_space(self.heketi_client_node, self.heketi_server_url) if nodenum < 3: self.skipTest("Skipping the test since number of nodes" "online are less than 3") free_space_available = int(free_space / nodenum) default_bhv_size = get_default_block_hosting_volume_size( self.heketi_client_node, self.heketi_dc_name) if free_space_available < default_bhv_size: self.skipTest("Skipping the test since free_space_available %s" "is less than the default_bhv_size %s" % (free_space_available, default_bhv_size)) h_volume_name = ("autotests-heketi-volume-%s" % utils.get_random_str()) block_host_create_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, default_bhv_size, json=True, block=True) block_vol_size = block_host_create_info["blockinfo"]["freesize"] block_hosting_vol_id = block_host_create_info["id"] block_vol_info = {"blockhostingvolume": "init_value"} while (block_vol_info['blockhostingvolume'] != block_hosting_vol_id): block_vol = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, block_vol_size, json=True, ha=3, auth=True) self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, block_vol["id"], raise_on_error=True) block_vol_info = heketi_blockvolume_info(self.heketi_client_node, self.heketi_server_url, block_vol["id"], json=True) bhv_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, block_hosting_vol_id, json=True) self.assertIn(block_vol_info["id"], bhv_info["blockinfo"]["blockvolume"])
def test_heketi_block_volume_create_with_size_more_than_bhv_free_space( self): """ Test to create heketi block volume of size greater than free space in BHV so that it will create a new BHV. """ h_node, h_url = self.heketi_client_node, self.heketi_server_url default_bhv_size = get_default_block_hosting_volume_size( self.node, self.heketi_dc_name) reserve_size = math.ceil(default_bhv_size * 0.02) bhv_list, pvc_size = [], (default_bhv_size - reserve_size) # Get existing BHV list bhv_list = list(get_block_hosting_volume_list(h_node, h_url).keys()) for vol in bhv_list: info = heketi_volume_info(h_node, h_url, vol, json=True) if info['blockinfo']['freesize'] >= pvc_size: self.skipTest( "Skip test case since there is atleast one BHV with free" " space {} greater than the default value {}".format( info['blockinfo']['freesize'], pvc_size)) # To verify if there is enough free space for two BHVs self.verify_free_space(2 * (default_bhv_size + 1)) sc_name = self.create_storage_class() self._dynamic_provisioning_block_with_bhv_cleanup( sc_name, pvc_size, bhv_list) self._dynamic_provisioning_block_with_bhv_cleanup( sc_name, pvc_size, bhv_list) bhv_post = len(get_block_hosting_volume_list(h_node, h_url)) err_msg = ("New BHVs were not created to satisfy the block PV requests" " No. of BHV before the test : {} \n" " No. of BHV after the test : {}".format( len(bhv_list), bhv_post)) self.assertEqual(bhv_post, (len(bhv_list) + 2), err_msg)
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_pos( self): """Verify that block volume creation succeed when we create BHV of size greater than the default size of BHV. """ default_bhv_size = get_default_block_hosting_volume_size( self.node, self.heketi_dc_name) reserve_size = default_bhv_size * 0.02 reserve_size = int(math.ceil(reserve_size)) self.verify_free_space(default_bhv_size + reserve_size + 2) # create block hosting volume greater than default BHV size vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, (default_bhv_size + reserve_size + 2), block=True, json=True) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # create a block pvc greater than default BHV size self.create_and_wait_for_pvc(pvc_size=(default_bhv_size + 1))
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_pos( self): """Verify that block volume creation succeed when we create BHV of size greater than the default size of BHV. """ default_bhv_size = get_default_block_hosting_volume_size( self.node, self.heketi_dc_name) reserve_size = default_bhv_size * 0.02 reserve_size = int(math.ceil(reserve_size)) self.verify_free_space(default_bhv_size + reserve_size + 2) # create block hosting volume greater than default BHV size vol_info = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + reserve_size + 2), block=True, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # create a block pvc greater than default BHV size self.create_and_wait_for_pvc(pvc_size=(default_bhv_size + 1))
def test_heketi_manual_cleanup_operation_in_bhv(self): """Validate heketi db cleanup will resolve the mismatch in the free size of the block hosting volume with failed block device create operations. """ bhv_size_before, bhv_size_after, vol_count = [], [], 5 ocp_node, g_node = self.ocp_master_node[0], self.gluster_servers[0] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Get nodes id list node_id_list = heketi_node_list(h_node, h_url) # Disable 4th and other nodes for node_id in node_id_list[3:]: heketi_node_disable(h_node, h_url, node_id) self.addCleanup(heketi_node_enable, h_node, h_url, node_id) # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 50: h_volume_size = 50 else: h_volume_size, vol_count = 1, free_space_available # Create BHV in case blockvolume size is greater than default BHV size default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Get BHV list h_bhv_list = get_block_hosting_volume_list(h_node, h_url).keys() self.assertTrue(h_bhv_list, "Failed to get the BHV list") # Get BHV size for bhv in h_bhv_list: vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bhv_vol_size_before = vol_info.get("freesize") bhv_size_before.append(bhv_vol_size_before) # Kill Tcmu-runner service services = ("tcmu-runner", "gluster-block-target", "gluster-blockd") kill_service_on_gluster_pod_or_node(ocp_node, "tcmu-runner", g_node) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') self.addCleanup(wait_for_service_status_on_gluster_pod_or_node, ocp_node, service, 'active', state, g_node) self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node, service, g_node) def run_async(cmd, hostname, raise_on_error=True): return g.run_async(host=hostname, command=cmd) # Create stale block volumes in async for count in range(vol_count): with mock.patch.object(json, 'loads', side_effect=(lambda j: j)): with mock.patch.object(command, 'cmd_run', side_effect=run_async): heketi_blockvolume_create(h_node, h_url, h_volume_size, json=True) # Wait for pending operation to get generated self._check_for_pending_operations(h_node, h_url) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') restart_service_on_gluster_pod_or_node(ocp_node, service, g_node) wait_for_service_status_on_gluster_pod_or_node( ocp_node, service, 'active', state, g_node) # Cleanup pending operation heketi_server_operation_cleanup(h_node, h_url) # wait for pending operation to get cleaned up for w in waiter.Waiter(timeout=120, interval=10): # Get BHV size for bhv in h_bhv_list: vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bhv_vol_size_after = vol_info.get("freesize") bhv_size_after.append(bhv_vol_size_after) if (set(bhv_size_before) == set(bhv_size_after)): break if w.expired: raise exceptions.ExecutionError( "Failed to Validate volume size Actual:{}," " Expected:{}".format(set(bhv_size_before), set(bhv_size_after)))
def test_verify_create_heketi_volumes_pending_entries_in_db( self, vol_type): """Verify pending entries of file/block volumes in db during volumes creation from heketi side """ # Create large volumes to observe the pending operations vol_count, h_vol_creation_async_op = 3, [] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Verify file/block volumes pending operation before creation, h_db_check_before = heketi_db_check(h_node, h_url) h_db_check_vol_before = (h_db_check_before.get( "{}volumes".format(vol_type))) # Delete heketi pod to clean db operations if (h_db_check_vol_before.get("pending") or h_db_check_before.get("bricks").get("pending")): self._respin_heketi_pod() # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 30: h_volume_size = 30 else: h_volume_size, vol_count = 1, free_space_available # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add cleanup function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Create BHV in case blockvolume size is greater than default BHV size if vol_type: default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Temporary replace g.run with g.async_run in heketi_blockvolume_create # func to be able to run it in background.Also, avoid parsing the # output as it won't be json at that moment. Parse it after reading # the async operation results. def run_async(cmd, hostname, raise_on_error=True): return g.run_async(host=hostname, command=cmd) for count in range(vol_count): with mock.patch.object(json, 'loads', side_effect=(lambda j: j)): with mock.patch.object(command, 'cmd_run', side_effect=run_async): h_vol_creation_async_op.append( eval("heketi_{}volume_create".format(vol_type))( h_node, h_url, h_volume_size, json=True)) # Check for pending operations for w in waiter.Waiter(timeout=120, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if h_db_check_vol.get("pending"): h_db_check_bricks = h_db_check.get("bricks") break if w.expired: raise exceptions.ExecutionError( "No any pending operations found during {}volumes creation " "{}".format(vol_type, h_db_check_vol.get("pending"))) # Verify bricks pending operation during creation if not vol_type: self.assertTrue(h_db_check_bricks.get("pending"), "Expecting at least one bricks pending count") self.assertFalse( h_db_check_bricks.get("pending") % 3, "Expecting bricks pending count to be multiple of 3 but " "found {}".format(h_db_check_bricks.get("pending"))) # Wait for all counts of pending operations to be zero for w in waiter.Waiter(timeout=300, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if not h_db_check_vol.get("pending"): break if w.expired: raise exceptions.ExecutionError( "Expecting no pending operations after 300 sec but " "found {} operation".format(h_db_check_vol.get("pending"))) # Get heketi server DB details h_db_check_after = heketi_db_check(h_node, h_url) h_db_check_vol_after = (h_db_check_after.get( "{}volumes".format(vol_type))) h_db_check_bricks_after = h_db_check_after.get("bricks") # Verify if initial and final file/block volumes are same act_vol_count = h_db_check_vol_after.get("total") exp_vol_count = h_db_check_vol_before.get("total") + vol_count err_msg = ( "Actual {} and expected {} {}volume counts are not matched".format( act_vol_count, exp_vol_count, vol_type)) self.assertEqual(act_vol_count, exp_vol_count, err_msg) # Verify if initial and final bricks are same for file volume volumes = heketi_volume_list(h_node, h_url, json=True).get("volumes") new_volumes = list(set(volumes) - set(existing_volumes)) exp_brick_count = 0 for volume in new_volumes: vol_info = heketi_volume_info(h_node, h_url, volume, json=True) exp_brick_count += len(vol_info.get("bricks")) err_msg = "Actual {} and expected {} bricks counts are not matched" act_brick_count = h_db_check_bricks_after.get("total") self.assertEqual(act_brick_count, exp_brick_count, err_msg.format(act_brick_count, exp_brick_count))
def test_verify_delete_heketi_volumes_pending_entries_in_db( self, vol_type): """Verify pending entries of blockvolumes/volumes and bricks in heketi db during blockvolume/volume delete operation. """ # Create a large volumes to observe the pending operation vol_count, volume_ids, async_obj = 10, [], [] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Verify file/block volumes pending operation before creation, h_db_check_before = heketi_db_check(h_node, h_url) h_db_check_bricks_before = h_db_check_before.get("bricks") h_db_check_vol_before = (h_db_check_before.get( "{}volumes".format(vol_type))) # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add cleanup function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Delete heketi pod to clean db operations if (h_db_check_bricks_before.get("pending") or h_db_check_vol_before.get("pending")): self._respin_heketi_pod() # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 50: h_volume_size = 50 else: h_volume_size, vol_count = 1, free_space_available # Create BHV in case blockvolume size is greater than default BHV size if vol_type: default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Create file/block volumes for _ in range(vol_count): vol_id = eval("heketi_{}volume_create".format(vol_type))( h_node, h_url, h_volume_size, json=True).get("id") volume_ids.append(vol_id) self.addCleanup(eval("heketi_{}volume_delete".format(vol_type)), h_node, h_url, vol_id, raise_on_error=False) def run_async(cmd, hostname, raise_on_error=True): async_op = g.run_async(host=hostname, command=cmd) async_obj.append(async_op) return async_op bhv_list = [] for vol_id in volume_ids: # Get BHV ids to delete in case of block volumes if vol_type: vol_info = (heketi_blockvolume_info(h_node, h_url, vol_id, json=True)) if not vol_info.get("blockhostingvolume") in bhv_list: bhv_list.append(vol_info.get("blockhostingvolume")) # Temporary replace g.run with g.async_run in heketi_volume_delete # and heketi_blockvolume_delete func to be able to run it in # background. with mock.patch.object(command, 'cmd_run', side_effect=run_async): eval("heketi_{}volume_delete".format(vol_type))(h_node, h_url, vol_id) # Wait for pending operations to get generate for w in waiter.Waiter(timeout=30, interval=3): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if h_db_check_vol.get("pending"): h_db_check_bricks = h_db_check.get("bricks") break if w.expired: raise exceptions.ExecutionError( "No any pending operations found during {}volumes deletion " "{}".format(vol_type, h_db_check_vol.get("pending"))) # Verify bricks pending operation during creation if not vol_type: self.assertTrue(h_db_check_bricks.get("pending"), "Expecting at least one bricks pending count") self.assertFalse( h_db_check_bricks.get("pending") % 3, "Expecting bricks pending count to be multiple of 3 but " "found {}".format(h_db_check_bricks.get("pending"))) # Verify file/block volume pending operation during delete for w in waiter.Waiter(timeout=120, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) h_db_check_bricks = h_db_check.get("bricks") if ((not h_db_check_bricks.get("pending")) and (not h_db_check_vol.get("pending"))): break if w.expired: raise AssertionError( "Failed to delete {}volumes after 120 secs".format(vol_type)) # Check that all background processes got exited for obj in async_obj: ret, out, err = obj.async_communicate() self.assertFalse( ret, "Failed to delete {}volume due to error: {}".format( vol_type, err)) # Delete BHV created during block volume creation if vol_type: for bhv_id in bhv_list: heketi_volume_delete(h_node, h_url, bhv_id) # Verify bricks and volume pending operations h_db_check_after = heketi_db_check(h_node, h_url) h_db_check_bricks_after = h_db_check_after.get("bricks") h_db_check_vol_after = (h_db_check_after.get( "{}volumes".format(vol_type))) act_brick_count = h_db_check_bricks_after.get("pending") act_vol_count = h_db_check_vol_after.get("pending") # Verify bricks pending operation after delete err_msg = "{} operations are pending for {} after {}volume deletion" if not vol_type: self.assertFalse( act_brick_count, err_msg.format(act_brick_count, "brick", vol_type)) # Verify file/bock volumes pending operation after delete self.assertFalse(act_vol_count, err_msg.format(act_vol_count, "volume", vol_type)) act_brick_count = h_db_check_bricks_after.get("total") act_vol_count = h_db_check_vol_after.get("total") exp_brick_count = h_db_check_bricks_before.get("total") exp_vol_count = h_db_check_vol_before.get("total") err_msg = "Actual {} and expected {} {} counts are not matched" # Verify if initial and final file/block volumes are same self.assertEqual( act_vol_count, exp_vol_count, err_msg.format(act_vol_count, exp_vol_count, "volume")) # Verify if initial and final bricks are same self.assertEqual( act_brick_count, exp_brick_count, err_msg.format(act_brick_count, exp_brick_count, "brick"))
def test_create_max_num_blockhostingvolumes(self): num_of_bv = 10 new_bhv_list, bv_list, g_nodes = [], [], [] free_space, nodenum = get_total_free_space(self.heketi_client_node, self.heketi_server_url) if nodenum < 3: self.skipTest("Skip the test case since number of" "online nodes is less than 3.") free_space_available = int(free_space / nodenum) default_bhv_size = get_default_block_hosting_volume_size( self.heketi_client_node, self.heketi_dc_name) # Get existing list of BHV's existing_bhv_list = get_block_hosting_volume_list( self.heketi_client_node, self.heketi_server_url) # Skip the test if available space is less than default_bhv_size if free_space_available < default_bhv_size: self.skipTest("Skip the test case since free_space_available %s" "is less than space_required_for_bhv %s ." % (free_space_available, default_bhv_size)) # Create BHV's while free_space_available > default_bhv_size: block_host_create_info = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, default_bhv_size, json=True, block=True) if block_host_create_info["id"] not in existing_bhv_list.keys(): new_bhv_list.append(block_host_create_info["id"]) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, block_host_create_info["id"], raise_on_error=False) block_vol_size = int( block_host_create_info["blockinfo"]["freesize"] / num_of_bv) # Create specified number of BV's in BHV's created for i in range(0, num_of_bv): block_vol = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, block_vol_size, json=True, ha=3, auth=True) self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, block_vol["id"], raise_on_error=False) bv_list.append(block_vol["id"]) free_space_available = int(free_space_available - default_bhv_size) # Get gluster node ips h_nodes_ids = heketi_node_list(self.heketi_client_node, self.heketi_server_url) for h_node in h_nodes_ids[:2]: g_node = heketi_node_info(self.heketi_client_node, self.heketi_server_url, h_node, json=True) g_nodes.append(g_node['hostnames']['manage'][0]) # Check if there is no crash in gluster related services & heketi services = (("glusterd", "running"), ("gluster-blockd", "running"), ("tcmu-runner", "running"), ("gluster-block-target", "exited")) for g_node in g_nodes: for service, state in services: wait_for_service_status_on_gluster_pod_or_node( self.ocp_client[0], service, 'active', state, g_node, raise_on_error=False) out = hello_heketi(self.heketi_client_node, self.heketi_server_url) self.assertTrue( out, "Heketi server %s is not alive" % self.heketi_server_url) # Delete all the BHV's and BV's created for bv_volume in bv_list: heketi_blockvolume_delete(self.heketi_client_node, self.heketi_server_url, bv_volume) # Check if any blockvolume exist in heketi & gluster for bhv_volume in new_bhv_list[:]: heketi_vol_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, bhv_volume, json=True) self.assertNotIn("blockvolume", heketi_vol_info["blockinfo"].keys()) gluster_vol_info = get_block_list('auto_get_gluster_endpoint', volname="vol_%s" % bhv_volume) self.assertIsNotNone(gluster_vol_info, "Failed to get volume info %s" % bhv_volume) new_bhv_list.remove(bhv_volume) for blockvol in gluster_vol_info: self.assertNotIn("blockvol_", blockvol) heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, bhv_volume) # Check if all blockhosting volumes are deleted from heketi self.assertFalse(new_bhv_list)