def test_create_vol_and_retrieve_topology_info(self): volume_names = [] volume_ids = [] # Create 3 volumes and make 3rd volume of type distributed replica g.log.info("Creating 3 volumes") for i in range(3): out = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) g.log.info("Heketi volume %s successfully created" % out) volume_names.append(out["name"]) volume_ids.append(out["bricks"][0]["volume"]) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_ids[i], raise_on_error=(i == 2)) heketi_volume_expand(self.heketi_client_node, self.heketi_server_url, volume_ids[1], 1) # Check if volume is shown in the heketi topology topology_volumes = get_heketi_volume_and_brick_count_list( self.heketi_client_node, self.heketi_server_url) existing_volumes = [v for v, _ in topology_volumes] for v in volume_names: self.assertIn(v, existing_volumes) for v, b_count in topology_volumes: expected_bricks_count = 6 if v == volume_names[1] else 3 self.assertGreaterEqual( b_count, expected_bricks_count, 'Bricks number of the %s volume is %s and it is expected ' 'to be greater or equal to %s' % (v, b_count, expected_bricks_count)) # Delete first 2 volumes and verify their deletion in the topology for vol_id in volume_ids[:2]: g.log.info("Deleting volume %s" % vol_id) heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol_id) topology_volumes = get_heketi_volume_and_brick_count_list( self.heketi_client_node, self.heketi_server_url) existing_volumes = [v for v, _ in topology_volumes] for vol_name in volume_names[:2]: self.assertNotIn( vol_name, existing_volumes, ("volume %s shown in the heketi topology after deletion" "\nTopology info:\n%s" % (vol_name, existing_volumes))) # Check the existence of third volume self.assertIn( volume_names[2], existing_volumes, "volume %s not " "shown in the heketi topology\nTopology info" "\n%s" % (volume_ids[2], existing_volumes)) g.log.info("Sucessfully verified the topology info")
def test_heketi_with_expand_volume(self): """ Test volume expand and size if updated correctly in heketi-cli info """ vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) self.assertTrue(vol_info, ("Failed to create heketi volume of size %s" % self.volume_size)) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) self.assertEqual(vol_info['size'], self.volume_size, ("Failed to create volume." "Expected Size: %s, Actual Size: %s" % (self.volume_size, vol_info['size']))) volume_id = vol_info["id"] expand_size = 2 ret = heketi_volume_expand(self.heketi_client_node, self.heketi_server_url, volume_id, expand_size) self.assertTrue(ret, ("Failed to expand heketi volume of id %s" % volume_id)) volume_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, volume_id, json=True) expected_size = self.volume_size + expand_size self.assertEqual(volume_info['size'], expected_size, ("Volume Expansion failed Expected Size: %s, Actual " "Size: %s" % (str(expected_size), str(volume_info['size']))))
def test_heketi_with_expand_volume(self): """ Test volume expand and size if updated correctly in heketi-cli info """ vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) self.assertTrue( vol_info, ("Failed to create heketi volume of size %s" % self.volume_size)) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) self.assertEqual(vol_info['size'], self.volume_size, ("Failed to create volume." "Expected Size: %s, Actual Size: %s" % (self.volume_size, vol_info['size']))) volume_id = vol_info["id"] expand_size = 2 ret = heketi_volume_expand(self.heketi_client_node, self.heketi_server_url, volume_id, expand_size) self.assertTrue( ret, ("Failed to expand heketi volume of id %s" % volume_id)) volume_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, volume_id, json=True) expected_size = self.volume_size + expand_size self.assertEqual(volume_info['size'], expected_size, ("Volume Expansion failed Expected Size: %s, Actual " "Size: %s" % (str(expected_size), str(volume_info['size']))))
def test_promethoues_validation_while_creation_or_expansion(self, motive): """Validate mertics data after volume creation or expansion""" # Define the variables to perform validations metrics = [ 'heketi_device_size_bytes', 'heketi_device_free_bytes', 'heketi_device_used_bytes', 'heketi_device_brick_count' ] h_client, h_server = self.heketi_client_node, self.heketi_server_url vol_size = 1 # Collect the metrics data from prometheus pod if motive == 'creation': initial_result = self._get_and_manipulate_metric_data(metrics) # Create a volume volume_id = heketi_ops.heketi_volume_create( h_client, h_server, vol_size, json=True)["bricks"][0]["volume"] self.addCleanup(heketi_ops.heketi_volume_delete, h_client, h_server, volume_id) # Expand the volume if motive == 'expansion': initial_result = self._get_and_manipulate_metric_data(metrics) heketi_ops.heketi_volume_expand(h_client, h_server, volume_id, vol_size) # Fetch the latest metrics data form prometheus pod final_result = self._get_and_manipulate_metric_data(metrics) # Validate the data variation for metric in metrics: msg = ( "intial {} and final value {} of metric '{} should be".format( initial_result[metric], final_result[metric], metric)) if metric == 'heketi_device_size_bytes': self.assertEqual(initial_result[metric], final_result[metric], msg + " same") if metric == 'heketi_device_free_bytes': self.assertGreater(initial_result[metric], final_result[metric], msg + " differnt") if metric == ('heketi_device_used_bytes' or 'heketi_device_brick_count'): self.assertLess(initial_result[metric], final_result[metric], msg + " differnt")
def test_replica_volume_expand(self): """ Test expansion of a replica volume """ h_node, h_server = self.heketi_client_node, self.heketi_server_url volume_name = ("autotests-heketi-volume-{}".format( utils.get_random_str())) volume_size = 10 creation_info = self.create_heketi_volume_with_name_and_wait( volume_name, volume_size, json=True, raise_on_cleanup_error=False) volume_id = creation_info["id"] volume_info = heketi_ops.heketi_volume_info(h_node, h_server, volume_id, json=True) # Get gluster volume info gluster_vol = volume_ops.get_volume_info('auto_get_gluster_endpoint', volname=volume_name) self.assertTrue(gluster_vol, "Failed to get volume {} info".format(volume_name)) vol_name = gluster_vol[volume_name] self.assertEqual( vol_name['replicaCount'], "3", "Replica count is different for volume {} Actual:{} " "Expected : 3".format(vol_name, vol_name['replicaCount'])) expand_size = 5 heketi_ops.heketi_volume_expand(h_node, h_server, volume_id, expand_size) volume_info = heketi_ops.heketi_volume_info(h_node, h_server, volume_id, json=True) expected_size = volume_size + expand_size self.assertEqual( volume_info['size'], expected_size, "Volume Expansion failed, Expected Size: {}, Actual " "Size: {}".format(str(expected_size), str(volume_info['size']))) self.get_brick_and_volume_status(volume_name) self.get_rebalance_status(volume_name)
def test_volume_expansion_rebalance_brick(self): """Validate volume expansion with brick and check rebalance""" creation_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.assertNotEqual(creation_info, False, "Volume creation failed") volume_name = creation_info["name"] volume_id = creation_info["id"] free_space_after_creation = self.get_devices_summary_free_space() volume_info_before_expansion = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(volume_info_before_expansion, False, "Volume info for %s failed" % volume_id) heketi_vol_info_size_before_expansion = ( volume_info_before_expansion["size"]) self.get_brick_and_volume_status(volume_name) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) expansion_info = heketi_ops.heketi_volume_expand( self.heketi_client_node, self.heketi_server_url, volume_id, 5) self.assertNotEqual(expansion_info, False, "Volume expansion of %s failed" % volume_id) free_space_after_expansion = self.get_devices_summary_free_space() self.assertTrue( free_space_after_creation > free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) volume_info_after_expansion = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(volume_info_after_expansion, False, "Volume info failed for %s" % volume_id) heketi_vol_info_size_after_expansion = ( volume_info_after_expansion["size"]) difference_size = (heketi_vol_info_size_after_expansion - heketi_vol_info_size_before_expansion) self.assertTrue( difference_size > 0, "Size not increased after expansion of %s" % volume_id) self.get_brick_and_volume_status(volume_name) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) num_of_bricks_added = (num_of_bricks_after_expansion - num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_added, 3, "Number of bricks added is not 3 for %s" % volume_id) self.get_rebalance_status(volume_name) deletion_info = heketi_ops.heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(deletion_info, False, "Deletion of volume %s failed" % volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertTrue( free_space_after_deletion > free_space_after_expansion, "Free space is not reclaimed after volume deletion of %s" % volume_id)
def test_volume_expansion_no_free_space(self): """Validate volume expansion when there is no free space""" vol_size, expand_size, additional_devices_attached = None, 10, {} h_node, h_server_url = self.heketi_client_node, self.heketi_server_url # Get nodes info heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url) if len(heketi_node_id_list) < 3: self.skipTest("3 Heketi nodes are required.") # Disable 4th and other nodes for node_id in heketi_node_id_list[3:]: heketi_ops.heketi_node_disable(h_node, h_server_url, node_id) self.addCleanup( heketi_ops.heketi_node_enable, h_node, h_server_url, node_id) # Prepare first 3 nodes smallest_size = None err_msg = '' for node_id in heketi_node_id_list[0:3]: node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) # Disable second and other devices devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_id) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if (smallest_size is None or devices[0]["storage"]["free"] < smallest_size): smallest_size = devices[0]["storage"]["free"] for device in node_info["devices"][1:]: heketi_ops.heketi_device_disable( h_node, h_server_url, device["id"]) self.addCleanup( heketi_ops.heketi_device_enable, h_node, h_server_url, device["id"]) # Gather info about additional devices additional_device_name = None for gluster_server in self.gluster_servers: gluster_server_data = self.gluster_servers_info[gluster_server] g_manage = gluster_server_data["manage"] g_storage = gluster_server_data["storage"] if not (g_manage in node_info["hostnames"]["manage"] or g_storage in node_info["hostnames"]["storage"]): continue additional_device_name = (( gluster_server_data.get("additional_devices") or [''])[0]) break if not additional_device_name: err_msg += ("No 'additional_devices' are configured for " "'%s' node, which has following hostnames and " "IP addresses: %s.\n" % ( node_id, ', '.join(node_info["hostnames"]["manage"] + node_info["hostnames"]["storage"]))) continue heketi_ops.heketi_device_add( h_node, h_server_url, additional_device_name, node_id) additional_devices_attached.update( {node_id: additional_device_name}) # Schedule cleanup of the added devices for node_id in additional_devices_attached.keys(): node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) for device in node_info["devices"]: if device["name"] != additional_devices_attached[node_id]: continue self.addCleanup(self.detach_devices_attached, device["id"]) break else: self.fail("Could not find ID for added device on " "'%s' node." % node_id) if err_msg: self.skipTest(err_msg) # Temporary disable new devices self.disable_devices(additional_devices_attached) # Create volume and save info about it vol_size = int(smallest_size / (1024**2)) - 1 creation_info = heketi_ops.heketi_volume_create( h_node, h_server_url, vol_size, json=True) volume_name, volume_id = creation_info["name"], creation_info["id"] self.addCleanup( heketi_ops.heketi_volume_delete, h_node, h_server_url, volume_id, raise_on_error=False) volume_info_before_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) free_space_before_expansion = self.get_devices_summary_free_space() # Try to expand volume with not enough device space self.assertRaises( ExecutionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume self.enable_devices(additional_devices_attached) # Expand volume and validate results heketi_ops.heketi_volume_expand( h_node, h_server_url, volume_id, expand_size, json=True) free_space_after_expansion = self.get_devices_summary_free_space() self.assertGreater( free_space_before_expansion, free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) volume_info_after_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) self.assertGreater( volume_info_after_expansion["size"], volume_info_before_expansion["size"], "Size of %s not increased" % volume_id) self.assertGreater( num_of_bricks_after_expansion, num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0) # Delete volume and validate release of the used space heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertGreater( free_space_after_deletion, free_space_after_expansion, "Free space not reclaimed after deletion of volume %s" % volume_id)
def test_expansion_of_block_hosting_volume_using_heketi(self): """Verify that after expanding block hosting volume we are able to consume the expanded space""" h_node = self.heketi_client_node h_url = self.heketi_server_url bvols_in_bhv = set([]) bvols_pv = set([]) BHVS = get_block_hosting_volume_list(h_node, h_url) free_BHVS_count = 0 for vol in BHVS.keys(): info = heketi_volume_info(h_node, h_url, vol, json=True) if info['blockinfo']['freesize'] > 0: free_BHVS_count += 1 if free_BHVS_count > 1: self.skipTest("Skip test case because there is more than one" " Block Hosting Volume with free space") # create block volume of 1gb bvol_info = heketi_blockvolume_create(h_node, h_url, 1, json=True) expand_size = 20 try: self.verify_free_space(expand_size) bhv = bvol_info['blockhostingvolume'] vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bvols_in_bhv.update(vol_info['blockinfo']['blockvolume']) finally: # cleanup BHV if there is only one block volume inside it if len(bvols_in_bhv) == 1: self.addCleanup(heketi_volume_delete, h_node, h_url, bhv, json=True) self.addCleanup(heketi_blockvolume_delete, h_node, h_url, bvol_info['id']) size = vol_info['size'] free_size = vol_info['blockinfo']['freesize'] bvol_count = int(free_size / expand_size) bricks = vol_info['bricks'] # create pvs to fill the BHV pvcs = self.create_and_wait_for_pvcs( pvc_size=(expand_size if bvol_count else free_size), pvc_amount=(bvol_count or 1), timeout=300) vol_expand = True for i in range(2): # get the vol ids from pvcs for pvc in pvcs: pv = get_pv_name_from_pvc(self.node, pvc) custom = r':.metadata.annotations."gluster\.org\/volume-id"' bvol_id = oc_get_custom_resource(self.node, 'pv', custom, pv) bvols_pv.add(bvol_id[0]) vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bvols = vol_info['blockinfo']['blockvolume'] bvols_in_bhv.update(bvols) self.assertEqual(bvols_pv, (bvols_in_bhv & bvols_pv)) # Expand BHV and verify bricks and size of BHV if vol_expand: vol_expand = False heketi_volume_expand(h_node, h_url, bhv, expand_size, json=True) vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) self.assertEqual(size + expand_size, vol_info['size']) self.assertFalse(len(vol_info['bricks']) % 3) self.assertLess(len(bricks), len(vol_info['bricks'])) # create more PVCs in expanded BHV pvcs = self.create_and_wait_for_pvcs(pvc_size=(expand_size - 1), pvc_amount=1)
def test_set_heketi_vol_size_and_brick_amount_limits(self): # Get Heketi secret name cmd_get_heketi_secret_name = ( "oc get dc -n %s %s -o jsonpath='{.spec.template.spec.volumes" "[?(@.name==\"config\")].secret.secretName}'" % (self.storage_project_name, self.heketi_dc_name)) heketi_secret_name = self.cmd_run(cmd_get_heketi_secret_name) # Read Heketi secret data self.node = self.ocp_master_node[0] heketi_secret_data_str_base64 = oc_get_custom_resource( self.node, "secret", ":.data.'heketi\.json'", # noqa name=heketi_secret_name)[0] heketi_secret_data_str = self.cmd_run("echo %s | base64 -d" % heketi_secret_data_str_base64) heketi_secret_data = json.loads(heketi_secret_data_str) # Update Heketi secret data brick_min_size_gb, brick_max_size_gb = 2, 4 heketi_secret_data["glusterfs"].update({ "brick_min_size_gb": brick_min_size_gb, "brick_max_size_gb": brick_max_size_gb, "max_bricks_per_volume": 3, }) heketi_secret_data_patched = json.dumps(heketi_secret_data) heketi_secret_data_str_encoded = self.cmd_run( "echo '%s' |base64" % heketi_secret_data_patched).replace( '\n', '') h_client, h_server = self.heketi_client_node, self.heketi_server_url try: # Patch Heketi secret cmd_patch_heketi_secret = ( 'oc patch secret -n %s %s -p ' '"{\\"data\\": {\\"heketi.json\\": \\"%s\\"}}"') % ( self.storage_project_name, heketi_secret_name, "%s") self.cmd_run(cmd_patch_heketi_secret % heketi_secret_data_str_encoded) # Recreate the Heketi pod to make it reuse updated configuration scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 0) scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 1) # Try to create too small and too big volumes # It must fail because allowed range is not satisfied for gb in (brick_min_size_gb - 1, brick_max_size_gb + 1): try: vol_1 = heketi_volume_create(h_client, h_server, size=gb, json=True) except AssertionError: pass else: self.addCleanup(heketi_volume_delete, h_client, h_server, vol_1['id']) self.assertFalse( vol_1, "Volume '%s' got unexpectedly created. Heketi server " "configuration haven't made required effect." % (vol_1.get('id', 'failed_to_get_heketi_vol_id'))) # Create the smallest allowed volume vol_2 = heketi_volume_create(h_client, h_server, size=brick_min_size_gb, json=True) self.addCleanup(heketi_volume_delete, h_client, h_server, vol_2['id']) # Try to expand volume, it must fail due to the brick amount limit self.assertRaises(AssertionError, heketi_volume_expand, h_client, h_server, vol_2['id'], 2) # Create the largest allowed volume vol_3 = heketi_volume_create(h_client, h_server, size=brick_max_size_gb, json=True) heketi_volume_delete(h_client, h_server, vol_3['id']) finally: # Revert the Heketi configuration back self.cmd_run(cmd_patch_heketi_secret % heketi_secret_data_str_base64) scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 0) scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 1) # Create volume less than the old minimum limit vol_4 = heketi_volume_create(h_client, h_server, size=(brick_min_size_gb - 1), json=True) self.addCleanup(heketi_volume_delete, h_client, h_server, vol_4['id']) # Create volume bigger than the old maximum limit and expand it vol_5 = heketi_volume_create(h_client, h_server, size=(brick_max_size_gb + 1), json=True) self.addCleanup(heketi_volume_delete, h_client, h_server, vol_5['id']) heketi_volume_expand(h_client, h_server, vol_5['id'], 2)
def test_volume_expansion_rebalance_brick(self): """Validate volume expansion with brick and check rebalance""" creation_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.assertNotEqual(creation_info, False, "Volume creation failed") volume_name = creation_info["name"] volume_id = creation_info["id"] free_space_after_creation = self.get_devices_summary_free_space() volume_info_before_expansion = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(volume_info_before_expansion, False, "Volume info for %s failed" % volume_id) heketi_vol_info_size_before_expansion = ( volume_info_before_expansion["size"]) self.get_brick_and_volume_status(volume_name) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) expansion_info = heketi_ops.heketi_volume_expand( self.heketi_client_node, self.heketi_server_url, volume_id, 5) self.assertNotEqual(expansion_info, False, "Volume expansion of %s failed" % volume_id) free_space_after_expansion = self.get_devices_summary_free_space() self.assertTrue( free_space_after_creation > free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) volume_info_after_expansion = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(volume_info_after_expansion, False, "Volume info failed for %s" % volume_id) heketi_vol_info_size_after_expansion = ( volume_info_after_expansion["size"]) difference_size = (heketi_vol_info_size_after_expansion - heketi_vol_info_size_before_expansion) self.assertTrue( difference_size > 0, "Size not increased after expansion of %s" % volume_id) self.get_brick_and_volume_status(volume_name) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) num_of_bricks_added = ( num_of_bricks_after_expansion - num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_added, 3, "Number of bricks added is not 3 for %s" % volume_id) self.get_rebalance_status(volume_name) deletion_info = heketi_ops.heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(deletion_info, False, "Deletion of volume %s failed" % volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertTrue( free_space_after_deletion > free_space_after_expansion, "Free space is not reclaimed after volume deletion of %s" % volume_id)
def test_volume_expansion_no_free_space(self): """Validate volume expansion when there is no free space""" vol_size, expand_size, additional_devices_attached = None, 10, {} h_node, h_server_url = self.heketi_client_node, self.heketi_server_url # Get nodes info heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url) if len(heketi_node_id_list) < 3: self.skipTest("3 Heketi nodes are required.") # Disable 4th and other nodes for node_id in heketi_node_id_list[3:]: heketi_ops.heketi_node_disable(h_node, h_server_url, node_id) self.addCleanup( heketi_ops.heketi_node_enable, h_node, h_server_url, node_id) # Prepare first 3 nodes smallest_size = None err_msg = '' for node_id in heketi_node_id_list[0:3]: node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) # Disable second and other devices devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_id) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if (smallest_size is None or devices[0]["storage"]["free"] < smallest_size): smallest_size = devices[0]["storage"]["free"] for device in node_info["devices"][1:]: heketi_ops.heketi_device_disable( h_node, h_server_url, device["id"]) self.addCleanup( heketi_ops.heketi_device_enable, h_node, h_server_url, device["id"]) # Gather info about additional devices additional_device_name = None for gluster_server in self.gluster_servers: gluster_server_data = self.gluster_servers_info[gluster_server] g_manage = gluster_server_data["manage"] g_storage = gluster_server_data["storage"] if not (g_manage in node_info["hostnames"]["manage"] or g_storage in node_info["hostnames"]["storage"]): continue additional_device_name = (( gluster_server_data.get("additional_devices") or [''])[0]) break if not additional_device_name: err_msg += ("No 'additional_devices' are configured for " "'%s' node, which has following hostnames and " "IP addresses: %s.\n" % ( node_id, ', '.join( node_info["hostnames"]["manage"] + node_info["hostnames"]["storage"]))) continue heketi_ops.heketi_device_add( h_node, h_server_url, additional_device_name, node_id) additional_devices_attached.update( {node_id: additional_device_name}) # Schedule cleanup of the added devices for node_id in additional_devices_attached.keys(): node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) for device in node_info["devices"]: if device["name"] != additional_devices_attached[node_id]: continue self.addCleanup(self.detach_devices_attached, device["id"]) break else: self.fail("Could not find ID for added device on " "'%s' node." % node_id) if err_msg: self.skipTest(err_msg) # Temporary disable new devices self.disable_devices(additional_devices_attached) # Create volume and save info about it vol_size = int(smallest_size / (1024**2)) - 1 creation_info = heketi_ops.heketi_volume_create( h_node, h_server_url, vol_size, json=True) volume_name, volume_id = creation_info["name"], creation_info["id"] self.addCleanup( heketi_ops.heketi_volume_delete, h_node, h_server_url, volume_id, raise_on_error=False) volume_info_before_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) free_space_before_expansion = self.get_devices_summary_free_space() # Try to expand volume with not enough device space self.assertRaises( AssertionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume self.enable_devices(additional_devices_attached) # Expand volume and validate results heketi_ops.heketi_volume_expand( h_node, h_server_url, volume_id, expand_size, json=True) free_space_after_expansion = self.get_devices_summary_free_space() self.assertGreater( free_space_before_expansion, free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) volume_info_after_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) self.assertGreater( volume_info_after_expansion["size"], volume_info_before_expansion["size"], "Size of %s not increased" % volume_id) self.assertGreater( num_of_bricks_after_expansion, num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0) # Delete volume and validate release of the used space heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertGreater( free_space_after_deletion, free_space_after_expansion, "Free space not reclaimed after deletion of volume %s" % volume_id)
def test_expansion_of_block_hosting_volume_using_heketi(self): """Verify that after expanding block hosting volume we are able to consume the expanded space""" h_node = self.heketi_client_node h_url = self.heketi_server_url bvols_in_bhv = set([]) bvols_pv = set([]) BHVS = get_block_hosting_volume_list(h_node, h_url) free_BHVS_count = 0 for vol in BHVS.keys(): info = heketi_volume_info(h_node, h_url, vol, json=True) if info['blockinfo']['freesize'] > 0: free_BHVS_count += 1 if free_BHVS_count > 1: self.skipTest("Skip test case because there is more than one" " Block Hosting Volume with free space") # create block volume of 1gb bvol_info = heketi_blockvolume_create(h_node, h_url, 1, json=True) expand_size = 20 try: self.verify_free_space(expand_size) bhv = bvol_info['blockhostingvolume'] vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bvols_in_bhv.update(vol_info['blockinfo']['blockvolume']) finally: # cleanup BHV if there is only one block volume inside it if len(bvols_in_bhv) == 1: self.addCleanup( heketi_volume_delete, h_node, h_url, bhv, json=True) self.addCleanup( heketi_blockvolume_delete, h_node, h_url, bvol_info['id']) size = vol_info['size'] free_size = vol_info['blockinfo']['freesize'] bvol_count = int(free_size / expand_size) bricks = vol_info['bricks'] # create pvs to fill the BHV pvcs = self.create_and_wait_for_pvcs( pvc_size=(expand_size if bvol_count else free_size), pvc_amount=(bvol_count or 1), timeout=300) vol_expand = True for i in range(2): # get the vol ids from pvcs for pvc in pvcs: pv = get_pv_name_from_pvc(self.node, pvc) custom = r':.metadata.annotations."gluster\.org\/volume-id"' bvol_id = oc_get_custom_resource(self.node, 'pv', custom, pv) bvols_pv.add(bvol_id[0]) vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bvols = vol_info['blockinfo']['blockvolume'] bvols_in_bhv.update(bvols) self.assertEqual(bvols_pv, (bvols_in_bhv & bvols_pv)) # Expand BHV and verify bricks and size of BHV if vol_expand: vol_expand = False heketi_volume_expand( h_node, h_url, bhv, expand_size, json=True) vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) self.assertEqual(size + expand_size, vol_info['size']) self.assertFalse(len(vol_info['bricks']) % 3) self.assertLess(len(bricks), len(vol_info['bricks'])) # create more PVCs in expanded BHV pvcs = self.create_and_wait_for_pvcs( pvc_size=(expand_size - 1), pvc_amount=1)