def test_blockvolume_create_no_free_space(self): """Validate error is returned when free capacity is exhausted""" # Create first small blockvolume blockvol1 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(blockvol1, "Failed to create block volume.") self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol1['id']) # Get info about block hosting volumes file_volumes = heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(file_volumes) self.assertIn("volumes", file_volumes) self.assertTrue(file_volumes["volumes"]) max_block_hosting_vol_size, file_volumes_debug_info = 0, [] for vol_id in file_volumes["volumes"]: vol = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, vol_id, json=True) current_block_hosting_vol_size = vol.get('size', 0) if current_block_hosting_vol_size > max_block_hosting_vol_size: max_block_hosting_vol_size = current_block_hosting_vol_size if current_block_hosting_vol_size: file_volumes_debug_info.append(six.text_type({ 'id': vol.get('id', '?'), 'name': vol.get('name', '?'), 'size': current_block_hosting_vol_size, 'blockinfo': vol.get('blockinfo', '?'), })) self.assertGreater(max_block_hosting_vol_size, 0) # Try to create blockvolume with size bigger than available too_big_vol_size = max_block_hosting_vol_size + 1 try: blockvol2 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, too_big_vol_size, json=True) except ExecutionError: return if blockvol2 and blockvol2.get('id'): self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol2['id']) block_hosting_vol = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, blockvol2.get('blockhostingvolume'), json=True) self.assertGreater( block_hosting_vol.get('size', -2), blockvol2.get('size', -1), ("Block volume unexpectedly was created. " "Calculated 'max free size' is '%s'.\nBlock volume info is: %s \n" "File volume info, which hosts block volume: \n%s," "Block hosting volumes which were considered: \n%s" % ( max_block_hosting_vol_size, blockvol2, block_hosting_vol, '\n'.join(file_volumes_debug_info))))
def test_blockvolume_create_no_free_space(self): """Validate error is returned when free capacity is exhausted""" # Create first small blockvolume blockvol1 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(blockvol1, "Failed to create block volume.") self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol1['id']) # Get info about block hosting volumes file_volumes = heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(file_volumes) self.assertIn("volumes", file_volumes) self.assertTrue(file_volumes["volumes"]) max_block_hosting_vol_size, file_volumes_debug_info = 0, [] for vol_id in file_volumes["volumes"]: vol = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, vol_id, json=True) current_block_hosting_vol_size = vol.get('size', 0) if current_block_hosting_vol_size > max_block_hosting_vol_size: max_block_hosting_vol_size = current_block_hosting_vol_size if current_block_hosting_vol_size: file_volumes_debug_info.append(six.text_type({ 'id': vol.get('id', '?'), 'name': vol.get('name', '?'), 'size': current_block_hosting_vol_size, 'blockinfo': vol.get('blockinfo', '?'), })) self.assertGreater(max_block_hosting_vol_size, 0) # Try to create blockvolume with size bigger than available too_big_vol_size = max_block_hosting_vol_size + 1 try: blockvol2 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, too_big_vol_size, json=True) except AssertionError: return if blockvol2 and blockvol2.get('id'): self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol2['id']) block_hosting_vol = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, blockvol2.get('blockhostingvolume'), json=True) self.assertGreater( block_hosting_vol.get('size', -2), blockvol2.get('size', -1), ("Block volume unexpectedly was created. " "Calculated 'max free size' is '%s'.\nBlock volume info is: %s \n" "File volume info, which hosts block volume: \n%s," "Block hosting volumes which were considered: \n%s" % ( max_block_hosting_vol_size, blockvol2, block_hosting_vol, '\n'.join(file_volumes_debug_info))))
def test_sc_create_with_clusterid(self): """Create storage class with 'cluster id'""" h_cluster_list = heketi_cluster_list(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(h_cluster_list, "Failed to list heketi cluster") cluster_id = h_cluster_list["clusters"][0] sc = self.create_storage_class(clusterid=cluster_id) pvc_name = self.create_and_wait_for_pvc(sc_name=sc) # Validate if cluster id is correct in the heketi volume info pv_name = get_pv_name_from_pvc(self.ocp_master_node[0], pvc_name) volume_id = oc_get_custom_resource( self.ocp_master_node[0], 'pv', r':metadata.annotations."gluster\.kubernetes\.io' r'\/heketi-volume-id"', name=pv_name)[0] volume_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertEqual( cluster_id, volume_info["cluster"], "Cluster ID %s has NOT been used to" "create the PVC %s. Found %s" % (cluster_id, pvc_name, volume_info["cluster"]))
def test_create_vol_and_retrieve_vol_info(self): """Validate heketi and gluster volume info""" g.log.info("Create a heketi volume") out = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) self.assertTrue(out, ("Failed to create heketi " "volume of size %s" % self.volume_size)) g.log.info("Heketi volume successfully created" % out) volume_id = out["bricks"][0]["volume"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) g.log.info("Retrieving heketi volume info") out = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertTrue(out, ("Failed to get heketi volume info")) g.log.info("Successfully got the heketi volume info") name = out["name"] vol_info = get_volume_info('auto_get_gluster_endpoint', volname=name) self.assertTrue(vol_info, "Failed to get volume info %s" % name) g.log.info("Successfully got the volume info %s" % name)
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix): # Create secret and storage class self.create_storage_class( create_vol_name_prefix=create_vol_name_prefix) # Create PVC pvc_name = self.create_and_wait_for_pvc() # Create DC with POD and attached PVC to it. dc_name = oc_create_app_dc_with_io(self.node, pvc_name) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Verify Heketi volume name for prefix presence if provided if create_vol_name_prefix: ret = verify_volume_name_prefix(self.node, self.sc['volumenameprefix'], self.sc['secretnamespace'], pvc_name, self.sc['resturl']) self.assertTrue(ret, "verify volnameprefix failed") else: # Get the volume name and volume id from PV pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name) custom = [ r':spec.glusterfs.path', r':metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi-volume-id"' ] pv_vol_name, vol_id = oc_get_custom_resource( self.ocp_client[0], 'pv', custom, pv_name) # check if the pv_volume_name is present in heketi # Check if volume name is "vol_"+volumeid or not heketi_vol_name = heketi_volume_info( self.ocp_client[0], self.heketi_server_url, vol_id, json=True)['name'] self.assertEqual(pv_vol_name, heketi_vol_name, 'Volume with vol_id = %s not found' 'in heketidb' % vol_id) self.assertEqual(heketi_vol_name, 'vol_' + vol_id, 'Volume with vol_id = %s have a' 'custom perfix' % vol_id) out = cmd_run_on_gluster_pod_or_node(self.ocp_master_node[0], "gluster volume list") self.assertIn(pv_vol_name, out, "Volume with id %s does not exist" % vol_id) # Make sure we are able to work with files on the mounted volume filepath = "/mnt/file_for_testing_io.log" for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100", "ls -lrt %s", "rm -rf %s"): cmd = cmd % filepath ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to execute '%s' command on %s" % (cmd, self.node))
def test_heketi_volume_create_with_clusterid(self): """Validate creation of heketi volume with clusters argument""" h_node, h_url = self.heketi_client_node, self.heketi_server_url # Get one of the cluster id from heketi cluster list creation_cluster_id = heketi_cluster_list(h_node, h_url, json=True)['clusters'][0] # Create a heketi volume specific to cluster list volume_id = heketi_volume_create(h_node, h_url, self.volume_size, clusters=creation_cluster_id, json=True)["bricks"][0]["volume"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) # Get the cluster id from heketi volume info info_cluster_id = heketi_volume_info(h_node, h_url, volume_id, json=True)['cluster'] # Match the creation cluster id with the info cluster id self.assertEqual( info_cluster_id, creation_cluster_id, "Volume creation cluster id {} not matching the info cluster id " "{}".format(creation_cluster_id, info_cluster_id))
def test_heketi_brick_evict(self): """Test brick evict basic functionality and verify it replace a brick properly """ h_node, h_server = self.heketi_client_node, self.heketi_server_url size = 1 vol_info_old = heketi_ops.heketi_volume_create( h_node, h_server, size, json=True) self.addCleanup( heketi_ops.heketi_volume_delete, h_node, h_server, vol_info_old['id']) heketi_ops.heketi_brick_evict( h_node, h_server, vol_info_old["bricks"][0]['id']) vol_info_new = heketi_ops.heketi_volume_info( h_node, h_server, vol_info_old['id'], json=True) bricks_old = set({brick['path'] for brick in vol_info_old["bricks"]}) bricks_new = set({brick['path'] for brick in vol_info_new["bricks"]}) self.assertEqual( len(bricks_new - bricks_old), 1, "Brick was not replaced with brick evict for vol \n {}".format( vol_info_new)) gvol_info = self._get_gluster_vol_info(vol_info_new['name']) gbricks = set( {brick['name'].split(":")[1] for brick in gvol_info["bricks"]["brick"]}) self.assertEqual( bricks_new, gbricks, "gluster vol info and heketi vol info " "mismatched after brick evict {} \n {}".format( gvol_info, vol_info_new))
def test_heketi_with_expand_volume(self): """ Test volume expand and size if updated correctly in heketi-cli info """ vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) self.assertTrue(vol_info, ("Failed to create heketi volume of size %s" % self.volume_size)) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) self.assertEqual(vol_info['size'], self.volume_size, ("Failed to create volume." "Expected Size: %s, Actual Size: %s" % (self.volume_size, vol_info['size']))) volume_id = vol_info["id"] expand_size = 2 ret = heketi_volume_expand(self.heketi_client_node, self.heketi_server_url, volume_id, expand_size) self.assertTrue(ret, ("Failed to expand heketi volume of id %s" % volume_id)) volume_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, volume_id, json=True) expected_size = self.volume_size + expand_size self.assertEqual(volume_info['size'], expected_size, ("Volume Expansion failed Expected Size: %s, Actual " "Size: %s" % (str(expected_size), str(volume_info['size']))))
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix): # Create secret and storage class self.create_storage_class( create_vol_name_prefix=create_vol_name_prefix) # Create PVC pvc_name = self.create_and_wait_for_pvc() # Create DC with POD and attached PVC to it. dc_name = oc_create_app_dc_with_io(self.node, pvc_name) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Verify Heketi volume name for prefix presence if provided if create_vol_name_prefix: ret = verify_volume_name_prefix(self.node, self.sc['volumenameprefix'], self.sc['secretnamespace'], pvc_name, self.sc['resturl']) self.assertTrue(ret, "verify volnameprefix failed") else: # Get the volume name and volume id from PV pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name) custom = [ r':spec.glusterfs.path', r':metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi-volume-id"' ] pv_vol_name, vol_id = oc_get_custom_resource( self.ocp_client[0], 'pv', custom, pv_name) # check if the pv_volume_name is present in heketi # Check if volume name is "vol_"+volumeid or not heketi_vol_name = heketi_volume_info( self.ocp_client[0], self.heketi_server_url, vol_id, json=True)['name'] self.assertEqual(pv_vol_name, heketi_vol_name, 'Volume with vol_id = %s not found' 'in heketidb' % vol_id) self.assertEqual(heketi_vol_name, 'vol_' + vol_id, 'Volume with vol_id = %s have a' 'custom perfix' % vol_id) out = cmd_run_on_gluster_pod_or_node(self.ocp_master_node[0], "gluster volume list") self.assertIn(pv_vol_name, out, "Volume with id %s does not exist" % vol_id) # Make sure we are able to work with files on the mounted volume filepath = "/mnt/file_for_testing_io.log" for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100", "ls -lrt %s", "rm -rf %s"): cmd = cmd % filepath ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to execute '%s' command on %s" % (cmd, self.node))
def test_delete_heketidb_volume(self): """Method to test heketidb volume deletion via heketi-cli.""" for i in range(0, 2): volume_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.addCleanup(heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_info["id"]) volume_list_info = heketi_ops.heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(volume_list_info["volumes"], "Heketi volume list empty.") for volume_id in volume_list_info["volumes"]: volume_info = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) if volume_info["name"] == "heketidbstorage": self.assertRaises(AssertionError, heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) return raise ExecutionError( "Warning: heketidbstorage doesn't exist in list of volumes")
def test_heketi_with_expand_volume(self): """ Test volume expand and size if updated correctly in heketi-cli info """ vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) self.assertTrue( vol_info, ("Failed to create heketi volume of size %s" % self.volume_size)) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) self.assertEqual(vol_info['size'], self.volume_size, ("Failed to create volume." "Expected Size: %s, Actual Size: %s" % (self.volume_size, vol_info['size']))) volume_id = vol_info["id"] expand_size = 2 ret = heketi_volume_expand(self.heketi_client_node, self.heketi_server_url, volume_id, expand_size) self.assertTrue( ret, ("Failed to expand heketi volume of id %s" % volume_id)) volume_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, volume_id, json=True) expected_size = self.volume_size + expand_size self.assertEqual(volume_info['size'], expected_size, ("Volume Expansion failed Expected Size: %s, Actual " "Size: %s" % (str(expected_size), str(volume_info['size']))))
def verify_volume_bricks_are_present_or_not_on_heketi_node( self, vol_ids, node_id, state='present'): for vol_id in vol_ids: vol_info = heketi_ops.heketi_volume_info( self.h_node, self.h_url, vol_id, json=True) bricks = vol_info['bricks'] self.assertFalse((len(bricks) % 3)) if state == 'present': found = False for brick in bricks: if brick['node'] == node_id: found = True break self.assertTrue( found, 'Bricks of vol %s does not present on node %s' % (vol_id, node_id)) elif state == 'absent': for brick in bricks: self.assertNotEqual( brick['node'], node_id, 'Bricks of vol %s is present ' 'on node %s' % (vol_id, node_id)) else: msg = "State %s is other than present, absent" % state raise AssertionError(msg)
def get_vol_names_from_vol_ids(self, vol_ids): vol_names = [] for vol_id in vol_ids: vol_info = heketi_ops.heketi_volume_info( self.h_node, self.h_url, vol_id, json=True) vol_names.append(vol_info['name']) return vol_names
def test_create_vol_and_retrieve_vol_info(self): """Validate heketi and gluster volume info""" g.log.info("Create a heketi volume") out = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) self.assertTrue(out, ("Failed to create heketi " "volume of size %s" % self.volume_size)) g.log.info("Heketi volume successfully created" % out) volume_id = out["bricks"][0]["volume"] self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) g.log.info("Retrieving heketi volume info") out = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertTrue(out, ("Failed to get heketi volume info")) g.log.info("Successfully got the heketi volume info") name = out["name"] vol_info = get_volume_info('auto_get_gluster_endpoint', volname=name) self.assertTrue(vol_info, "Failed to get volume info %s" % name) g.log.info("Successfully got the volume info %s" % name)
def test_brick_evict_on_more_than_three_node_with_one_down(self): """Test brick evict basic functionality and verify brick evict will success after one node down out of more than three nodes""" h_node, h_server = self.heketi_client_node, self.heketi_server_url # Create heketi volume vol_info = heketi_ops.heketi_volume_create( h_node, h_server, 1, json=True) self.addCleanup( heketi_ops.heketi_volume_delete, h_node, h_server, vol_info.get('id')) # Get node on which heketi pod is scheduled heketi_pod = openshift_ops.get_pod_name_from_dc( self.ocp_client, self.heketi_dc_name) heketi_node = openshift_ops.oc_get_custom_resource( self.ocp_client, 'pod', '.:spec.nodeName', heketi_pod)[0] # Get brick id and glusterfs node which is not heketi node for node in vol_info.get('bricks', {}): node_info = heketi_ops.heketi_node_info( h_node, h_server, node.get('node'), json=True) hostname = node_info.get('hostnames').get('manage')[0] if hostname != heketi_node: brick_id = node.get('id') break self._power_off_node_and_wait_node_to_be_not_ready(hostname) # Perform brick evict operation heketi_ops.heketi_brick_evict(h_node, h_server, brick_id) # Get volume info after brick evict operation vol_info_new = heketi_ops.heketi_volume_info( h_node, h_server, vol_info.get('id'), json=True) # Get previous and new bricks from volume bricks_old = set( {brick.get('path') for brick in vol_info.get("bricks")}) bricks_new = set( {brick.get('path') for brick in vol_info_new.get("bricks")}) self.assertEqual( len(bricks_new - bricks_old), 1, "Brick was not replaced with brick evict for vol \n {}".format( vol_info_new)) # Get gluster volume info g_vol_info = self._get_gluster_vol_info(vol_info_new.get('name')) # Validate bricks on gluster volume and heketi volume g_bricks = set( {brick.get('name').split(":")[1] for brick in g_vol_info.get("bricks", {}).get("brick")}) self.assertEqual( bricks_new, g_bricks, "gluster vol info and heketi vol info " "mismatched after brick evict {} \n {}".format( g_bricks, g_vol_info))
def test_replica_volume_expand(self): """ Test expansion of a replica volume """ h_node, h_server = self.heketi_client_node, self.heketi_server_url volume_name = ("autotests-heketi-volume-{}".format( utils.get_random_str())) volume_size = 10 creation_info = self.create_heketi_volume_with_name_and_wait( volume_name, volume_size, json=True, raise_on_cleanup_error=False) volume_id = creation_info["id"] volume_info = heketi_ops.heketi_volume_info(h_node, h_server, volume_id, json=True) # Get gluster volume info gluster_vol = volume_ops.get_volume_info('auto_get_gluster_endpoint', volname=volume_name) self.assertTrue(gluster_vol, "Failed to get volume {} info".format(volume_name)) vol_name = gluster_vol[volume_name] self.assertEqual( vol_name['replicaCount'], "3", "Replica count is different for volume {} Actual:{} " "Expected : 3".format(vol_name, vol_name['replicaCount'])) expand_size = 5 heketi_ops.heketi_volume_expand(h_node, h_server, volume_id, expand_size) volume_info = heketi_ops.heketi_volume_info(h_node, h_server, volume_id, json=True) expected_size = volume_size + expand_size self.assertEqual( volume_info['size'], expected_size, "Volume Expansion failed, Expected Size: {}, Actual " "Size: {}".format(str(expected_size), str(volume_info['size']))) self.get_brick_and_volume_status(volume_name) self.get_rebalance_status(volume_name)
def _cleanup_heketi_volumes(self, existing_volumes): h_node, h_url = self.heketi_client_node, self.heketi_server_url volumes = heketi_volume_list(h_node, h_url, json=True).get("volumes") new_volumes = list(set(volumes) - set(existing_volumes)) for volume in new_volumes: h_vol_info = heketi_volume_info(h_node, h_url, volume, json=True) if h_vol_info.get("block"): for block_vol in ( h_vol_info.get("blockinfo").get("blockvolume")): heketi_blockvolume_delete(h_node, h_url, block_vol) heketi_volume_delete(h_node, h_url, volume, raise_on_error=False)
def _block_vol_expand_common_offline_vs_online(self, is_online_expand): node = self.ocp_master_node[0] h_node, h_server = self.heketi_client_node, self.heketi_server_url version = heketi_version.get_heketi_version(h_node) if version < '9.0.0-14': self.skipTest("heketi-client package {} does not support " "blockvolume expand".format(version.v_str)) pvc_name = self.create_and_wait_for_pvc() dc_name = self.create_dc_with_pvc(pvc_name) pv_name = get_pv_name_from_pvc(node, pvc_name) # get block volume id custom = r":.metadata.annotations.'gluster\.org\/volume-id'" bvol_id = oc_get_custom_resource(node, 'pv', custom, pv_name) self.assertNotEqual(bvol_id[0], "<none>", "volume name not found from pv {}".format(pv_name)) bvol_info = heketi_blockvolume_info(h_node, h_server, bvol_id[0], json=True) # verify required blockhostingvolume free size bhv_id = bvol_info["blockhostingvolume"] bhv_info = heketi_volume_info(h_node, h_server, bhv_id, json=True) if bhv_info["blockinfo"]["freesize"] < 1: self.skipTest("blockhostingvolume doesn't have required freespace") if not is_online_expand: scale_dc_pod_amount_and_wait(node, dc_name[0], pod_amount=0) # expand block volume and verify usable size bvol_info = heketi_blockvolume_expand(h_node, h_server, bvol_id[0], 2, json=True) self.assertEqual(bvol_info["size"], 2, "Block volume expand does not works") self.assertEqual( bvol_info["size"], bvol_info["usablesize"], "block volume size is not equal to the usablesize: {}".format( bvol_info)) return pvc_name, dc_name, bvol_info
def test_delete_heketidb_volume(self): """ Method to test heketidb volume deletion via heketi-cli """ heketidbexists = False msg = "Error: Cannot delete volume containing the Heketi database" for i in range(0, 2): volume_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.addCleanup( heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_info["id"]) volume_list_info = heketi_ops.heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) if volume_list_info["volumes"] == []: raise ExecutionError("Heketi volume list empty") for volume_id in volume_list_info["volumes"]: volume_info = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) if volume_info["name"] == "heketidbstorage": heketidbexists = True delete_ret, delete_output, delete_error = ( heketi_ops.heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, volume_id, raw_cli_output=True)) self.assertNotEqual(delete_ret, 0, "Return code not 0") self.assertEqual( delete_error.strip(), msg, "Invalid reason for heketidb deletion failure") if not heketidbexists: raise ExecutionError( "Warning: heketidbstorage doesn't exist in list of volumes")
def create_heketi_volume_with_name_and_wait(self, name, size, raise_on_cleanup_error=True, timeout=600, wait_step=10, **kwargs): json = kwargs.get("json", False) try: h_volume_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size, name=name, **kwargs) except Exception as e: if ('more required' in six.text_type(e) or ('Failed to allocate new volume' in six.text_type(e))): raise for w in Waiter(timeout, wait_step): h_volumes = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) h_volume_match = re.search(HEKETI_VOLUME_REGEX % name, h_volumes) if h_volume_match: h_volume_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, h_volume_match.group(1), json=json) break if w.expired: g.log.info( "Heketi volume with name %s not created in 600 sec" % name) raise self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, h_volume_info["id"], raise_on_error=raise_on_cleanup_error) return h_volume_info
def test_block_host_volume_delete_block_volume_delete(self): """Validate block volume and BHV removal using heketi""" free_space, nodenum = get_total_free_space(self.heketi_client_node, self.heketi_server_url) if nodenum < 3: self.skipTest("Skipping the test since number of nodes" "online are less than 3") free_space_available = int(free_space / nodenum) default_bhv_size = get_default_block_hosting_volume_size( self.heketi_client_node, self.heketi_dc_name) if free_space_available < default_bhv_size: self.skipTest("Skipping the test since free_space_available %s" "is less than the default_bhv_size %s" % (free_space_available, default_bhv_size)) h_volume_name = ("autotests-heketi-volume-%s" % utils.get_random_str()) block_host_create_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, default_bhv_size, json=True, block=True) block_vol_size = block_host_create_info["blockinfo"]["freesize"] block_hosting_vol_id = block_host_create_info["id"] block_vol_info = {"blockhostingvolume": "init_value"} while (block_vol_info['blockhostingvolume'] != block_hosting_vol_id): block_vol = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, block_vol_size, json=True, ha=3, auth=True) self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, block_vol["id"], raise_on_error=True) block_vol_info = heketi_blockvolume_info(self.heketi_client_node, self.heketi_server_url, block_vol["id"], json=True) bhv_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, block_hosting_vol_id, json=True) self.assertIn(block_vol_info["id"], bhv_info["blockinfo"]["blockvolume"])
def validate_block_volumes_count(self, h_node, h_server, node_ip): # get list of block volumes using heketi h_blockvol_list = heketi_blockvolume_list( h_node, h_server, json=True) # Get existing BHV list bhv_list = list( get_block_hosting_volume_list(h_node, h_server).keys()) for vol in bhv_list: bhv_info = heketi_volume_info(h_node, h_server, vol, json=True) bhv_name = bhv_info['name'] gluster_block_list = get_block_list(node_ip, volname=bhv_name) self.assertIsNotNone( gluster_block_list, "Failed to get gluster block list") self.assertEqual( len(h_blockvol_list['blockvolumes']), len(gluster_block_list), "Failed to verify blockvolume count Expected:'{}', " "Actual:'{}'".format( len(h_blockvol_list['blockvolumes']), len(gluster_block_list)))
def _cleanup_heketi_volumes(self, existing_volumes): """Cleanup created BHV and BV""" volumes = heketi_ops.heketi_volume_list(self.h_node, self.h_server, json=True).get("volumes") new_volumes = list(set(volumes) - set(existing_volumes)) for volume in new_volumes: h_vol_info = heketi_ops.heketi_volume_info(self.h_node, self.h_server, volume, json=True) if h_vol_info.get("block"): for block_vol in ( h_vol_info.get("blockinfo").get("blockvolume")): heketi_ops.heketi_blockvolume_delete(self.h_node, self.h_server, block_vol, raise_on_error=False) heketi_ops.heketi_volume_delete(self.h_node, self.h_server, volume, raise_on_error=False)
def test_heketi_block_volume_create_with_size_more_than_bhv_free_space( self): """ Test to create heketi block volume of size greater than free space in BHV so that it will create a new BHV. """ h_node, h_url = self.heketi_client_node, self.heketi_server_url default_bhv_size = get_default_block_hosting_volume_size( self.node, self.heketi_dc_name) reserve_size = math.ceil(default_bhv_size * 0.02) bhv_list, pvc_size = [], (default_bhv_size - reserve_size) # Get existing BHV list bhv_list = list(get_block_hosting_volume_list(h_node, h_url).keys()) for vol in bhv_list: info = heketi_volume_info(h_node, h_url, vol, json=True) if info['blockinfo']['freesize'] >= pvc_size: self.skipTest( "Skip test case since there is atleast one BHV with free" " space {} greater than the default value {}".format( info['blockinfo']['freesize'], pvc_size)) # To verify if there is enough free space for two BHVs self.verify_free_space(2 * (default_bhv_size + 1)) sc_name = self.create_storage_class() self._dynamic_provisioning_block_with_bhv_cleanup( sc_name, pvc_size, bhv_list) self._dynamic_provisioning_block_with_bhv_cleanup( sc_name, pvc_size, bhv_list) bhv_post = len(get_block_hosting_volume_list(h_node, h_url)) err_msg = ("New BHVs were not created to satisfy the block PV requests" " No. of BHV before the test : {} \n" " No. of BHV after the test : {}".format( len(bhv_list), bhv_post)) self.assertEqual(bhv_post, (len(bhv_list) + 2), err_msg)
def test_volume_expansion_rebalance_brick(self): """Validate volume expansion with brick and check rebalance""" creation_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.assertNotEqual(creation_info, False, "Volume creation failed") volume_name = creation_info["name"] volume_id = creation_info["id"] free_space_after_creation = self.get_devices_summary_free_space() volume_info_before_expansion = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(volume_info_before_expansion, False, "Volume info for %s failed" % volume_id) heketi_vol_info_size_before_expansion = ( volume_info_before_expansion["size"]) self.get_brick_and_volume_status(volume_name) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) expansion_info = heketi_ops.heketi_volume_expand( self.heketi_client_node, self.heketi_server_url, volume_id, 5) self.assertNotEqual(expansion_info, False, "Volume expansion of %s failed" % volume_id) free_space_after_expansion = self.get_devices_summary_free_space() self.assertTrue( free_space_after_creation > free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) volume_info_after_expansion = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(volume_info_after_expansion, False, "Volume info failed for %s" % volume_id) heketi_vol_info_size_after_expansion = ( volume_info_after_expansion["size"]) difference_size = (heketi_vol_info_size_after_expansion - heketi_vol_info_size_before_expansion) self.assertTrue( difference_size > 0, "Size not increased after expansion of %s" % volume_id) self.get_brick_and_volume_status(volume_name) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) num_of_bricks_added = (num_of_bricks_after_expansion - num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_added, 3, "Number of bricks added is not 3 for %s" % volume_id) self.get_rebalance_status(volume_name) deletion_info = heketi_ops.heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(deletion_info, False, "Deletion of volume %s failed" % volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertTrue( free_space_after_deletion > free_space_after_expansion, "Free space is not reclaimed after volume deletion of %s" % volume_id)
def test_volume_expansion_no_free_space(self): """Validate volume expansion when there is no free space""" vol_size, expand_size, additional_devices_attached = None, 10, {} h_node, h_server_url = self.heketi_client_node, self.heketi_server_url # Get nodes info heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url) if len(heketi_node_id_list) < 3: self.skipTest("3 Heketi nodes are required.") # Disable 4th and other nodes for node_id in heketi_node_id_list[3:]: heketi_ops.heketi_node_disable(h_node, h_server_url, node_id) self.addCleanup( heketi_ops.heketi_node_enable, h_node, h_server_url, node_id) # Prepare first 3 nodes smallest_size = None err_msg = '' for node_id in heketi_node_id_list[0:3]: node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) # Disable second and other devices devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_id) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if (smallest_size is None or devices[0]["storage"]["free"] < smallest_size): smallest_size = devices[0]["storage"]["free"] for device in node_info["devices"][1:]: heketi_ops.heketi_device_disable( h_node, h_server_url, device["id"]) self.addCleanup( heketi_ops.heketi_device_enable, h_node, h_server_url, device["id"]) # Gather info about additional devices additional_device_name = None for gluster_server in self.gluster_servers: gluster_server_data = self.gluster_servers_info[gluster_server] g_manage = gluster_server_data["manage"] g_storage = gluster_server_data["storage"] if not (g_manage in node_info["hostnames"]["manage"] or g_storage in node_info["hostnames"]["storage"]): continue additional_device_name = (( gluster_server_data.get("additional_devices") or [''])[0]) break if not additional_device_name: err_msg += ("No 'additional_devices' are configured for " "'%s' node, which has following hostnames and " "IP addresses: %s.\n" % ( node_id, ', '.join(node_info["hostnames"]["manage"] + node_info["hostnames"]["storage"]))) continue heketi_ops.heketi_device_add( h_node, h_server_url, additional_device_name, node_id) additional_devices_attached.update( {node_id: additional_device_name}) # Schedule cleanup of the added devices for node_id in additional_devices_attached.keys(): node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) for device in node_info["devices"]: if device["name"] != additional_devices_attached[node_id]: continue self.addCleanup(self.detach_devices_attached, device["id"]) break else: self.fail("Could not find ID for added device on " "'%s' node." % node_id) if err_msg: self.skipTest(err_msg) # Temporary disable new devices self.disable_devices(additional_devices_attached) # Create volume and save info about it vol_size = int(smallest_size / (1024**2)) - 1 creation_info = heketi_ops.heketi_volume_create( h_node, h_server_url, vol_size, json=True) volume_name, volume_id = creation_info["name"], creation_info["id"] self.addCleanup( heketi_ops.heketi_volume_delete, h_node, h_server_url, volume_id, raise_on_error=False) volume_info_before_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) free_space_before_expansion = self.get_devices_summary_free_space() # Try to expand volume with not enough device space self.assertRaises( ExecutionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume self.enable_devices(additional_devices_attached) # Expand volume and validate results heketi_ops.heketi_volume_expand( h_node, h_server_url, volume_id, expand_size, json=True) free_space_after_expansion = self.get_devices_summary_free_space() self.assertGreater( free_space_before_expansion, free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) volume_info_after_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) self.assertGreater( volume_info_after_expansion["size"], volume_info_before_expansion["size"], "Size of %s not increased" % volume_id) self.assertGreater( num_of_bricks_after_expansion, num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0) # Delete volume and validate release of the used space heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertGreater( free_space_after_deletion, free_space_after_expansion, "Free space not reclaimed after deletion of volume %s" % volume_id)
def test_heketi_manual_cleanup_operation_in_bhv(self): """Validate heketi db cleanup will resolve the mismatch in the free size of the block hosting volume with failed block device create operations. """ bhv_size_before, bhv_size_after, vol_count = [], [], 5 ocp_node, g_node = self.ocp_master_node[0], self.gluster_servers[0] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Get nodes id list node_id_list = heketi_node_list(h_node, h_url) # Disable 4th and other nodes for node_id in node_id_list[3:]: heketi_node_disable(h_node, h_url, node_id) self.addCleanup(heketi_node_enable, h_node, h_url, node_id) # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 50: h_volume_size = 50 else: h_volume_size, vol_count = 1, free_space_available # Create BHV in case blockvolume size is greater than default BHV size default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Get BHV list h_bhv_list = get_block_hosting_volume_list(h_node, h_url).keys() self.assertTrue(h_bhv_list, "Failed to get the BHV list") # Get BHV size for bhv in h_bhv_list: vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bhv_vol_size_before = vol_info.get("freesize") bhv_size_before.append(bhv_vol_size_before) # Kill Tcmu-runner service services = ("tcmu-runner", "gluster-block-target", "gluster-blockd") kill_service_on_gluster_pod_or_node(ocp_node, "tcmu-runner", g_node) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') self.addCleanup(wait_for_service_status_on_gluster_pod_or_node, ocp_node, service, 'active', state, g_node) self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node, service, g_node) def run_async(cmd, hostname, raise_on_error=True): return g.run_async(host=hostname, command=cmd) # Create stale block volumes in async for count in range(vol_count): with mock.patch.object(json, 'loads', side_effect=(lambda j: j)): with mock.patch.object(command, 'cmd_run', side_effect=run_async): heketi_blockvolume_create(h_node, h_url, h_volume_size, json=True) # Wait for pending operation to get generated self._check_for_pending_operations(h_node, h_url) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') restart_service_on_gluster_pod_or_node(ocp_node, service, g_node) wait_for_service_status_on_gluster_pod_or_node( ocp_node, service, 'active', state, g_node) # Cleanup pending operation heketi_server_operation_cleanup(h_node, h_url) # wait for pending operation to get cleaned up for w in waiter.Waiter(timeout=120, interval=10): # Get BHV size for bhv in h_bhv_list: vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bhv_vol_size_after = vol_info.get("freesize") bhv_size_after.append(bhv_vol_size_after) if (set(bhv_size_before) == set(bhv_size_after)): break if w.expired: raise exceptions.ExecutionError( "Failed to Validate volume size Actual:{}," " Expected:{}".format(set(bhv_size_before), set(bhv_size_after)))
def test_volume_creation_of_size_greater_than_the_device_size(self): """Validate creation of a volume of size greater than the size of a device. """ h_node, h_url = self.heketi_client_node, self.heketi_server_url # Remove existing BHV to calculate freespace bhv_list = heketi_ops.get_block_hosting_volume_list(h_node, h_url) if bhv_list: for bhv in bhv_list: bhv_info = heketi_ops.heketi_volume_info(h_node, h_url, bhv, json=True) if bhv_info['blockinfo'].get('blockvolume') is None: heketi_ops.heketi_volume_delete(h_node, h_url, bhv) topology = heketi_ops.heketi_topology_info(h_node, h_url, json=True) nodes_free_space, nodes_ips = [], [] selected_nodes, selected_devices = [], [] cluster = topology['clusters'][0] node_count = len(cluster['nodes']) msg = ("At least 3 Nodes are required in cluster. " "But only %s Nodes are present." % node_count) if node_count < 3: self.skipTest(msg) online_nodes_count = 0 for node in cluster['nodes']: nodes_ips.append(node['hostnames']['storage'][0]) if node['state'] != 'online': continue online_nodes_count += 1 # Disable nodes after 3rd online nodes if online_nodes_count > 3: heketi_ops.heketi_node_disable(h_node, h_url, node['id']) self.addCleanup(heketi_ops.heketi_node_enable, h_node, h_url, node['id']) continue selected_nodes.append(node['id']) device_count = len(node['devices']) msg = ("At least 2 Devices are required on each Node." "But only %s Devices are present." % device_count) if device_count < 2: self.skipTest(msg) sel_devices, online_devices_count, free_space = [], 0, 0 for device in node['devices']: if device['state'] != 'online': continue online_devices_count += 1 # Disable devices after 2nd online devices if online_devices_count > 2: heketi_ops.heketi_device_disable(h_node, h_url, device['id']) self.addCleanup(heketi_ops.heketi_device_enable, h_node, h_url, device['id']) continue sel_devices.append(device['id']) free_space += int(device['storage']['free'] / (1024**2)) selected_devices.append(sel_devices) nodes_free_space.append(free_space) msg = ("At least 2 online Devices are required on each Node. " "But only %s Devices are online on Node: %s." % (online_devices_count, node['id'])) if online_devices_count < 2: self.skipTest(msg) msg = ("At least 3 online Nodes are required in cluster. " "But only %s Nodes are online in Cluster: %s." % (online_nodes_count, cluster['id'])) if online_nodes_count < 3: self.skipTest(msg) # Select node with minimum free space min_free_size = min(nodes_free_space) index = nodes_free_space.index(min_free_size) # Get max device size from selected node device_size = 0 for device in selected_devices[index]: device_info = heketi_ops.heketi_device_info(h_node, h_url, device, json=True) device_size = max(device_size, (int(device_info['storage']['total'] / (1024**2)))) vol_size = device_size + 1 if vol_size >= min_free_size: self.skipTest('Required free space %s is not available' % vol_size) # Create heketi volume with device size + 1 vol_info = self.create_heketi_volume_with_name_and_wait( name="volume_size_greater_than_device_size", size=vol_size, json=True) # Get gluster server IP's from heketi volume info glusterfs_servers = heketi_ops.get_vol_file_servers_and_hosts( h_node, h_url, vol_info['id']) # Verify gluster server IP's in heketi volume info msg = ("gluster IP's '%s' does not match with IP's '%s' found in " "heketi volume info" % (nodes_ips, glusterfs_servers['vol_servers'])) self.assertEqual(set(glusterfs_servers['vol_servers']), set(nodes_ips), msg) vol_name = vol_info['name'] gluster_v_info = self.get_gluster_vol_info(vol_name) # Verify replica count in gluster v info msg = "Volume %s is replica %s instead of replica 3" % ( vol_name, gluster_v_info['replicaCount']) self.assertEqual('3', gluster_v_info['replicaCount']) # Verify distCount in gluster v info msg = "Volume %s distCount is %s instead of distCount as 3" % ( vol_name, int(gluster_v_info['distCount'])) self.assertEqual( int(gluster_v_info['brickCount']) // 3, int(gluster_v_info['distCount']), msg) # Verify bricks count in gluster v info msg = ( "Volume %s does not have bricks count multiple of 3. It has %s" % (vol_name, gluster_v_info['brickCount'])) self.assertFalse(int(gluster_v_info['brickCount']) % 3, msg)
def test_heket_block_volume_info_with_gluster_block_volume_info(self): """Verify heketi block volume info with the backend gluster block volume info """ h_node, h_server = self.heketi_client_node, self.heketi_server_url vol_size = 1 h_block_vol = heketi_blockvolume_create(h_node, h_server, vol_size, auth=True, json=True) self.addCleanup(heketi_blockvolume_delete, h_node, h_server, h_block_vol["id"]) h_block_vol["blockvolume"]["hosts"].sort() h_block_vol_gbid = h_block_vol["blockvolume"]["username"] h_block_vol_name = h_block_vol["name"] h_block_vol_paswd = h_block_vol["blockvolume"]["password"] h_block_vol_hosts = h_block_vol["blockvolume"]["hosts"] h_block_host_vol_id = h_block_vol["blockhostingvolume"] h_block_vol_ha = h_block_vol["hacount"] # Fetch heketi blockhostingvolume info h_bhv_info = heketi_volume_info(h_node, h_server, h_block_host_vol_id, json=True) err_msg = "Failed to get heketi blockhostingvolume info for {}" self.assertTrue(h_bhv_info, err_msg.format(h_block_host_vol_id)) h_bhv_name = h_bhv_info['name'] err_msg = "Failed to get heketi BHV name for heketi blockvolume Id {}" self.assertTrue(h_bhv_name, err_msg.format(h_block_host_vol_id)) # Get gluster blockvolume list g_block_vol_list = get_block_list('auto_get_gluster_endpoint', h_bhv_name) err_msg = ("Failed to get gluter blockvolume list {}".format( g_block_vol_list)) self.assertTrue(g_block_vol_list, err_msg) err_msg = ( "Heketi block volume {} not present in gluster blockvolumes {}". format(h_block_vol_name, g_block_vol_list)) self.assertIn(h_block_vol_name, g_block_vol_list, err_msg) g_block_info = get_block_info('auto_get_gluster_endpoint', h_bhv_name, h_block_vol_name) g_block_info["EXPORTED ON"].sort() g_block_vol_hosts = g_block_info["EXPORTED ON"] g_block_vol_gbid = g_block_info["GBID"] g_block_vol_name = g_block_info["NAME"] g_block_vol_paswd = g_block_info["PASSWORD"] g_block_vol_id = g_block_info["VOLUME"][4:] g_block_vol_ha = g_block_info["HA"] # verfiy block device info and glusterblock volume info err_msg = ("Did not match {} from heketi {} and gluster {} side") self.assertEqual( h_block_vol_gbid, g_block_vol_gbid, err_msg.format("GBID", h_block_vol_gbid, g_block_vol_gbid, err_msg)) self.assertEqual( h_block_vol_name, g_block_vol_name, err_msg.format("blockvolume", h_block_vol_name, g_block_vol_name, err_msg)) self.assertEqual( h_block_vol_paswd, g_block_vol_paswd, err_msg.format("password", h_block_vol_paswd, g_block_vol_paswd, err_msg)) self.assertEqual( h_block_vol_hosts, g_block_vol_hosts, err_msg.format("hosts", h_block_vol_hosts, g_block_vol_hosts, err_msg)) self.assertEqual( h_block_host_vol_id, g_block_vol_id, err_msg.format("blockhost vol id", h_block_host_vol_id, g_block_vol_id, err_msg)) self.assertEqual( h_block_vol_ha, g_block_vol_ha, err_msg.format("ha", h_block_vol_ha, g_block_vol_ha, err_msg))
def test_volume_expansion_no_free_space(self): """Validate volume expansion when there is no free space""" vol_size, expand_size, additional_devices_attached = None, 10, {} h_node, h_server_url = self.heketi_client_node, self.heketi_server_url # Get nodes info heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url) if len(heketi_node_id_list) < 3: self.skipTest("3 Heketi nodes are required.") # Disable 4th and other nodes for node_id in heketi_node_id_list[3:]: heketi_ops.heketi_node_disable(h_node, h_server_url, node_id) self.addCleanup( heketi_ops.heketi_node_enable, h_node, h_server_url, node_id) # Prepare first 3 nodes smallest_size = None err_msg = '' for node_id in heketi_node_id_list[0:3]: node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) # Disable second and other devices devices = node_info["devices"] self.assertTrue( devices, "Node '%s' does not have devices." % node_id) if devices[0]["state"].strip().lower() != "online": self.skipTest("Test expects first device to be enabled.") if (smallest_size is None or devices[0]["storage"]["free"] < smallest_size): smallest_size = devices[0]["storage"]["free"] for device in node_info["devices"][1:]: heketi_ops.heketi_device_disable( h_node, h_server_url, device["id"]) self.addCleanup( heketi_ops.heketi_device_enable, h_node, h_server_url, device["id"]) # Gather info about additional devices additional_device_name = None for gluster_server in self.gluster_servers: gluster_server_data = self.gluster_servers_info[gluster_server] g_manage = gluster_server_data["manage"] g_storage = gluster_server_data["storage"] if not (g_manage in node_info["hostnames"]["manage"] or g_storage in node_info["hostnames"]["storage"]): continue additional_device_name = (( gluster_server_data.get("additional_devices") or [''])[0]) break if not additional_device_name: err_msg += ("No 'additional_devices' are configured for " "'%s' node, which has following hostnames and " "IP addresses: %s.\n" % ( node_id, ', '.join( node_info["hostnames"]["manage"] + node_info["hostnames"]["storage"]))) continue heketi_ops.heketi_device_add( h_node, h_server_url, additional_device_name, node_id) additional_devices_attached.update( {node_id: additional_device_name}) # Schedule cleanup of the added devices for node_id in additional_devices_attached.keys(): node_info = heketi_ops.heketi_node_info( h_node, h_server_url, node_id, json=True) for device in node_info["devices"]: if device["name"] != additional_devices_attached[node_id]: continue self.addCleanup(self.detach_devices_attached, device["id"]) break else: self.fail("Could not find ID for added device on " "'%s' node." % node_id) if err_msg: self.skipTest(err_msg) # Temporary disable new devices self.disable_devices(additional_devices_attached) # Create volume and save info about it vol_size = int(smallest_size / (1024**2)) - 1 creation_info = heketi_ops.heketi_volume_create( h_node, h_server_url, vol_size, json=True) volume_name, volume_id = creation_info["name"], creation_info["id"] self.addCleanup( heketi_ops.heketi_volume_delete, h_node, h_server_url, volume_id, raise_on_error=False) volume_info_before_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) free_space_before_expansion = self.get_devices_summary_free_space() # Try to expand volume with not enough device space self.assertRaises( AssertionError, heketi_ops.heketi_volume_expand, h_node, h_server_url, volume_id, expand_size) # Enable new devices to be able to expand our volume self.enable_devices(additional_devices_attached) # Expand volume and validate results heketi_ops.heketi_volume_expand( h_node, h_server_url, volume_id, expand_size, json=True) free_space_after_expansion = self.get_devices_summary_free_space() self.assertGreater( free_space_before_expansion, free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) self.get_brick_and_volume_status(volume_name) volume_info_after_expansion = heketi_ops.heketi_volume_info( h_node, h_server_url, volume_id, json=True) self.assertGreater( volume_info_after_expansion["size"], volume_info_before_expansion["size"], "Size of %s not increased" % volume_id) self.assertGreater( num_of_bricks_after_expansion, num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0) # Delete volume and validate release of the used space heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertGreater( free_space_after_deletion, free_space_after_expansion, "Free space not reclaimed after deletion of volume %s" % volume_id)
def test_expansion_of_block_hosting_volume_using_heketi(self): """Verify that after expanding block hosting volume we are able to consume the expanded space""" h_node = self.heketi_client_node h_url = self.heketi_server_url bvols_in_bhv = set([]) bvols_pv = set([]) BHVS = get_block_hosting_volume_list(h_node, h_url) free_BHVS_count = 0 for vol in BHVS.keys(): info = heketi_volume_info(h_node, h_url, vol, json=True) if info['blockinfo']['freesize'] > 0: free_BHVS_count += 1 if free_BHVS_count > 1: self.skipTest("Skip test case because there is more than one" " Block Hosting Volume with free space") # create block volume of 1gb bvol_info = heketi_blockvolume_create(h_node, h_url, 1, json=True) expand_size = 20 try: self.verify_free_space(expand_size) bhv = bvol_info['blockhostingvolume'] vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bvols_in_bhv.update(vol_info['blockinfo']['blockvolume']) finally: # cleanup BHV if there is only one block volume inside it if len(bvols_in_bhv) == 1: self.addCleanup(heketi_volume_delete, h_node, h_url, bhv, json=True) self.addCleanup(heketi_blockvolume_delete, h_node, h_url, bvol_info['id']) size = vol_info['size'] free_size = vol_info['blockinfo']['freesize'] bvol_count = int(free_size / expand_size) bricks = vol_info['bricks'] # create pvs to fill the BHV pvcs = self.create_and_wait_for_pvcs( pvc_size=(expand_size if bvol_count else free_size), pvc_amount=(bvol_count or 1), timeout=300) vol_expand = True for i in range(2): # get the vol ids from pvcs for pvc in pvcs: pv = get_pv_name_from_pvc(self.node, pvc) custom = r':.metadata.annotations."gluster\.org\/volume-id"' bvol_id = oc_get_custom_resource(self.node, 'pv', custom, pv) bvols_pv.add(bvol_id[0]) vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bvols = vol_info['blockinfo']['blockvolume'] bvols_in_bhv.update(bvols) self.assertEqual(bvols_pv, (bvols_in_bhv & bvols_pv)) # Expand BHV and verify bricks and size of BHV if vol_expand: vol_expand = False heketi_volume_expand(h_node, h_url, bhv, expand_size, json=True) vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) self.assertEqual(size + expand_size, vol_info['size']) self.assertFalse(len(vol_info['bricks']) % 3) self.assertLess(len(bricks), len(vol_info['bricks'])) # create more PVCs in expanded BHV pvcs = self.create_and_wait_for_pvcs(pvc_size=(expand_size - 1), pvc_amount=1)
def test_pv_resize_when_heketi_down(self): """Create a PVC and try to expand it when heketi is down, It should fail. After heketi is up, expand PVC should work. """ self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc() dc_name, pod_name = self.create_dc_with_pvc(pvc_name) pv_name = get_pv_name_from_pvc(self.node, pvc_name) custom = (r':metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi-volume-id"') vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0] h_vol_info = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, vol_id, json=True) # Bring the heketi POD down scale_dc_pod_amount_and_wait( self.node, self.heketi_dc_name, pod_amount=0) self.addCleanup( scale_dc_pod_amount_and_wait, self.node, self.heketi_dc_name, pod_amount=1) cmd = 'dd if=/dev/urandom of=/mnt/%s bs=614400k count=1' ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file1') self.assertFalse(ret, 'Not able to write file with err: %s' % err) wait_for_pod_be_ready(self.node, pod_name, 10, 5) resize_pvc(self.node, pvc_name, 2) wait_for_events( self.node, pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='VolumeResizeFailed') # Verify volume was not expanded vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) self.assertEqual(vol_info['gluster_vol_id'], h_vol_info['name']) self.assertEqual( len(vol_info['bricks']['brick']), len(h_vol_info['bricks'])) # Bring the heketi POD up scale_dc_pod_amount_and_wait( self.node, self.heketi_dc_name, pod_amount=1) # Verify volume expansion verify_pvc_size(self.node, pvc_name, 2) vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) self.assertFalse(len(vol_info['bricks']['brick']) % 3) self.assertLess( len(h_vol_info['bricks']), len(vol_info['bricks']['brick'])) # Wait for remount after expansion for w in waiter.Waiter(timeout=30, interval=5): ret, out, err = oc_rsh( self.node, pod_name, "df -Ph /mnt | awk '{print $2}' | tail -1") self.assertFalse(ret, 'Failed with err: %s and Output: %s' % ( err, out)) if out.strip() == '2.0G': break # Write data making sure we have more space than it was ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file2') self.assertFalse(ret, 'Not able to write file with err: %s' % err) # Verify pod is running wait_for_pod_be_ready(self.node, pod_name, 10, 5)
def test_volume_expansion_rebalance_brick(self): """Validate volume expansion with brick and check rebalance""" creation_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.assertNotEqual(creation_info, False, "Volume creation failed") volume_name = creation_info["name"] volume_id = creation_info["id"] free_space_after_creation = self.get_devices_summary_free_space() volume_info_before_expansion = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(volume_info_before_expansion, False, "Volume info for %s failed" % volume_id) heketi_vol_info_size_before_expansion = ( volume_info_before_expansion["size"]) self.get_brick_and_volume_status(volume_name) num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name) expansion_info = heketi_ops.heketi_volume_expand( self.heketi_client_node, self.heketi_server_url, volume_id, 5) self.assertNotEqual(expansion_info, False, "Volume expansion of %s failed" % volume_id) free_space_after_expansion = self.get_devices_summary_free_space() self.assertTrue( free_space_after_creation > free_space_after_expansion, "Free space not consumed after expansion of %s" % volume_id) volume_info_after_expansion = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(volume_info_after_expansion, False, "Volume info failed for %s" % volume_id) heketi_vol_info_size_after_expansion = ( volume_info_after_expansion["size"]) difference_size = (heketi_vol_info_size_after_expansion - heketi_vol_info_size_before_expansion) self.assertTrue( difference_size > 0, "Size not increased after expansion of %s" % volume_id) self.get_brick_and_volume_status(volume_name) num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name) num_of_bricks_added = ( num_of_bricks_after_expansion - num_of_bricks_before_expansion) self.assertEqual( num_of_bricks_added, 3, "Number of bricks added is not 3 for %s" % volume_id) self.get_rebalance_status(volume_name) deletion_info = heketi_ops.heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) self.assertNotEqual(deletion_info, False, "Deletion of volume %s failed" % volume_id) free_space_after_deletion = self.get_devices_summary_free_space() self.assertTrue( free_space_after_deletion > free_space_after_expansion, "Free space is not reclaimed after volume deletion of %s" % volume_id)
def test_create_max_num_blockhostingvolumes(self): num_of_bv = 10 new_bhv_list, bv_list, g_nodes = [], [], [] free_space, nodenum = get_total_free_space(self.heketi_client_node, self.heketi_server_url) if nodenum < 3: self.skipTest("Skip the test case since number of" "online nodes is less than 3.") free_space_available = int(free_space / nodenum) default_bhv_size = get_default_block_hosting_volume_size( self.heketi_client_node, self.heketi_dc_name) # Get existing list of BHV's existing_bhv_list = get_block_hosting_volume_list( self.heketi_client_node, self.heketi_server_url) # Skip the test if available space is less than default_bhv_size if free_space_available < default_bhv_size: self.skipTest("Skip the test case since free_space_available %s" "is less than space_required_for_bhv %s ." % (free_space_available, default_bhv_size)) # Create BHV's while free_space_available > default_bhv_size: block_host_create_info = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, default_bhv_size, json=True, block=True) if block_host_create_info["id"] not in existing_bhv_list.keys(): new_bhv_list.append(block_host_create_info["id"]) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, block_host_create_info["id"], raise_on_error=False) block_vol_size = int( block_host_create_info["blockinfo"]["freesize"] / num_of_bv) # Create specified number of BV's in BHV's created for i in range(0, num_of_bv): block_vol = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, block_vol_size, json=True, ha=3, auth=True) self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, block_vol["id"], raise_on_error=False) bv_list.append(block_vol["id"]) free_space_available = int(free_space_available - default_bhv_size) # Get gluster node ips h_nodes_ids = heketi_node_list(self.heketi_client_node, self.heketi_server_url) for h_node in h_nodes_ids[:2]: g_node = heketi_node_info(self.heketi_client_node, self.heketi_server_url, h_node, json=True) g_nodes.append(g_node['hostnames']['manage'][0]) # Check if there is no crash in gluster related services & heketi services = (("glusterd", "running"), ("gluster-blockd", "running"), ("tcmu-runner", "running"), ("gluster-block-target", "exited")) for g_node in g_nodes: for service, state in services: wait_for_service_status_on_gluster_pod_or_node( self.ocp_client[0], service, 'active', state, g_node, raise_on_error=False) out = hello_heketi(self.heketi_client_node, self.heketi_server_url) self.assertTrue( out, "Heketi server %s is not alive" % self.heketi_server_url) # Delete all the BHV's and BV's created for bv_volume in bv_list: heketi_blockvolume_delete(self.heketi_client_node, self.heketi_server_url, bv_volume) # Check if any blockvolume exist in heketi & gluster for bhv_volume in new_bhv_list[:]: heketi_vol_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, bhv_volume, json=True) self.assertNotIn("blockvolume", heketi_vol_info["blockinfo"].keys()) gluster_vol_info = get_block_list('auto_get_gluster_endpoint', volname="vol_%s" % bhv_volume) self.assertIsNotNone(gluster_vol_info, "Failed to get volume info %s" % bhv_volume) new_bhv_list.remove(bhv_volume) for blockvol in gluster_vol_info: self.assertNotIn("blockvol_", blockvol) heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, bhv_volume) # Check if all blockhosting volumes are deleted from heketi self.assertFalse(new_bhv_list)
def test_expansion_of_block_hosting_volume_using_heketi(self): """Verify that after expanding block hosting volume we are able to consume the expanded space""" h_node = self.heketi_client_node h_url = self.heketi_server_url bvols_in_bhv = set([]) bvols_pv = set([]) BHVS = get_block_hosting_volume_list(h_node, h_url) free_BHVS_count = 0 for vol in BHVS.keys(): info = heketi_volume_info(h_node, h_url, vol, json=True) if info['blockinfo']['freesize'] > 0: free_BHVS_count += 1 if free_BHVS_count > 1: self.skipTest("Skip test case because there is more than one" " Block Hosting Volume with free space") # create block volume of 1gb bvol_info = heketi_blockvolume_create(h_node, h_url, 1, json=True) expand_size = 20 try: self.verify_free_space(expand_size) bhv = bvol_info['blockhostingvolume'] vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bvols_in_bhv.update(vol_info['blockinfo']['blockvolume']) finally: # cleanup BHV if there is only one block volume inside it if len(bvols_in_bhv) == 1: self.addCleanup( heketi_volume_delete, h_node, h_url, bhv, json=True) self.addCleanup( heketi_blockvolume_delete, h_node, h_url, bvol_info['id']) size = vol_info['size'] free_size = vol_info['blockinfo']['freesize'] bvol_count = int(free_size / expand_size) bricks = vol_info['bricks'] # create pvs to fill the BHV pvcs = self.create_and_wait_for_pvcs( pvc_size=(expand_size if bvol_count else free_size), pvc_amount=(bvol_count or 1), timeout=300) vol_expand = True for i in range(2): # get the vol ids from pvcs for pvc in pvcs: pv = get_pv_name_from_pvc(self.node, pvc) custom = r':.metadata.annotations."gluster\.org\/volume-id"' bvol_id = oc_get_custom_resource(self.node, 'pv', custom, pv) bvols_pv.add(bvol_id[0]) vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bvols = vol_info['blockinfo']['blockvolume'] bvols_in_bhv.update(bvols) self.assertEqual(bvols_pv, (bvols_in_bhv & bvols_pv)) # Expand BHV and verify bricks and size of BHV if vol_expand: vol_expand = False heketi_volume_expand( h_node, h_url, bhv, expand_size, json=True) vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) self.assertEqual(size + expand_size, vol_info['size']) self.assertFalse(len(vol_info['bricks']) % 3) self.assertLess(len(bricks), len(vol_info['bricks'])) # create more PVCs in expanded BHV pvcs = self.create_and_wait_for_pvcs( pvc_size=(expand_size - 1), pvc_amount=1)
def test_verify_create_heketi_volumes_pending_entries_in_db( self, vol_type): """Verify pending entries of file/block volumes in db during volumes creation from heketi side """ # Create large volumes to observe the pending operations vol_count, h_vol_creation_async_op = 3, [] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Verify file/block volumes pending operation before creation, h_db_check_before = heketi_db_check(h_node, h_url) h_db_check_vol_before = (h_db_check_before.get( "{}volumes".format(vol_type))) # Delete heketi pod to clean db operations if (h_db_check_vol_before.get("pending") or h_db_check_before.get("bricks").get("pending")): self._respin_heketi_pod() # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 30: h_volume_size = 30 else: h_volume_size, vol_count = 1, free_space_available # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add cleanup function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Create BHV in case blockvolume size is greater than default BHV size if vol_type: default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Temporary replace g.run with g.async_run in heketi_blockvolume_create # func to be able to run it in background.Also, avoid parsing the # output as it won't be json at that moment. Parse it after reading # the async operation results. def run_async(cmd, hostname, raise_on_error=True): return g.run_async(host=hostname, command=cmd) for count in range(vol_count): with mock.patch.object(json, 'loads', side_effect=(lambda j: j)): with mock.patch.object(command, 'cmd_run', side_effect=run_async): h_vol_creation_async_op.append( eval("heketi_{}volume_create".format(vol_type))( h_node, h_url, h_volume_size, json=True)) # Check for pending operations for w in waiter.Waiter(timeout=120, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if h_db_check_vol.get("pending"): h_db_check_bricks = h_db_check.get("bricks") break if w.expired: raise exceptions.ExecutionError( "No any pending operations found during {}volumes creation " "{}".format(vol_type, h_db_check_vol.get("pending"))) # Verify bricks pending operation during creation if not vol_type: self.assertTrue(h_db_check_bricks.get("pending"), "Expecting at least one bricks pending count") self.assertFalse( h_db_check_bricks.get("pending") % 3, "Expecting bricks pending count to be multiple of 3 but " "found {}".format(h_db_check_bricks.get("pending"))) # Wait for all counts of pending operations to be zero for w in waiter.Waiter(timeout=300, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if not h_db_check_vol.get("pending"): break if w.expired: raise exceptions.ExecutionError( "Expecting no pending operations after 300 sec but " "found {} operation".format(h_db_check_vol.get("pending"))) # Get heketi server DB details h_db_check_after = heketi_db_check(h_node, h_url) h_db_check_vol_after = (h_db_check_after.get( "{}volumes".format(vol_type))) h_db_check_bricks_after = h_db_check_after.get("bricks") # Verify if initial and final file/block volumes are same act_vol_count = h_db_check_vol_after.get("total") exp_vol_count = h_db_check_vol_before.get("total") + vol_count err_msg = ( "Actual {} and expected {} {}volume counts are not matched".format( act_vol_count, exp_vol_count, vol_type)) self.assertEqual(act_vol_count, exp_vol_count, err_msg) # Verify if initial and final bricks are same for file volume volumes = heketi_volume_list(h_node, h_url, json=True).get("volumes") new_volumes = list(set(volumes) - set(existing_volumes)) exp_brick_count = 0 for volume in new_volumes: vol_info = heketi_volume_info(h_node, h_url, volume, json=True) exp_brick_count += len(vol_info.get("bricks")) err_msg = "Actual {} and expected {} bricks counts are not matched" act_brick_count = h_db_check_bricks_after.get("total") self.assertEqual(act_brick_count, exp_brick_count, err_msg.format(act_brick_count, exp_brick_count))
def test_pv_resize_when_heketi_down(self): """Create a PVC and try to expand it when heketi is down, It should fail. After heketi is up, expand PVC should work. """ self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc() dc_name, pod_name = self.create_dc_with_pvc(pvc_name) pv_name = get_pv_name_from_pvc(self.node, pvc_name) custom = (r':metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi-volume-id"') vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0] h_vol_info = heketi_ops.heketi_volume_info(self.heketi_client_node, self.heketi_server_url, vol_id, json=True) # Bring the heketi POD down scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, pod_amount=0) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, self.heketi_dc_name, pod_amount=1) cmd = 'dd if=/dev/urandom of=/mnt/%s bs=614400k count=1' ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file1') self.assertFalse(ret, 'Not able to write file with err: %s' % err) wait_for_pod_be_ready(self.node, pod_name, 10, 5) resize_pvc(self.node, pvc_name, 2) wait_for_events(self.node, pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='VolumeResizeFailed') # Verify volume was not expanded vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) self.assertEqual(vol_info['gluster_vol_id'], h_vol_info['name']) self.assertEqual(len(vol_info['bricks']['brick']), len(h_vol_info['bricks'])) # Bring the heketi POD up scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, pod_amount=1) # Verify volume expansion verify_pvc_size(self.node, pvc_name, 2) vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) self.assertFalse(len(vol_info['bricks']['brick']) % 3) self.assertLess(len(h_vol_info['bricks']), len(vol_info['bricks']['brick'])) # Wait for remount after expansion for w in waiter.Waiter(timeout=30, interval=5): ret, out, err = oc_rsh(self.node, pod_name, "df -Ph /mnt | awk '{print $2}' | tail -1") self.assertFalse(ret, 'Failed with err: %s and Output: %s' % (err, out)) if out.strip() == '2.0G': break # Write data making sure we have more space than it was ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file2') self.assertFalse(ret, 'Not able to write file with err: %s' % err) # Verify pod is running wait_for_pod_be_ready(self.node, pod_name, 10, 5)
def test_targetcli_when_block_hosting_volume_down(self): """Validate no inconsistencies occur in targetcli when block volumes are created with one block hosting volume down.""" h_node, h_server = self.heketi_client_node, self.heketi_server_url cmd = ("targetcli ls | egrep '%s' || echo unavailable") error_msg = ("targetcli has inconsistencies when block devices are " "created with one block hosting volume %s is down") # Delete BHV which has no BV or fill it completely bhv_list = get_block_hosting_volume_list(h_node, h_server).keys() for bhv in bhv_list: bhv_info = heketi_volume_info(h_node, h_server, bhv, json=True) if not bhv_info["blockinfo"].get("blockvolume", []): heketi_volume_delete(h_node, h_server, bhv) continue free_size = bhv_info["blockinfo"].get("freesize", 0) if free_size: bv = heketi_volume_create(h_node, h_server, free_size, json=True) self.addCleanup(heketi_volume_delete, h_node, h_server, bv["id"]) # Create BV bv = heketi_blockvolume_create(h_node, h_server, 2, json=True) self.addCleanup(heketi_blockvolume_delete, h_node, h_server, bv["id"]) # Bring down BHV bhv_name = get_block_hosting_volume_name(h_node, h_server, bv["id"]) ret, out, err = volume_stop("auto_get_gluster_endpoint", bhv_name) if ret != 0: err_msg = "Failed to stop gluster volume %s. error: %s" % ( bhv_name, err) g.log.error(err_msg) raise AssertionError(err_msg) self.addCleanup(podcmd.GlustoPod()(volume_start), "auto_get_gluster_endpoint", bhv_name) ocp_node = self.ocp_master_node[0] gluster_block_svc = "gluster-block-target" self.addCleanup(wait_for_service_status_on_gluster_pod_or_node, ocp_node, gluster_block_svc, "active", "exited", gluster_node=self.gluster_servers[0]) self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node, gluster_block_svc, self.gluster_servers[0]) for condition in ("continue", "break"): restart_service_on_gluster_pod_or_node( ocp_node, gluster_block_svc, gluster_node=self.gluster_servers[0]) wait_for_service_status_on_gluster_pod_or_node( ocp_node, gluster_block_svc, "active", "exited", gluster_node=self.gluster_servers[0]) targetcli = cmd_run_on_gluster_pod_or_node(ocp_node, cmd % bv["id"], self.gluster_servers[0]) if condition == "continue": self.assertEqual(targetcli, "unavailable", error_msg % bhv_name) else: self.assertNotEqual(targetcli, "unavailable", error_msg % bhv_name) break # Bring up the same BHV ret, out, err = volume_start("auto_get_gluster_endpoint", bhv_name) if ret != 0: err = "Failed to start gluster volume %s on %s. error: %s" % ( bhv_name, h_node, err) raise exceptions.ExecutionError(err)