def test_sc_glusterfile_missing_parameter(self, parameter): """Validate glusterfile storage with missing parameters""" node, sc = self.ocp_master_node[0], self.sc secret_name = self.create_secret() parameters = {'resturl': sc['resturl'], 'restuser': sc['restuser']} if parameter == 'secretName': parameters['secretName'] = secret_name elif parameter == 'secretNamespace': parameters['secretNamespace'] = sc['secretnamespace'] sc_name = oc_create_sc(node, **parameters) self.addCleanup(oc_delete, node, 'sc', sc_name) # Create PVC pvc_name = oc_create_pvc(node, sc_name) self.addCleanup(wait_for_resource_absence, node, 'pvc', pvc_name) self.addCleanup(oc_delete, node, 'pvc', pvc_name) # Wait for event with error wait_for_events(node, obj_name=pvc_name, obj_type='PersistentVolumeClaim', event_reason='ProvisioningFailed') # Verify PVC did not get bound with self.assertRaises(ExecutionError): verify_pvc_status_is_bound(node, pvc_name, timeout=1)
def _pv_resize(self, exceed_free_space): dir_path = "/mnt" pvc_size_gb = 1 available_size_gb = self._available_disk_free_space() # Create PVC self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb) # Create DC with POD and attached PVC to it dc_name = oc_create_app_dc_with_io( self.node, pvc_name, image=self.io_container_image_cirros) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) if exceed_free_space: exceed_size = available_size_gb + 10 # Try to expand existing PVC exceeding free space resize_pvc(self.node, pvc_name, exceed_size) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeFailed') # Check that app POD is up and runnig then try to write data wait_for_pod_be_ready(self.node, pod_name) cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to write data after failed attempt to expand PVC.") else: # Expand existing PVC using all the available free space expand_size_gb = available_size_gb - pvc_size_gb resize_pvc(self.node, pvc_name, expand_size_gb) verify_pvc_size(self.node, pvc_name, expand_size_gb) pv_name = get_pv_name_from_pvc(self.node, pvc_name) verify_pv_size(self.node, pv_name, expand_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeSuccessful') # Recreate app POD oc_delete(self.node, 'pod', pod_name) wait_for_resource_absence(self.node, 'pod', pod_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Write data on the expanded PVC cmd = ("dd if=/dev/urandom of=%s/autotest " "bs=1M count=1025" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual(ret, 0, "Failed to write data on the expanded PVC")
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_neg( self): """Verify that block volume creation fails when we create block volume of size greater than the default size of BHV. Verify that block volume creation succeed when we create BHV of size greater than the default size of BHV. """ default_bhv_size = get_default_block_hosting_volume_size( self.node, self.heketi_dc_name) reserve_size = default_bhv_size * 0.02 reserve_size = int(math.ceil(reserve_size)) self.verify_free_space(default_bhv_size + reserve_size + 2) with self.assertRaises(ExecutionError): # create a block vol greater than default BHV size bvol_info = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + 1), json=True) self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, bvol_info['id']) sc_name = self.create_storage_class() # create a block pvc greater than default BHV size pvc_name = oc_create_pvc( self.node, sc_name, pvc_size=(default_bhv_size + 1)) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) wait_for_events( self.node, pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='ProvisioningFailed') # create block hosting volume greater than default BHV size vol_info = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + reserve_size + 2), block=True, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # Cleanup PVC before block hosting volume to avoid failures self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) verify_pvc_status_is_bound(self.node, pvc_name)
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_neg( self): """Verify that block volume creation fails when we create block volume of size greater than the default size of BHV. Verify that block volume creation succeed when we create BHV of size greater than the default size of BHV. """ default_bhv_size = get_default_block_hosting_volume_size( self.node, self.heketi_dc_name) reserve_size = default_bhv_size * 0.02 reserve_size = int(math.ceil(reserve_size)) self.verify_free_space(default_bhv_size + reserve_size + 2) with self.assertRaises(AssertionError): # create a block vol greater than default BHV size bvol_info = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + 1), json=True) self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, bvol_info['id']) sc_name = self.create_storage_class() # create a block pvc greater than default BHV size pvc_name = oc_create_pvc( self.node, sc_name, pvc_size=(default_bhv_size + 1)) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) wait_for_events( self.node, pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='ProvisioningFailed') # create block hosting volume greater than default BHV size vol_info = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + reserve_size + 2), block=True, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # Cleanup PVC before block hosting volume to avoid failures self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) verify_pvc_status_is_bound(self.node, pvc_name)
def test_creation_of_pvc_when_one_node_is_down(self): """Test PVC creation when one node is down than hacount""" node_count = len(self.gluster_servers) # Check at least 4 nodes available. if node_count < 4: self.skipTest("At least 4 nodes are required, found %s." % node_count) # Skip test if not able to connect to Cloud Provider try: find_vm_name_by_ip_or_hostname(self.node) except (NotImplementedError, ConfigError) as e: self.skipTest(e) # Get gluster node on which heketi pod is not scheduled heketi_node = oc_get_custom_resource(self.node, 'pod', custom='.:status.hostIP', selector='deploymentconfig=%s' % self.heketi_dc_name)[0] gluster_node = random.sample( (set(self.gluster_servers) - set(heketi_node)), 1)[0] gluster_hostname = self.gluster_servers_info[gluster_node]["manage"] # Get VM name by VM hostname vm_name = find_vm_name_by_ip_or_hostname(gluster_node) # Power off one of the nodes self.power_off_gluster_node_vm(vm_name, gluster_hostname) # Create a PVC with SC of hacount equal to node count sc_name = self.create_storage_class(hacount=node_count) pvc_name = oc_create_pvc(self.node, sc_name=sc_name) self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) events = wait_for_events(self.node, obj_name=pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='ProvisioningFailed', timeout=180) error = 'insufficient block hosts online' err_msg = ("Haven't found expected error message containing " "following string: \n%s\nEvents: %s" % (error, events)) self.assertTrue( list(filter((lambda e: error in e['message']), events)), err_msg) # Create a PVC with SC of hacount less than node count sc_name = self.create_storage_class(hacount=(node_count - 1)) self.create_and_wait_for_pvc(sc_name=sc_name)
def _pv_resize(self, exceed_free_space): dir_path = "/mnt" pvc_size_gb, min_free_space_gb = 1, 3 # Get available free space disabling redundant devices and nodes heketi_url = self.heketi_server_url node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_id_list) nodes = {} min_free_space = min_free_space_gb * 1024**2 for node_id in node_id_list: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue if len(nodes) > 2: out = heketi_ops.heketi_node_disable(self.heketi_client_node, heketi_url, node_id) self.assertTrue(out) self.addCleanup(heketi_ops.heketi_node_enable, self.heketi_client_node, heketi_url, node_id) for device in node_info['devices']: if device['state'].lower() != 'online': continue free_space = device['storage']['free'] if (node_id in nodes.keys() or free_space < min_free_space): out = heketi_ops.heketi_device_disable( self.heketi_client_node, heketi_url, device['id']) self.assertTrue(out) self.addCleanup(heketi_ops.heketi_device_enable, self.heketi_client_node, heketi_url, device['id']) continue nodes[node_id] = free_space if len(nodes) < 3: raise self.skipTest("Could not find 3 online nodes with, " "at least, 1 online device having free space " "bigger than %dGb." % min_free_space_gb) # Calculate maximum available size for PVC available_size_gb = int(min(nodes.values()) / (1024**2)) # Create PVC self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb) # Create DC with POD and attached PVC to it dc_name = oc_create_app_dc_with_io(self.node, pvc_name) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) if exceed_free_space: # Try to expand existing PVC exceeding free space resize_pvc(self.node, pvc_name, available_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeFailed') # Check that app POD is up and runnig then try to write data wait_for_pod_be_ready(self.node, pod_name) cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to write data after failed attempt to expand PVC.") else: # Expand existing PVC using all the available free space expand_size_gb = available_size_gb - pvc_size_gb resize_pvc(self.node, pvc_name, expand_size_gb) verify_pvc_size(self.node, pvc_name, expand_size_gb) pv_name = get_pv_name_from_pvc(self.node, pvc_name) verify_pv_size(self.node, pv_name, expand_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeSuccessful') # Recreate app POD oc_delete(self.node, 'pod', pod_name) wait_for_resource_absence(self.node, 'pod', pod_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Write data on the expanded PVC cmd = ("dd if=/dev/urandom of=%s/autotest " "bs=1M count=1025" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual(ret, 0, "Failed to write data on the expanded PVC")
def test_pv_resize_when_heketi_down(self): """Create a PVC and try to expand it when heketi is down, It should fail. After heketi is up, expand PVC should work. """ self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc() dc_name, pod_name = self.create_dc_with_pvc(pvc_name) pv_name = get_pv_name_from_pvc(self.node, pvc_name) custom = (r':metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi-volume-id"') vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0] h_vol_info = heketi_ops.heketi_volume_info(self.heketi_client_node, self.heketi_server_url, vol_id, json=True) # Bring the heketi POD down scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, pod_amount=0) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, self.heketi_dc_name, pod_amount=1) cmd = 'dd if=/dev/urandom of=/mnt/%s bs=614400k count=1' ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file1') self.assertFalse(ret, 'Not able to write file with err: %s' % err) wait_for_pod_be_ready(self.node, pod_name, 10, 5) resize_pvc(self.node, pvc_name, 2) wait_for_events(self.node, pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='VolumeResizeFailed') # Verify volume was not expanded vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) self.assertEqual(vol_info['gluster_vol_id'], h_vol_info['name']) self.assertEqual(len(vol_info['bricks']['brick']), len(h_vol_info['bricks'])) # Bring the heketi POD up scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, pod_amount=1) # Verify volume expansion verify_pvc_size(self.node, pvc_name, 2) vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) self.assertFalse(len(vol_info['bricks']['brick']) % 3) self.assertLess(len(h_vol_info['bricks']), len(vol_info['bricks']['brick'])) # Wait for remount after expansion for w in waiter.Waiter(timeout=30, interval=5): ret, out, err = oc_rsh(self.node, pod_name, "df -Ph /mnt | awk '{print $2}' | tail -1") self.assertFalse(ret, 'Failed with err: %s and Output: %s' % (err, out)) if out.strip() == '2.0G': break # Write data making sure we have more space than it was ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file2') self.assertFalse(ret, 'Not able to write file with err: %s' % err) # Verify pod is running wait_for_pod_be_ready(self.node, pod_name, 10, 5)
def create_sc_with_parameter(self, vol_type, success=False, parameter={}): """creates storage class, pvc and validates event Args: vol_type (str): storage type either gluster file or block success (bool): if True check for successfull else failure for pvc creation event parameter (dict): dictionary with storage class parameters """ if vol_type == "glusterfile": sc = self.storage_classes.get( 'storage_class1', self.storage_classes.get('file_storage_class')) # Create secret file for usage in storage class self.secret_name = oc_create_secret( self.ocp_master_node[0], namespace=sc.get('secretnamespace', 'default'), data_key=self.heketi_cli_key, secret_type=sc.get('provisioner', 'kubernetes.io/glusterfs')) self.addCleanup(oc_delete, self.ocp_master_node[0], 'secret', self.secret_name) sc_parameter = { "secretnamespace": sc['secretnamespace'], "secretname": self.secret_name, "volumetype": "replicate:3" } elif vol_type == "glusterblock": sc = self.storage_classes.get( 'storage_class2', self.storage_classes.get('block_storage_class')) # Create secret file for usage in storage class self.secret_name = oc_create_secret( self.ocp_master_node[0], namespace=sc.get('restsecretnamespace', 'default'), data_key=self.heketi_cli_key, secret_type=sc.get('provisioner', 'gluster.org/glusterblock')) self.addCleanup(oc_delete, self.ocp_master_node[0], 'secret', self.secret_name) sc_parameter = { "provisioner": self.get_block_provisioner_for_sc(), "restsecretnamespace": sc['restsecretnamespace'], "restsecretname": self.secret_name, "hacount": sc['hacount'] } else: err_msg = "invalid vol_type %s" % vol_type g.log.error(err_msg) raise AssertionError(err_msg) sc_parameter['resturl'] = sc['resturl'] sc_parameter['restuser'] = sc['restuser'] sc_parameter.update(parameter) # Create storage class self.sc_name = oc_create_sc(self.ocp_master_node[0], **sc_parameter) self.addCleanup(oc_delete, self.ocp_master_node[0], 'sc', self.sc_name) # Create PVC self.pvc_name = oc_create_pvc(self.ocp_client[0], self.sc_name) self.addCleanup(wait_for_resource_absence, self.ocp_master_node[0], 'pvc', self.pvc_name) self.addCleanup(oc_delete, self.ocp_master_node[0], 'pvc', self.pvc_name) # Wait for event with error event_reason = 'ProvisioningFailed' if success: event_reason = 'ProvisioningSucceeded' wait_for_events(self.ocp_master_node[0], obj_name=self.pvc_name, obj_type='PersistentVolumeClaim', event_reason=event_reason)
def _pv_resize(self, exceed_free_space): dir_path = "/mnt" pvc_size_gb, min_free_space_gb = 1, 3 # Get available free space disabling redundant devices and nodes heketi_url = self.heketi_server_url node_id_list = heketi_ops.heketi_node_list( self.heketi_client_node, heketi_url) self.assertTrue(node_id_list) nodes = {} min_free_space = min_free_space_gb * 1024**2 for node_id in node_id_list: node_info = heketi_ops.heketi_node_info( self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue if len(nodes) > 2: out = heketi_ops.heketi_node_disable( self.heketi_client_node, heketi_url, node_id) self.assertTrue(out) self.addCleanup( heketi_ops.heketi_node_enable, self.heketi_client_node, heketi_url, node_id) for device in node_info['devices']: if device['state'].lower() != 'online': continue free_space = device['storage']['free'] if (node_id in nodes.keys() or free_space < min_free_space): out = heketi_ops.heketi_device_disable( self.heketi_client_node, heketi_url, device['id']) self.assertTrue(out) self.addCleanup( heketi_ops.heketi_device_enable, self.heketi_client_node, heketi_url, device['id']) continue nodes[node_id] = free_space if len(nodes) < 3: raise self.skipTest( "Could not find 3 online nodes with, " "at least, 1 online device having free space " "bigger than %dGb." % min_free_space_gb) # Calculate maximum available size for PVC available_size_gb = int(min(nodes.values()) / (1024**2)) # Create PVC self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb) # Create DC with POD and attached PVC to it dc_name = oc_create_app_dc_with_io(self.node, pvc_name) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) if exceed_free_space: # Try to expand existing PVC exceeding free space resize_pvc(self.node, pvc_name, available_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeFailed') # Check that app POD is up and runnig then try to write data wait_for_pod_be_ready(self.node, pod_name) cmd = ( "dd if=/dev/urandom of=%s/autotest bs=100K count=1" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to write data after failed attempt to expand PVC.") else: # Expand existing PVC using all the available free space expand_size_gb = available_size_gb - pvc_size_gb resize_pvc(self.node, pvc_name, expand_size_gb) verify_pvc_size(self.node, pvc_name, expand_size_gb) pv_name = get_pv_name_from_pvc(self.node, pvc_name) verify_pv_size(self.node, pv_name, expand_size_gb) wait_for_events( self.node, obj_name=pvc_name, event_reason='VolumeResizeSuccessful') # Recreate app POD oc_delete(self.node, 'pod', pod_name) wait_for_resource_absence(self.node, 'pod', pod_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Write data on the expanded PVC cmd = ("dd if=/dev/urandom of=%s/autotest " "bs=1M count=1025" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to write data on the expanded PVC")
def test_pv_resize_when_heketi_down(self): """Create a PVC and try to expand it when heketi is down, It should fail. After heketi is up, expand PVC should work. """ self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc() dc_name, pod_name = self.create_dc_with_pvc(pvc_name) pv_name = get_pv_name_from_pvc(self.node, pvc_name) custom = (r':metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi-volume-id"') vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0] h_vol_info = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, vol_id, json=True) # Bring the heketi POD down scale_dc_pod_amount_and_wait( self.node, self.heketi_dc_name, pod_amount=0) self.addCleanup( scale_dc_pod_amount_and_wait, self.node, self.heketi_dc_name, pod_amount=1) cmd = 'dd if=/dev/urandom of=/mnt/%s bs=614400k count=1' ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file1') self.assertFalse(ret, 'Not able to write file with err: %s' % err) wait_for_pod_be_ready(self.node, pod_name, 10, 5) resize_pvc(self.node, pvc_name, 2) wait_for_events( self.node, pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='VolumeResizeFailed') # Verify volume was not expanded vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) self.assertEqual(vol_info['gluster_vol_id'], h_vol_info['name']) self.assertEqual( len(vol_info['bricks']['brick']), len(h_vol_info['bricks'])) # Bring the heketi POD up scale_dc_pod_amount_and_wait( self.node, self.heketi_dc_name, pod_amount=1) # Verify volume expansion verify_pvc_size(self.node, pvc_name, 2) vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name) self.assertFalse(len(vol_info['bricks']['brick']) % 3) self.assertLess( len(h_vol_info['bricks']), len(vol_info['bricks']['brick'])) # Wait for remount after expansion for w in waiter.Waiter(timeout=30, interval=5): ret, out, err = oc_rsh( self.node, pod_name, "df -Ph /mnt | awk '{print $2}' | tail -1") self.assertFalse(ret, 'Failed with err: %s and Output: %s' % ( err, out)) if out.strip() == '2.0G': break # Write data making sure we have more space than it was ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file2') self.assertFalse(ret, 'Not able to write file with err: %s' % err) # Verify pod is running wait_for_pod_be_ready(self.node, pod_name, 10, 5)
def create_sc_with_parameter(self, vol_type, success=False, parameter={}): """creates storage class, pvc and validates event Args: vol_type (str): storage type either gluster file or block success (bool): if True check for successfull else failure for pvc creation event parameter (dict): dictionary with storage class parameters """ if vol_type == "glusterfile": sc = self.storage_classes.get( 'storage_class1', self.storage_classes.get('file_storage_class')) # Create secret file for usage in storage class self.secret_name = oc_create_secret( self.ocp_master_node[0], namespace=sc.get('secretnamespace', 'default'), data_key=self.heketi_cli_key, secret_type=sc.get('provisioner', 'kubernetes.io/glusterfs')) self.addCleanup( oc_delete, self.ocp_master_node[0], 'secret', self.secret_name) sc_parameter = { "secretnamespace": sc['secretnamespace'], "secretname": self.secret_name, "volumetype": "replicate:3" } elif vol_type == "glusterblock": sc = self.storage_classes.get( 'storage_class2', self.storage_classes.get('block_storage_class')) # Create secret file for usage in storage class self.secret_name = oc_create_secret( self.ocp_master_node[0], namespace=sc.get('restsecretnamespace', 'default'), data_key=self.heketi_cli_key, secret_type=sc.get('provisioner', 'gluster.org/glusterblock')) self.addCleanup( oc_delete, self.ocp_master_node[0], 'secret', self.secret_name) sc_parameter = { "provisioner": "gluster.org/glusterblock", "restsecretnamespace": sc['restsecretnamespace'], "restsecretname": self.secret_name, "hacount": sc['hacount'] } else: err_msg = "invalid vol_type %s" % vol_type g.log.error(err_msg) raise AssertionError(err_msg) sc_parameter['resturl'] = sc['resturl'] sc_parameter['restuser'] = sc['restuser'] sc_parameter.update(parameter) # Create storage class self.sc_name = oc_create_sc( self.ocp_master_node[0], **sc_parameter) self.addCleanup(oc_delete, self.ocp_master_node[0], 'sc', self.sc_name) # Create PVC self.pvc_name = oc_create_pvc(self.ocp_client[0], self.sc_name) self.addCleanup( wait_for_resource_absence, self.ocp_master_node[0], 'pvc', self.pvc_name) self.addCleanup(oc_delete, self.ocp_master_node[0], 'pvc', self.pvc_name) # Wait for event with error event_reason = 'ProvisioningFailed' if success: event_reason = 'ProvisioningSucceeded' wait_for_events(self.ocp_master_node[0], obj_name=self.pvc_name, obj_type='PersistentVolumeClaim', event_reason=event_reason)
def test_prometheus_pv_resize(self): """ Validate prometheus metrics with pv resize""" # Fetch the metrics and storing initial_metrics as dictionary pvc_name, pod_name, initial_metrics = self._fetch_initial_metrics( vol_name_prefix="for-pv-resize", volume_expansion=True) # Write data on the pvc and confirm it is reflected in the prometheus self._perform_io_and_fetch_metrics( pod_name=pod_name, pvc_name=pvc_name, filename="filename1", dirname="dirname1", metric_data=initial_metrics, operation="create") # Resize the pvc to 2GiB openshift_ops.switch_oc_project( self._master, self.storage_project_name) pvc_size = 2 openshift_ops.resize_pvc(self._master, pvc_name, pvc_size) openshift_ops.wait_for_events(self._master, obj_name=pvc_name, event_reason='VolumeResizeSuccessful') openshift_ops.verify_pvc_size(self._master, pvc_name, pvc_size) pv_name = openshift_ops.get_pv_name_from_pvc( self._master, pvc_name) openshift_ops.verify_pv_size(self._master, pv_name, pvc_size) heketi_volume_name = heketi_ops.heketi_volume_list_by_name_prefix( self.heketi_client_node, self.heketi_server_url, "for-pv-resize", json=True)[0][2] self.assertIsNotNone( heketi_volume_name, "Failed to fetch volume with prefix {}". format("for-pv-resize")) openshift_ops.oc_delete(self._master, 'pod', pod_name) openshift_ops.wait_for_resource_absence(self._master, 'pod', pod_name) pod_name = openshift_ops.get_pod_name_from_dc( self._master, self.dc_name) openshift_ops.wait_for_pod_be_ready(self._master, pod_name) # Check whether the metrics are updated or not for w in waiter.Waiter(120, 10): resize_metrics = self._get_and_manipulate_metric_data( self.metrics, pvc_name) if bool(resize_metrics) and int(resize_metrics[ 'kubelet_volume_stats_capacity_bytes']) > int( initial_metrics['kubelet_volume_stats_capacity_bytes']): break if w.expired: raise AssertionError("Failed to reflect PVC Size after resizing") openshift_ops.switch_oc_project( self._master, self.storage_project_name) time.sleep(240) # Lookup and trigger rebalance and wait for the its completion for _ in range(100): self.cmd_run("oc rsh {} ls /mnt/".format(pod_name)) self._rebalance_completion(heketi_volume_name) # Write data on the resized pvc and compared with the resized_metrics self._perform_io_and_fetch_metrics( pod_name=pod_name, pvc_name=pvc_name, filename="secondfilename", dirname="seconddirname", metric_data=resize_metrics, operation="create")