def test_restart_gluster_block_provisioner_pod(self): """Restart gluster-block provisioner pod """ # create heketi block volume vol_info = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, size=5, json=True) self.assertTrue(vol_info, "Failed to create heketi block" "volume of size 5") self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # restart gluster-block-provisioner-pod dc_name = "glusterblock-%s-provisioner-dc" % self.storage_project_name pod_name = get_pod_name_from_dc(self.ocp_master_node, dc_name) oc_delete(self.ocp_master_node, 'pod', pod_name) wait_for_resource_absence(self.ocp_master_node, 'pod', pod_name) # new gluster-pod name pod_name = get_pod_name_from_dc(self.ocp_master_node, dc_name) wait_for_pod_be_ready(self.ocp_master_node, pod_name) # create new heketi block volume vol_info = heketi_blockvolume_create(self.heketi_client_node, self.heketi_server_url, size=2, json=True) self.assertTrue(vol_info, "Failed to create heketi block" "volume of size 2") heketi_blockvolume_delete(self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def test_pvc_deletion_while_pod_is_running(self): """Validate PVC deletion while pod is running""" # Create DC with POD and attached PVC to it sc_name = self.create_storage_class() pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name) dc_name, pod_name = self.create_dc_with_pvc(pvc_name) # Delete PVC oc_delete(self.node, 'pvc', self.pvc_name) with self.assertRaises(ExecutionError): wait_for_resource_absence(self.node, 'pvc', self.pvc_name, interval=3, timeout=30) # Make sure we are able to work with files on the mounted volume # after deleting pvc. filepath = "/mnt/file_for_testing_volume.log" cmd = "dd if=/dev/urandom of=%s bs=1K count=100" % filepath ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
def test_dynamic_provisioning_glusterfile_glusterpod_failure(self): """Validate dynamic provisioning for gluster file when gluster pod down """ mount_path = "/mnt" datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id()) # Create secret and storage class self.create_storage_class() # Create PVC pvc_name = self.create_and_wait_for_pvc() # Create app POD with attached volume pod_name = oc_create_tiny_pod_with_volume( self.node, pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup( wait_for_resource_absence, self.node, 'pod', pod_name) self.addCleanup(oc_delete, self.node, 'pod', pod_name) # Wait for app POD be up and running wait_for_pod_be_ready( self.node, pod_name, timeout=60, wait_step=2) # Run IO in background io_cmd = "oc rsh %s dd if=/dev/urandom of=%s bs=1000K count=900" % ( pod_name, datafile_path) async_io = g.run_async(self.node, io_cmd, "root") # Pick up one of the hosts which stores PV brick (4+ nodes case) gluster_pod_data = get_gluster_pod_names_by_pvc_name( self.node, pvc_name)[0] # Delete glusterfs POD from chosen host and wait for spawn of new one oc_delete(self.node, 'pod', gluster_pod_data["pod_name"]) cmd = ("oc get pods -o wide | grep glusterfs | grep %s | " "grep -v Terminating | awk '{print $1}'") % ( gluster_pod_data["host_name"]) for w in Waiter(600, 30): out = self.cmd_run(cmd) new_gluster_pod_name = out.strip().split("\n")[0].strip() if not new_gluster_pod_name: continue else: break if w.expired: error_msg = "exceeded timeout, new gluster pod not created" g.log.error(error_msg) raise ExecutionError(error_msg) new_gluster_pod_name = out.strip().split("\n")[0].strip() g.log.info("new gluster pod name is %s" % new_gluster_pod_name) wait_for_pod_be_ready(self.node, new_gluster_pod_name) # Check that async IO was not interrupted ret, out, err = async_io.async_communicate() self.assertEqual(ret, 0, "IO %s failed on %s" % (io_cmd, self.node))
def test_restart_heketi_pod(self): """Validate restarting heketi pod""" # create heketi volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size=1, json=True) self.assertTrue(vol_info, "Failed to create heketi volume of size 1") self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id'], raise_on_error=False) topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) # get heketi-pod name heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0], self.heketi_dc_name) # delete heketi-pod (it restarts the pod) oc_delete(self.ocp_master_node[0], 'pod', heketi_pod_name) wait_for_resource_absence(self.ocp_master_node[0], 'pod', heketi_pod_name) # get new heketi-pod name heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0], self.heketi_dc_name) wait_for_pod_be_ready(self.ocp_master_node[0], heketi_pod_name) # check heketi server is running self.assertTrue( hello_heketi(self.heketi_client_node, self.heketi_server_url), "Heketi server %s is not alive" % self.heketi_server_url ) # compare the topology new_topo_info = heketi_topology_info(self.heketi_client_node, self.heketi_server_url, json=True) self.assertEqual(new_topo_info, topo_info, "topology info is not same," " difference - %s" % diff(topo_info, new_topo_info)) # create new volume vol_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size=2, json=True) self.assertTrue(vol_info, "Failed to create heketi volume of size 20") heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def test_pv_resize_with_prefix_for_name(self, create_vol_name_prefix=False): """Validate PV resize with and without name prefix""" dir_path = "/mnt/" node = self.ocp_client[0] # Create PVC self.create_storage_class( allow_volume_expansion=True, create_vol_name_prefix=create_vol_name_prefix) pvc_name = self.create_and_wait_for_pvc() # Create DC with POD and attached PVC to it. dc_name = oc_create_app_dc_with_io(node, pvc_name) self.addCleanup(oc_delete, node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, node, dc_name, 0) pod_name = get_pod_name_from_dc(node, dc_name) wait_for_pod_be_ready(node, pod_name) if create_vol_name_prefix: ret = heketi_ops.verify_volume_name_prefix( node, self.sc['volumenameprefix'], self.sc['secretnamespace'], pvc_name, self.heketi_server_url) self.assertTrue(ret, "verify volnameprefix failed") cmd = ("dd if=/dev/urandom of=%sfile " "bs=100K count=1000") % dir_path ret, out, err = oc_rsh(node, pod_name, cmd) self.assertEqual(ret, 0, "failed to execute command %s on %s" % (cmd, node)) cmd = ("dd if=/dev/urandom of=%sfile2 " "bs=100K count=10000") % dir_path ret, out, err = oc_rsh(node, pod_name, cmd) self.assertNotEqual( ret, 0, " This IO did not fail as expected " "command %s on %s" % (cmd, node)) pvc_size = 2 resize_pvc(node, pvc_name, pvc_size) verify_pvc_size(node, pvc_name, pvc_size) pv_name = get_pv_name_from_pvc(node, pvc_name) verify_pv_size(node, pv_name, pvc_size) oc_delete(node, 'pod', pod_name) wait_for_resource_absence(node, 'pod', pod_name) pod_name = get_pod_name_from_dc(node, dc_name) wait_for_pod_be_ready(node, pod_name) cmd = ("dd if=/dev/urandom of=%sfile_new " "bs=50K count=10000") % dir_path ret, out, err = oc_rsh(node, pod_name, cmd) self.assertEqual(ret, 0, "failed to execute command %s on %s" % (cmd, node))
def test_dynamic_provisioning_glusterfile_heketidown_pvc_delete(self): """Validate deletion of PVC's when heketi is down""" # Create storage class, secret and PVCs self.create_storage_class() self.pvc_name_list = self.create_and_wait_for_pvcs( 1, 'pvc-heketi-down', 3) # remove heketi-pod scale_dc_pod_amount_and_wait(self.ocp_client[0], self.heketi_dc_name, 0, self.storage_project_name) try: # delete pvc for pvc in self.pvc_name_list: oc_delete(self.ocp_client[0], 'pvc', pvc) for pvc in self.pvc_name_list: with self.assertRaises(ExecutionError): wait_for_resource_absence(self.ocp_client[0], 'pvc', pvc, interval=3, timeout=30) finally: # bring back heketi-pod scale_dc_pod_amount_and_wait(self.ocp_client[0], self.heketi_dc_name, 1, self.storage_project_name) # verify PVC's are deleted for pvc in self.pvc_name_list: wait_for_resource_absence(self.ocp_client[0], 'pvc', pvc, interval=1, timeout=120) # create a new PVC self.create_and_wait_for_pvc()
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self): """Validate retain policy for glusterfs after deletion of pvc""" self.create_storage_class(reclaim_policy='Retain') self.create_and_wait_for_pvc() # get the name of the volume pv_name = get_pv_name_from_pvc(self.node, self.pvc_name) custom = [ r':.metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi\-volume\-id"', r':.spec.persistentVolumeReclaimPolicy' ] vol_id, reclaim_policy = oc_get_custom_resource( self.node, 'pv', custom, pv_name) self.assertEqual(reclaim_policy, 'Retain') # Create DC with POD and attached PVC to it. try: dc_name = oc_create_app_dc_with_io(self.node, self.pvc_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) finally: scale_dc_pod_amount_and_wait(self.node, dc_name, 0) oc_delete(self.node, 'dc', dc_name) wait_for_resource_absence(self.node, 'pod', pod_name) oc_delete(self.node, 'pvc', self.pvc_name) with self.assertRaises(ExecutionError): wait_for_resource_absence(self.node, 'pvc', self.pvc_name, interval=3, timeout=30) heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol_id) vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertNotIn(vol_id, vol_list) oc_delete(self.node, 'pv', pv_name) wait_for_resource_absence(self.node, 'pv', pv_name)
def _pv_resize(self, exceed_free_space): dir_path = "/mnt" pvc_size_gb, min_free_space_gb = 1, 3 # Get available free space disabling redundant devices and nodes heketi_url = self.heketi_server_url node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node, heketi_url) self.assertTrue(node_id_list) nodes = {} min_free_space = min_free_space_gb * 1024**2 for node_id in node_id_list: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, heketi_url, node_id, json=True) if (node_info['state'].lower() != 'online' or not node_info['devices']): continue if len(nodes) > 2: out = heketi_ops.heketi_node_disable(self.heketi_client_node, heketi_url, node_id) self.assertTrue(out) self.addCleanup(heketi_ops.heketi_node_enable, self.heketi_client_node, heketi_url, node_id) for device in node_info['devices']: if device['state'].lower() != 'online': continue free_space = device['storage']['free'] if (node_id in nodes.keys() or free_space < min_free_space): out = heketi_ops.heketi_device_disable( self.heketi_client_node, heketi_url, device['id']) self.assertTrue(out) self.addCleanup(heketi_ops.heketi_device_enable, self.heketi_client_node, heketi_url, device['id']) continue nodes[node_id] = free_space if len(nodes) < 3: raise self.skipTest("Could not find 3 online nodes with, " "at least, 1 online device having free space " "bigger than %dGb." % min_free_space_gb) # Calculate maximum available size for PVC available_size_gb = int(min(nodes.values()) / (1024**2)) # Create PVC self.create_storage_class(allow_volume_expansion=True) pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb) # Create DC with POD and attached PVC to it dc_name = oc_create_app_dc_with_io(self.node, pvc_name) self.addCleanup(oc_delete, self.node, 'dc', dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) if exceed_free_space: # Try to expand existing PVC exceeding free space resize_pvc(self.node, pvc_name, available_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeFailed') # Check that app POD is up and runnig then try to write data wait_for_pod_be_ready(self.node, pod_name) cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual( ret, 0, "Failed to write data after failed attempt to expand PVC.") else: # Expand existing PVC using all the available free space expand_size_gb = available_size_gb - pvc_size_gb resize_pvc(self.node, pvc_name, expand_size_gb) verify_pvc_size(self.node, pvc_name, expand_size_gb) pv_name = get_pv_name_from_pvc(self.node, pvc_name) verify_pv_size(self.node, pv_name, expand_size_gb) wait_for_events(self.node, obj_name=pvc_name, event_reason='VolumeResizeSuccessful') # Recreate app POD oc_delete(self.node, 'pod', pod_name) wait_for_resource_absence(self.node, 'pod', pod_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) # Write data on the expanded PVC cmd = ("dd if=/dev/urandom of=%s/autotest " "bs=1M count=1025" % dir_path) ret, out, err = oc_rsh(self.node, pod_name, cmd) self.assertEqual(ret, 0, "Failed to write data on the expanded PVC")
def test_create_delete_pvcs_to_make_gluster_reuse_released_space(self): """Validate reuse of volume space after deletion of PVCs""" min_storage_gb = 10 # Set arbiter:disabled tags to the first 2 nodes data_nodes = [] biggest_disks = [] self.assertGreater(len(self.node_id_list), 2) for node_id in self.node_id_list[0:2]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) biggest_disk_free_space = 0 for device in node_info['devices']: disk_free_space = int(device['storage']['free']) if disk_free_space < (min_storage_gb * 1024**2): self.skipTest("Devices are expected to have more than " "%sGb of free space" % min_storage_gb) if disk_free_space > biggest_disk_free_space: biggest_disk_free_space = disk_free_space self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'disabled', revert_to=device.get('tags', {}).get('arbiter')) biggest_disks.append(biggest_disk_free_space) self._set_arbiter_tag_with_further_revert(self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'disabled', revert_to=node_info.get( 'tags', {}).get('arbiter')) data_nodes.append(node_info) # Set arbiter:required tag to all other nodes and their devices arbiter_nodes = [] for node_id in self.node_id_list[2:]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'required', revert_to=device.get('tags', {}).get('arbiter')) self._set_arbiter_tag_with_further_revert(self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'required', revert_to=node_info.get( 'tags', {}).get('arbiter')) arbiter_nodes.append(node_info) # Calculate size and amount of volumes to be created pvc_size = int(min(biggest_disks) / 1024**2) pvc_amount = max([len(n['devices']) for n in data_nodes]) + 1 # Create sc with gluster arbiter info self.create_storage_class(is_arbiter_vol=True) # Create and delete 3 small volumes concurrently pvc_names = [] for i in range(3): pvc_name = oc_create_pvc(self.node, self.sc_name, pvc_name_prefix='arbiter-pvc', pvc_size=int(pvc_size / 3)) pvc_names.append(pvc_name) exception_exists = False for pvc_name in pvc_names: try: verify_pvc_status_is_bound(self.node, pvc_name) except Exception: for pvc_name in pvc_names: self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name) for pvc_name in pvc_names: self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) exception_exists = True if exception_exists: raise for pvc_name in pvc_names: oc_delete(self.node, 'pvc', pvc_name) for pvc_name in pvc_names: wait_for_resource_absence(self.node, 'pvc', pvc_name) # Create and delete big volumes in a loop for i in range(pvc_amount): pvc_name = oc_create_pvc(self.node, self.sc_name, pvc_name_prefix='arbiter-pvc', pvc_size=pvc_size) try: verify_pvc_status_is_bound(self.node, pvc_name) except Exception: self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) raise oc_delete(self.node, 'pvc', pvc_name) wait_for_resource_absence(self.node, 'pvc', pvc_name)
def delete_pvc(self, ocp_node): oc_delete(ocp_node, 'pvc', self.pvc_name)
def delete_storageclass(ocp_node, sc_name, timeout=120): wait_for_sc_unused(ocp_node, sc_name, timeout) oc_delete(ocp_node, 'storageclass', sc_name)