def test_rbd_space_reclaim_no_space(self): """ Test to verify RBD space reclamation Steps: 1. Create and attach RBD PVC of size 25 GiB to an app pod. 2. Get the used size of the RBD pool 3. Create a file of size 10GiB 4. Verify the used size of the RBD pool 5. Create ReclaimSpaceJob 6. Verify the size of the RBD pool, no changes should be seen. """ pvc_obj = self.pvc[0] pod_obj = self.pod[0] fio_filename1 = "fio_file1" fio_filename2 = "fio_file2" # Fetch the used size of pool cbp_name = self.sc_obj.get().get("parameters").get("pool") used_size_before_io = fetch_used_size(cbp_name) log.info(f"Used size before IO is {used_size_before_io}") # Create a 10 GB file for filename in [fio_filename1, fio_filename2]: pod_obj.run_io( storage_type="fs", size="10G", runtime=100, fio_filename=filename, end_fsync=1, ) pod_obj.get_fio_results() # Verify used size after IO exp_used_size_after_io = used_size_before_io + (20 * self.pool_replica) used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io) log.info(f"Used size after IO is {used_size_after_io}") # Create ReclaimSpaceJob reclaim_space_job = pvc_obj.create_reclaim_space_job() # Verify Succeeded result of ReclaimSpaceJob self.reclaim_space_job(reclaim_space_job) # Verify space is reclaimed by checking the used size of the RBD pool used_after_reclaiming_space = fetch_used_size(cbp_name, used_size_after_io) log.info( f"Memory remains intact. Used size after io is {used_after_reclaiming_space}." )
def test_pvc_delete_and_verify_size_is_returned_to_backend_pool( self, pause_and_resume_cluster_load, pvc_factory, pod_factory ): """ Test case to verify after delete pvc size returned to backend pools """ cbp_name = helpers.default_ceph_block_pool() # TODO: Get exact value of replica size replica_size = 3 pvc_obj = pvc_factory( interface=constants.CEPHBLOCKPOOL, size=10, status=constants.STATUS_BOUND ) pod_obj = pod_factory( interface=constants.CEPHBLOCKPOOL, pvc=pvc_obj, status=constants.STATUS_RUNNING, ) pvc_obj.reload() used_before_io = helpers.fetch_used_size(cbp_name) logger.info(f"Used before IO {used_before_io}") # Write 6Gb pod.run_io_and_verify_mount_point(pod_obj, bs="10M", count="600") exp_size = used_before_io + (6 * replica_size) used_after_io = helpers.fetch_used_size(cbp_name, exp_size) logger.info(f"Used space after IO {used_after_io}") rbd_image_id = pvc_obj.image_uuid pod_obj.delete() pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name) pvc_obj.delete() pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name) verify_pv_not_exists(pvc_obj, cbp_name, rbd_image_id) used_after_deleting_pvc = helpers.fetch_used_size(cbp_name, used_before_io) logger.info(f"Used after deleting PVC {used_after_deleting_pvc}")
def test_verify_rbd_thick_pvc_utilization( self, pvc_factory, pod_factory, ): """ Test to verify the storage utilization of RBD thick provisioned PVC """ pvc_size = 15 replica_size = 3 file1 = "fio_file1" file2 = "fio_file2" rbd_pool = default_ceph_block_pool() size_before_pvc = fetch_used_size(rbd_pool) log.info( f"Storage pool used size before creating the PVC is {size_before_pvc}" ) # Create RBD thick PVC pvc_obj = pvc_factory( interface=constants.CEPHBLOCKPOOL, project=self.proj_obj, storageclass=default_thick_storage_class(), size=pvc_size, access_mode=constants.ACCESS_MODE_RWO, status=constants.STATUS_BOUND, ) size_after_pvc = fetch_used_size( rbd_pool, size_before_pvc + (pvc_size * replica_size)) log.info( f"Verified: Storage pool used size after creating the PVC is {size_after_pvc}" ) pod_obj = pod_factory( interface=constants.CEPHBLOCKPOOL, pvc=pvc_obj, status=constants.STATUS_RUNNING, ) # Create 5GB file pod_obj.run_io( storage_type="fs", size="5G", runtime=60, fio_filename=file1, end_fsync=1, ) pod_obj.get_fio_results() # Verify the used size after IO fetch_used_size(rbd_pool, size_before_pvc + (pvc_size * replica_size)) # Create another 5GB file pod_obj.run_io( storage_type="fs", size="5G", runtime=60, fio_filename=file2, end_fsync=1, ) pod_obj.get_fio_results() # Verify the used size after IO fetch_used_size(rbd_pool, size_before_pvc + (pvc_size * replica_size)) # Delete the files created by fio mount_point = pod_obj.get_storage_path() rm_cmd = f"rm {path.join(mount_point, file1)} {path.join(mount_point, file2)}" pod_obj.exec_cmd_on_pod(command=rm_cmd, out_yaml_format=False) # Verify the used size after deleting the files fetch_used_size(rbd_pool, size_before_pvc + (pvc_size * replica_size)) # Delete the pod pod_obj.delete() pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name) # Delete the PVC pvc_obj.delete() pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name) # Verify used size after deleting the PVC size_after_pvc_delete = fetch_used_size(rbd_pool, size_before_pvc) log.info( f"Verified: Storage pool used size after deleting the PVC is {size_after_pvc_delete}" )
def test_rbd_space_reclaim_cronjob(self, pause_and_resume_cluster_load): """ Test to verify RBD space reclamation Steps: 1. Create and attach RBD PVC of size 25 GiB to an app pod. 2. Get the used size of the RBD pool 3. Create four files of size 4GiB 4. Verify the increased used size of the RBD pool 5. Delete three file 6. Create ReclaimSpaceJob 7. Verify the decreased used size of the RBD pool 8. Verify the presence of other files in the folder """ pvc_obj = self.pvc[0] pod_obj = self.pod[0] fio_filename1 = "fio_file1" fio_filename2 = "fio_file2" fio_filename3 = "fio_file3" fio_filename4 = "fio_file4" schedule = ["hourly", "midnight", "weekly"] # Fetch the used size of pool cbp_name = self.sc_obj.get().get("parameters").get("pool") log.info(f"Cephblock pool name {cbp_name}") used_size_before_io = fetch_used_size(cbp_name) log.info(f"Used size before IO is {used_size_before_io}") # Create four 4 GB file for filename in [ fio_filename1, fio_filename2, fio_filename3, fio_filename4 ]: pod_obj.run_io( storage_type="fs", size="4G", runtime=100, fio_filename=filename, end_fsync=1, ) pod_obj.get_fio_results() # Verify used size after IO exp_used_size_after_io = used_size_before_io + (16 * self.pool_replica) used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io) log.info(f"Used size after IO is {used_size_after_io}") # Delete the file and validate the reclaimspace cronjob for filename in [fio_filename1, fio_filename2, fio_filename3]: file_path = get_file_path(pod_obj, filename) pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}", out_yaml_format=False, timeout=100) # Verify file is deleted try: check_file_existence(pod_obj=pod_obj, file_path=file_path) except CommandFailed as cmdfail: if "No such file or directory" not in str(cmdfail): raise log.info(f"Verified: File {file_path} deleted.") # Create ReclaimSpaceCronJob for type in schedule: reclaim_space_job = pvc_obj.create_reclaim_space_cronjob( schedule=type) # Wait for the Succeeded result of ReclaimSpaceJob try: for reclaim_space_job_yaml in TimeoutSampler( timeout=120, sleep=5, func=reclaim_space_job.get): result = reclaim_space_job_yaml["spec"]["schedule"] if result == "@" + type: log.info( f"ReclaimSpaceJob {reclaim_space_job.name} succeeded" ) break else: log.info( f"Waiting for the Succeeded result of the ReclaimSpaceCronJob {reclaim_space_job.name}. " f"Present value of result is {result}") except TimeoutExpiredError: raise UnexpectedBehaviour( f"ReclaimSpaceJob {reclaim_space_job.name} is not successful. Yaml output:{reclaim_space_job.get()}" ) # Verify the presence of another file in the directory log.info("Verifying the existence of remaining file in the directory") file_path = get_file_path(pod_obj, fio_filename4) log.info(check_file_existence(pod_obj=pod_obj, file_path=file_path)) if check_file_existence(pod_obj=pod_obj, file_path=file_path): log.info(f"{fio_filename4} is intact")
def test_rbd_space_reclaim(self): """ Test to verify RBD space reclamation Steps: 1. Create and attach RBD PVC of size 25 GiB to an app pod. 2. Get the used size of the RBD pool 3. Create two files of size 10GiB 4. Verify the increased used size of the RBD pool 5. Delete one file 6. Create ReclaimSpaceJob 7. Verify the decreased used size of the RBD pool """ pvc_obj = self.pvc[0] pod_obj = self.pod[0] fio_filename1 = "fio_file1" fio_filename2 = "fio_file2" # Fetch the used size of pool cbp_name = self.sc_obj.get().get("parameters").get("pool") used_size_before_io = fetch_used_size(cbp_name) log.info(f"Used size before IO is {used_size_before_io}") # Create two 10 GB file for filename in fio_filename1, fio_filename2: pod_obj.run_io( storage_type="fs", size="10G", runtime=120, fio_filename=filename, end_fsync=1, ) pod_obj.get_fio_results() # Verify used size after IO exp_used_size_after_io = used_size_before_io + (20 * self.pool_replica) used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io) log.info(f"Used size after IO is {used_size_after_io}") # Delete one file file_path = get_file_path(pod_obj, fio_filename2) pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}", out_yaml_format=False) # Verify file is deleted try: check_file_existence(pod_obj=pod_obj, file_path=file_path) except CommandFailed as cmdfail: if "No such file or directory" not in str(cmdfail): raise log.info(f"Verified: File {file_path} deleted.") # Create ReclaimSpaceJob reclaim_space_job = pvc_obj.create_reclaim_space_job() # Wait for the Succeeded result of ReclaimSpaceJob try: for reclaim_space_job_yaml in TimeoutSampler( timeout=120, sleep=5, func=reclaim_space_job.get): result = reclaim_space_job_yaml.get("status", {}).get("result") if result == "Succeeded": log.info( f"ReclaimSpaceJob {reclaim_space_job.name} succeeded") break else: log.info( f"Waiting for the Succeeded result of the ReclaimSpaceJob {reclaim_space_job.name}. " f"Present value of result is {result}") except TimeoutExpiredError: raise UnexpectedBehaviour( f"ReclaimSpaceJob {reclaim_space_job.name} is not successful. Yaml output:{reclaim_space_job.get()}" ) # Verify space is reclaimed by checking the used size of the RBD pool used_after_reclaiming_space = fetch_used_size( cbp_name, used_size_after_io - (10 * self.pool_replica)) log.info( f"Space reclamation verified. Used size after reclaiming space is {used_after_reclaiming_space}." )
def uninstall_cluster_logging(): """ Function to uninstall cluster-logging from the cluster Deletes the project "openshift-logging" and "openshift-operators-redhat" """ # Validating the pods before deleting the instance pod_list = get_all_pods(namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) for pod in pod_list: logger.info( f"Pods running in the openshift-logging namespace {pod.name}") # Excluding cluster-logging-operator from pod_list and getting pod names pod_names_list = [ pod.name for pod in pod_list if not pod.name.startswith("cluster-logging-operator") ] pvc_objs = get_all_pvc_objs( namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) # Fetch image uuid associated with PVCs to be deleted pvc_uuid_map = {} for pvc_obj in pvc_objs: pvc_uuid_map[pvc_obj.name] = pvc_obj.image_uuid # Checking for used space cbp_name = default_ceph_block_pool() used_space_before_deletion = fetch_used_size(cbp_name) logger.info( f"Used space before deletion of cluster logging {used_space_before_deletion}" ) # Deleting the clusterlogging instance clusterlogging_obj = ocp.OCP( kind=constants.CLUSTER_LOGGING, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) assert clusterlogging_obj.delete(resource_name="instance") check_pod_vanished(pod_names_list) for pvc_obj in pvc_objs: pv_obj = pvc_obj.backed_pv_obj assert delete_pvcs(pvc_objs=pvc_objs), "PVCs deletion failed" for pvc_obj in pvc_objs: pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name, timeout=300) pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name, timeout=300) logger.info("Verified: PVCs are deleted.") logger.info("Verified: PV are deleted") for pvc_name, uuid in pvc_uuid_map.items(): rbd = verify_volume_deleted_in_backend( interface=constants.CEPHBLOCKPOOL, image_uuid=uuid, pool_name=cbp_name) assert rbd, f"Volume associated with PVC {pvc_name} still exists " f"in backend" # Checking for used space after PVC deletion used_space_after_deletion = fetch_used_size(cbp_name, exp_val=30) logger.info( f"Used space after deletion of cluster logging {used_space_after_deletion}" ) if used_space_after_deletion < used_space_before_deletion: logger.info("Expected !!! Space has reclaimed") else: logger.warning( "Unexpected !! No space reclaimed after deletion of PVC") # Deleting the RBAC permission set rbac_role = ocp.OCP( kind=constants.ROLE, namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE) rbac_role.delete(yaml_file=constants.EO_RBAC_YAML) openshift_logging_namespace = ocp.OCP( kind=constants.NAMESPACES, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) openshift_operators_redhat_namespace = ocp.OCP( kind=constants.NAMESPACES, namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE, ) if openshift_operators_redhat_namespace.get(): assert openshift_operators_redhat_namespace.delete( resource_name=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE) logger.info( "The project openshift-opertors-redhat got deleted successfully") if openshift_logging_namespace.get(): assert openshift_logging_namespace.delete( resource_name=constants.OPENSHIFT_LOGGING_NAMESPACE) logger.info("The namespace openshift-logging got deleted successfully")
def test_rbd_space_reclaim(self): """ Test to verify RBD space reclamation Steps: 1. Create and attach RBD PVC of size 25 GiB to an app pod. 2. Get the used size of the RBD pool 3. Create two files of size 10GiB 4. Verify the increased used size of the RBD pool 5. Delete one file 6. Create ReclaimSpaceJob 7. Verify the decreased used size of the RBD pool """ pvc_obj = self.pvc[0] pod_obj = self.pod[0] fio_filename1 = "fio_file1" fio_filename2 = "fio_file2" # Fetch the used size of pool cbp_name = self.sc_obj.get().get("parameters").get("pool") used_size_before_io = fetch_used_size(cbp_name) log.info(f"Used size before IO is {used_size_before_io}") # Create two 10 GB file for filename in fio_filename1, fio_filename2: pod_obj.run_io( storage_type="fs", size="10G", runtime=120, fio_filename=filename, end_fsync=1, ) pod_obj.get_fio_results() # Verify used size after IO exp_used_size_after_io = used_size_before_io + (20 * self.pool_replica) used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io) log.info(f"Used size after IO is {used_size_after_io}") # Delete one file file_path = get_file_path(pod_obj, fio_filename1) pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}", out_yaml_format=False) # Verify whether file is deleted try: check_file_existence(pod_obj=pod_obj, file_path=file_path) except CommandFailed as cmdfail: if "No such file or directory" not in str(cmdfail): raise log.info(f"Verified: File {file_path} deleted.") # Wait for 15 seconds after deleting the file time.sleep(15) # Create ReclaimSpaceJob reclaim_space_job = pvc_obj.create_reclaim_space_job() # Verify Succeeded result of ReclaimSpaceJob self.reclaim_space_job(reclaim_space_job) time.sleep(120) # Verify space is reclaimed by checking the used size of the RBD pool used_after_reclaiming_space = fetch_used_size( cbp_name, used_size_after_io - (10 * self.pool_replica)) log.info( f"Space has been reclaimed. Used size after io is {used_after_reclaiming_space}." ) # Verify the presence of another file in the directory log.info("Verifying the existence of remaining file in the pod") file_path = get_file_path(pod_obj, fio_filename2) log.info(check_file_existence(pod_obj=pod_obj, file_path=file_path)) if check_file_existence(pod_obj=pod_obj, file_path=file_path): log.info(f"{fio_filename2} is intact")
def test_no_volume_mounted(self): """ Test reclaimspace job with no volume mounted Steps: 1. Create and attach RBD PVC of size 25 GiB to an app pod. 2. Get the used size of the RBD pool 3. Create a file of size 10GiB 4. Delete the file 5. Delete the pod 6. Create ReclaimSpaceJob 7. No errors should be seen in reclaim space job """ pvc_obj = self.pvc[0] pod_obj = self.pod[0] fio_filename1 = "fio_file1" # Fetch the used size of pool cbp_name = self.sc_obj.get().get("parameters").get("pool") used_size_before_io = fetch_used_size(cbp_name) log.info(f"Used size before IO is {used_size_before_io}") # Create a 10 GB file pod_obj.run_io( storage_type="fs", size="10G", runtime=120, fio_filename=fio_filename1, end_fsync=1, ) pod_obj.get_fio_results() # Verify used size after IO exp_used_size_after_io = used_size_before_io + (10 * self.pool_replica) used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io) log.info(f"Used size after IO is {used_size_after_io}") # Delete the file file_path = get_file_path(pod_obj, fio_filename1) pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}", out_yaml_format=False) # Verify whether file is deleted try: check_file_existence(pod_obj=pod_obj, file_path=file_path) except CommandFailed as cmdfail: if "No such file or directory" not in str(cmdfail): raise log.info(f"Verified: File {file_path} deleted.") # Delete the pod log.info(f"Deleting the pod {pod_obj}") delete_pods([pod_obj]) # Create ReclaimSpaceJob reclaim_space_job = pvc_obj.create_reclaim_space_job() # Verify Succeeded result of ReclaimSpaceJob self.reclaim_space_job(reclaim_space_job)