def test_noobaa_db_backup_and_recovery( self, pvc_factory, pod_factory, snapshot_factory, bucket_factory, rgw_bucket_factory, ): """ Test case to verify noobaa backup and recovery 1. Take snapshot db-noobaa-db-0 PVC and retore it to PVC 2. Scale down the statefulset noobaa-db 3. Get the yaml of the current PVC, db-noobaa-db-0 and change the parameter persistentVolumeReclaimPolicy to Retain for restored PVC 4. Delete both PVCs, the PV for the original claim db-noobaa-db-0 will be removed. The PV for claim db-noobaa-db-0-snapshot-restore will move to ‘Released’ 5. Edit again restore PV and remove the claimRef section. The volume will transition to Available. 6. Edit the yaml db-noobaa-db-0.yaml and change the setting volumeName to restored PVC. 7. Scale up the stateful set again and the pod should be running """ # Initialise variable self.noobaa_db_sst_name = "noobaa-db-pg" # Get noobaa pods before execution noobaa_pods = get_noobaa_pods() # Get noobaa PVC before execution noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get( "volumeName") # Take snapshot db-noobaa-db-0 PVC log.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC") snap_obj = snapshot_factory( pvc_obj=noobaa_pvc_obj[0], wait=True, snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot", ) log.info( f"Successfully created snapshot {snap_obj.name} and in Ready state" ) # Restore it to PVC log.info(f"Restoring snapshot {snap_obj.name} to create new PVC") sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName") pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get( "requests").get("storage")) self.restore_pvc_obj = create_restore_pvc( sc_name=sc_name, snap_name=snap_obj.name, namespace=snap_obj.namespace, size=pvc_size, pvc_name=f"{snap_obj.name}-restore", volume_mode=snap_obj.parent_volume_mode, access_mode=snap_obj.parent_access_mode, ) wait_for_resource_state(self.restore_pvc_obj, constants.STATUS_BOUND) self.restore_pvc_obj.reload() log.info(f"Succeesfuly created PVC {self.restore_pvc_obj.name} " f"from snapshot {snap_obj.name}") # Scale down the statefulset noobaa-db modify_statefulset_replica_count( statefulset_name=self.noobaa_db_sst_name, replica_count=0 ), f"Failed to scale down the statefulset {self.noobaa_db_sst_name}" # Get the noobaa-db PVC pvc_obj = OCP(kind=constants.PVC, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name) # Get the restored noobaa PVC and # change the parameter persistentVolumeReclaimPolicy to Retain restored_noobaa_pvc_obj = get_pvc_objs( pvc_names=[f"{snap_obj.name}-restore"]) restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get( "spec").get("volumeName")) pv_obj = OCP(kind=constants.PV, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Retain {restored_noobaa_pv_name}") # Delete both PVCs delete_pvcs(pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]]) # Validate original claim db-noobaa-db-0 removed assert validate_pv_delete( pv_name=noobaa_pv_name ), f"PV not deleted, still exist {noobaa_pv_name}" # Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED, resource_name=restored_noobaa_pv_name) # Edit again restore PV and remove the claimRef section log.info( f"Remove the claimRef section from PVC {restored_noobaa_pv_name}") params = '[{"op": "remove", "path": "/spec/claimRef"}]' pv_obj.patch(resource_name=restored_noobaa_pv_name, params=params, format_type="json") log.info( f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}" ) # Validate PV is in Available state pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE, resource_name=restored_noobaa_pv_name) # Edit the yaml db-noobaa-db-0.yaml and change the # setting volumeName to restored PVC noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml) noobaa_pvc_yaml.create() # Validate noobaa PVC is in bound state pvc_obj.wait_for_resource( condition=constants.STATUS_BOUND, resource_name=noobaa_pvc_obj[0].name, timeout=120, ) # Scale up the statefulset again assert modify_statefulset_replica_count( statefulset_name=self.noobaa_db_sst_name, replica_count=1 ), f"Failed to scale up the statefulset {self.noobaa_db_sst_name}" # Validate noobaa pod is up and running pod_obj = OCP(kind=constants.POD, namespace=defaults.ROOK_CLUSTER_NAMESPACE) pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, resource_count=len(noobaa_pods), selector=constants.NOOBAA_APP_LABEL, ) # Change the parameter persistentVolumeReclaimPolicy to Delete again params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Delete {restored_noobaa_pv_name}") log.info( "Changed the parameter persistentVolumeReclaimPolicy to Delete again" ) # Verify all storage pods are running wait_for_storage_pods() # Creating Resources log.info("Creating Resources using sanity helpers") self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) # Deleting Resources self.sanity_helpers.delete_resources() # Verify everything running fine log.info( "Verifying All resources are Running and matches expected result") self.sanity_helpers.health_check(tries=120)
def test_pvc_snapshot_performance(self, pvc_size): """ 1. Run I/O on a pod file 2. Calculate md5sum of the file 3. Take a snapshot of the PVC 4. Measure the total snapshot creation time and the CSI snapshot creation time 4. Restore From the snapshot and measure the time 5. Attach a new pod to it 6. Verify that the file is present on the new pod also 7. Verify that the md5sum of the file on the new pod matches with the md5sum of the file on the original pod This scenario run 3 times and report all the average results of the 3 runs and will send them to the ES Args: pvc_size: the size of the PVC to be tested - parametrize """ # Getting the total Storage capacity ceph_cluster = CephCluster() ceph_capacity = ceph_cluster.get_ceph_capacity() log.info(f"Total capacity size is : {ceph_capacity}") log.info(f"PVC Size is : {pvc_size}") log.info(f"Needed capacity is {int(int(pvc_size) * 5)}") if int(ceph_capacity) < int(pvc_size) * 5: log.error( f"PVC size is {pvc_size}GiB and it is too large for this system" f" which have only {ceph_capacity}GiB") return # Calculating the file size as 25% of the PVC size # in the end the PVC will be 75% full filesize = self.pvc_obj.size * 0.25 # Change the file size to MB and from int to str file_size = f"{int(filesize * 1024)}M" all_results = [] self.results_path = get_full_test_logs_path(cname=self) log.info(f"Logs file path name is : {self.full_log_path}") # Produce ES report # Collecting environment information self.get_env_info() # Initialize the results doc file. self.full_results = self.init_full_results( ResultsAnalyse( self.uuid, self.crd_data, self.full_log_path, "pvc_snapshot_perf", )) self.full_results.add_key("pvc_size", pvc_size + " GiB") self.full_results.add_key("interface", self.sc) self.full_results.all_results["creation_time"] = [] self.full_results.all_results["csi_creation_time"] = [] self.full_results.all_results["creation_speed"] = [] self.full_results.all_results["restore_time"] = [] self.full_results.all_results["restore_speed"] = [] self.full_results.all_results["restore_csi_time"] = [] for test_num in range(self.tests_numbers): test_results = { "test_num": test_num + 1, "dataset": (test_num + 1) * filesize * 1024, # size in MiB "create": { "time": None, "csi_time": None, "speed": None }, "restore": { "time": None, "speed": None }, } log.info(f"Starting test phase number {test_num}") # Step 1. Run I/O on a pod file. file_name = f"{self.pod_object.name}-{test_num}" log.info(f"Starting IO on the POD {self.pod_object.name}") # Going to run only write IO to fill the PVC for the snapshot self.pod_object.fillup_fs(size=file_size, fio_filename=file_name) # Wait for fio to finish fio_result = self.pod_object.get_fio_results() err_count = fio_result.get("jobs")[0].get("error") assert ( err_count == 0 ), f"IO error on pod {self.pod_object.name}. FIO result: {fio_result}" log.info("IO on the PVC Finished") # Verify presence of the file file_path = pod.get_file_path(self.pod_object, file_name) log.info(f"Actual file path on the pod {file_path}") assert pod.check_file_existence( self.pod_object, file_path), f"File {file_name} doesn't exist" log.info(f"File {file_name} exists in {self.pod_object.name}") # Step 2. Calculate md5sum of the file. orig_md5_sum = pod.cal_md5sum(self.pod_object, file_name) # Step 3. Take a snapshot of the PVC and measure the time of creation. snap_name = self.pvc_obj.name.replace("pvc-test", f"snapshot-test{test_num}") log.info(f"Taking snapshot of the PVC {snap_name}") start_time = datetime.datetime.utcnow().strftime( "%Y-%m-%dT%H:%M:%SZ") test_results["create"]["time"] = self.measure_create_snapshot_time( pvc_name=self.pvc_obj.name, snap_name=snap_name, namespace=self.pod_object.namespace, interface=self.interface, start_time=start_time, ) test_results["create"][ "csi_time"] = performance_lib.measure_csi_snapshot_creation_time( interface=self.interface, snapshot_id=self.snap_uid, start_time=start_time, ) test_results["create"]["speed"] = int( test_results["dataset"] / test_results["create"]["time"]) log.info( f' Test {test_num} dataset is {test_results["dataset"]} MiB') log.info( f"Snapshot name {snap_name} and id {self.snap_uid} creation time is" f' : {test_results["create"]["time"]} sec.') log.info( f"Snapshot name {snap_name} and id {self.snap_uid} csi creation time is" f' : {test_results["create"]["csi_time"]} sec.') log.info( f'Snapshot speed is : {test_results["create"]["speed"]} MB/sec' ) # Step 4. Restore the PVC from the snapshot and measure the time # Same Storage class of the original PVC sc_name = self.pvc_obj.backed_sc # Size should be same as of the original PVC pvc_size = str(self.pvc_obj.size) + "Gi" # Create pvc out of the snapshot # Both, the snapshot and the restore PVC should be in same namespace log.info("Restoring from the Snapshot") restore_pvc_name = self.pvc_obj.name.replace( "pvc-test", f"restore-pvc{test_num}") restore_pvc_yaml = constants.CSI_RBD_PVC_RESTORE_YAML if self.interface == constants.CEPHFILESYSTEM: restore_pvc_yaml = constants.CSI_CEPHFS_PVC_RESTORE_YAML csi_start_time = self.get_time("csi") log.info("Restoring the PVC from Snapshot") restore_pvc_obj = pvc.create_restore_pvc( sc_name=sc_name, snap_name=self.snap_obj.name, namespace=self.snap_obj.namespace, size=pvc_size, pvc_name=restore_pvc_name, restore_pvc_yaml=restore_pvc_yaml, ) helpers.wait_for_resource_state( restore_pvc_obj, constants.STATUS_BOUND, timeout=3600 # setting this to 60 Min. # since it can be take long time to restore, and we want it to finished. ) restore_pvc_obj.reload() log.info("PVC was restored from the snapshot") test_results["restore"][ "time"] = helpers.measure_pvc_creation_time( self.interface, restore_pvc_obj.name) test_results["restore"]["speed"] = int( test_results["dataset"] / test_results["restore"]["time"]) log.info( f'Snapshot restore time is : {test_results["restore"]["time"]}' ) log.info( f'restore speed is : {test_results["restore"]["speed"]} MB/sec' ) test_results["restore"][ "csi_time"] = performance_lib.csi_pvc_time_measure( self.interface, restore_pvc_obj, "create", csi_start_time) log.info( f'Snapshot csi restore time is : {test_results["restore"]["csi_time"]}' ) # Step 5. Attach a new pod to the restored PVC restore_pod_object = helpers.create_pod( interface_type=self.interface, pvc_name=restore_pvc_obj.name, namespace=self.snap_obj.namespace, ) # Confirm that the pod is running helpers.wait_for_resource_state(resource=restore_pod_object, state=constants.STATUS_RUNNING) restore_pod_object.reload() # Step 6. Verify that the file is present on the new pod also. log.info(f"Checking the existence of {file_name} " f"on restore pod {restore_pod_object.name}") assert pod.check_file_existence( restore_pod_object, file_path), f"File {file_name} doesn't exist" log.info(f"File {file_name} exists in {restore_pod_object.name}") # Step 7. Verify that the md5sum matches log.info( f"Verifying that md5sum of {file_name} " f"on pod {self.pod_object.name} matches with md5sum " f"of the same file on restore pod {restore_pod_object.name}") assert pod.verify_data_integrity( restore_pod_object, file_name, orig_md5_sum), "Data integrity check failed" log.info("Data integrity check passed, md5sum are same") restore_pod_object.delete() restore_pvc_obj.delete() all_results.append(test_results) # clean the enviroment self.pod_object.delete() self.pvc_obj.delete() self.delete_test_project() # logging the test summary, all info in one place for easy log reading c_speed, c_runtime, c_csi_runtime, r_speed, r_runtime, r_csi_runtime = ( 0 for i in range(6)) log.info("Test summary :") for tst in all_results: c_speed += tst["create"]["speed"] c_runtime += tst["create"]["time"] c_csi_runtime += tst["create"]["csi_time"] r_speed += tst["restore"]["speed"] r_runtime += tst["restore"]["time"] r_csi_runtime += tst["restore"]["csi_time"] self.full_results.all_results["creation_time"].append( tst["create"]["time"]) self.full_results.all_results["csi_creation_time"].append( tst["create"]["csi_time"]) self.full_results.all_results["creation_speed"].append( tst["create"]["speed"]) self.full_results.all_results["restore_time"].append( tst["restore"]["time"]) self.full_results.all_results["restore_speed"].append( tst["restore"]["speed"]) self.full_results.all_results["restore_csi_time"].append( tst["restore"]["csi_time"]) self.full_results.all_results["dataset_inMiB"] = tst["dataset"] log.info( f"Test {tst['test_num']} results : dataset is {tst['dataset']} MiB. " f"Take snapshot time is {tst['create']['time']} " f"at {tst['create']['speed']} MiB/Sec " f"Restore from snapshot time is {tst['restore']['time']} " f"at {tst['restore']['speed']} MiB/Sec ") avg_snap_c_time = c_runtime / self.tests_numbers avg_snap_csi_c_time = c_csi_runtime / self.tests_numbers avg_snap_c_speed = c_speed / self.tests_numbers avg_snap_r_time = r_runtime / self.tests_numbers avg_snap_r_speed = r_speed / self.tests_numbers avg_snap_r_csi_time = r_csi_runtime / self.tests_numbers log.info(f" Average snapshot creation time is {avg_snap_c_time} sec.") log.info( f" Average csi snapshot creation time is {avg_snap_csi_c_time} sec." ) log.info( f" Average snapshot creation speed is {avg_snap_c_speed} MiB/sec") log.info(f" Average snapshot restore time is {avg_snap_r_time} sec.") log.info( f" Average snapshot restore speed is {avg_snap_r_speed} MiB/sec") log.info( f" Average snapshot restore csi time is {avg_snap_r_csi_time} sec." ) self.full_results.add_key("avg_snap_creation_time_insecs", avg_snap_c_time) self.full_results.add_key("avg_snap_csi_creation_time_insecs", avg_snap_csi_c_time) self.full_results.add_key("avg_snap_creation_speed", avg_snap_c_speed) self.full_results.add_key("avg_snap_restore_time_insecs", avg_snap_r_time) self.full_results.add_key("avg_snap_restore_speed", avg_snap_r_speed) self.full_results.add_key("avg_snap_restore_csi_time_insecs", avg_snap_r_csi_time) # Write the test results into the ES server log.info("writing results to elastic search server") if self.full_results.es_write(): res_link = self.full_results.results_link() # write the ES link to the test results in the test log. log.info(f"The result can be found at : {res_link}") self.write_result_to_file(res_link)
def test_pvc_snapshot_performance(self, teardown_factory, pvc_size): """ 1. Run I/O on a pod file. 2. Calculate md5sum of the file. 3. Take a snapshot of the PVC and measure the time of creation. 4. Restore From the snapshot and measure the time 5. Attach a new pod to it. 6. Verify that the file is present on the new pod also. 7. Verify that the md5sum of the file on the new pod matches with the md5sum of the file on the original pod. This scenario run 3 times and report all results Args: teardown_factory: A fixture to destroy objects pvc_size: the size of the PVC to be tested - parametrize """ # Getting the total Storage capacity ceph_cluster = CephCluster() ceph_capacity = ceph_cluster.get_ceph_capacity() log.info(f"Total capacity size is : {ceph_capacity}") log.info(f"PVC Size is : {pvc_size}") log.info(f"Needed capacity is {int(int(pvc_size) * 5)}") if int(ceph_capacity) < int(pvc_size) * 5: log.error( f"PVC size is {pvc_size}GiB and it is too large for this system" f" which have only {ceph_capacity}GiB") return # Calculating the file size as 25% of the PVC size # in the end the PVC will be 75% full filesize = self.pvc_obj.size * 0.25 # Change the file size to MB and from int to str file_size = f"{int(filesize * 1024)}M" all_results = [] for test_num in range(self.tests_numbers): test_results = { "test_num": test_num + 1, "dataset": (test_num + 1) * filesize * 1024, # size in MiB "create": { "time": None, "speed": None }, "restore": { "time": None, "speed": None }, } log.info(f"Starting test phase number {test_num}") # Step 1. Run I/O on a pod file. file_name = f"{self.pod_obj.name}-{test_num}" log.info(f"Starting IO on the POD {self.pod_obj.name}") # Going to run only write IO to fill the PVC for the snapshot self.pod_obj.fillup_fs(size=file_size, fio_filename=file_name) # Wait for fio to finish fio_result = self.pod_obj.get_fio_results() err_count = fio_result.get("jobs")[0].get("error") assert ( err_count == 0 ), f"IO error on pod {self.pod_obj.name}. FIO result: {fio_result}" log.info("IO on the PVC Finished") # Verify presence of the file file_path = pod.get_file_path(self.pod_obj, file_name) log.info(f"Actual file path on the pod {file_path}") assert pod.check_file_existence( self.pod_obj, file_path), f"File {file_name} doesn't exist" log.info(f"File {file_name} exists in {self.pod_obj.name}") # Step 2. Calculate md5sum of the file. orig_md5_sum = pod.cal_md5sum(self.pod_obj, file_name) # Step 3. Take a snapshot of the PVC and measure the time of creation. snap_name = self.pvc_obj.name.replace("pvc-test", f"snapshot-test{test_num}") log.info(f"Taking snapshot of the PVC {snap_name}") test_results["create"]["time"] = self.measure_create_snapshot_time( pvc_name=self.pvc_obj.name, snap_name=snap_name, interface=self.interface, ) test_results["create"]["speed"] = int( test_results["dataset"] / test_results["create"]["time"]) log.info( f' Test {test_num} dataset is {test_results["dataset"]} MiB') log.info( f'Snapshot creation time is : {test_results["create"]["time"]} sec.' ) log.info( f'Snapshot speed is : {test_results["create"]["speed"]} MB/sec' ) # Step 4. Restore the PVC from the snapshot and measure the time # Same Storage class of the original PVC sc_name = self.pvc_obj.backed_sc # Size should be same as of the original PVC pvc_size = str(self.pvc_obj.size) + "Gi" # Create pvc out of the snapshot # Both, the snapshot and the restore PVC should be in same namespace log.info("Restoring from the Snapshot") restore_pvc_name = self.pvc_obj.name.replace( "pvc-test", f"restore-pvc{test_num}") restore_pvc_yaml = constants.CSI_RBD_PVC_RESTORE_YAML if self.interface == constants.CEPHFILESYSTEM: restore_pvc_yaml = constants.CSI_CEPHFS_PVC_RESTORE_YAML log.info("Resorting the PVC from Snapshot") restore_pvc_obj = pvc.create_restore_pvc( sc_name=sc_name, snap_name=self.snap_obj.name, namespace=self.snap_obj.namespace, size=pvc_size, pvc_name=restore_pvc_name, restore_pvc_yaml=restore_pvc_yaml, ) helpers.wait_for_resource_state( restore_pvc_obj, constants.STATUS_BOUND, timeout=3600 # setting this to 60 Min. # since it can be take long time to restore, and we want it to finished. ) teardown_factory(restore_pvc_obj) restore_pvc_obj.reload() log.info("PVC was restored from the snapshot") test_results["restore"][ "time"] = helpers.measure_pvc_creation_time( self.interface, restore_pvc_obj.name) test_results["restore"]["speed"] = int( test_results["dataset"] / test_results["restore"]["time"]) log.info( f'Snapshot restore time is : {test_results["restore"]["time"]}' ) log.info( f'restore sped is : {test_results["restore"]["speed"]} MB/sec') # Step 5. Attach a new pod to the restored PVC restore_pod_obj = helpers.create_pod( interface_type=self.interface, pvc_name=restore_pvc_obj.name, namespace=self.snap_obj.namespace, pod_dict_path=constants.NGINX_POD_YAML, ) # Confirm that the pod is running helpers.wait_for_resource_state(resource=restore_pod_obj, state=constants.STATUS_RUNNING) teardown_factory(restore_pod_obj) restore_pod_obj.reload() # Step 6. Verify that the file is present on the new pod also. log.info(f"Checking the existence of {file_name} " f"on restore pod {restore_pod_obj.name}") assert pod.check_file_existence( restore_pod_obj, file_path), f"File {file_name} doesn't exist" log.info(f"File {file_name} exists in {restore_pod_obj.name}") # Step 7. Verify that the md5sum matches log.info(f"Verifying that md5sum of {file_name} " f"on pod {self.pod_obj.name} matches with md5sum " f"of the same file on restore pod {restore_pod_obj.name}") assert pod.verify_data_integrity( restore_pod_obj, file_name, orig_md5_sum), "Data integrity check failed" log.info("Data integrity check passed, md5sum are same") all_results.append(test_results) # logging the test summery, all info in one place for easy log reading c_speed, c_runtime, r_speed, r_runtime = (0 for i in range(4)) log.info("Test summery :") for tst in all_results: c_speed += tst["create"]["speed"] c_runtime += tst["create"]["time"] r_speed += tst["restore"]["speed"] r_runtime += tst["restore"]["time"] log.info( f"Test {tst['test_num']} results : dataset is {tst['dataset']} MiB. " f"Take snapshot time is {tst['create']['time']} " f"at {tst['create']['speed']} MiB/Sec " f"Restore from snapshot time is {tst['restore']['time']} " f"at {tst['restore']['speed']} MiB/Sec ") log.info( f" Average snapshot creation time is {c_runtime / self.tests_numbers} sec." ) log.info( f" Average snapshot creation speed is {c_speed / self.tests_numbers} MiB/sec" ) log.info( f" Average snapshot restore time is {r_runtime / self.tests_numbers} sec." ) log.info( f" Average snapshot restore speed is {r_speed / self.tests_numbers} MiB/sec" )
def test_pvc_snapshot(self, interface, teardown_factory): """ 1. Run I/O on a pod file. 2. Calculate md5sum of the file. 3. Take a snapshot of the PVC. 4. Create a new PVC out of that snapshot. 5. Attach a new pod to it. 6. Verify that the file is present on the new pod also. 7. Verify that the md5sum of the file on the new pod matches with the md5sum of the file on the original pod. Args: interface(str): The type of the interface (e.g. CephBlockPool, CephFileSystem) pvc_factory: A fixture to create new pvc teardown_factory: A fixture to destroy objects """ log.info(f"Running IO on pod {self.pod_obj.name}") file_name = self.pod_obj.name log.info(f"File created during IO {file_name}") self.pod_obj.run_io(storage_type="fs", size="1G", fio_filename=file_name) # Wait for fio to finish fio_result = self.pod_obj.get_fio_results() err_count = fio_result.get("jobs")[0].get("error") assert err_count == 0, (f"IO error on pod {self.pod_obj.name}. " f"FIO result: {fio_result}") log.info(f"Verified IO on pod {self.pod_obj.name}.") # Verify presence of the file file_path = pod.get_file_path(self.pod_obj, file_name) log.info(f"Actual file path on the pod {file_path}") assert pod.check_file_existence( self.pod_obj, file_path), f"File {file_name} doesn't exist" log.info(f"File {file_name} exists in {self.pod_obj.name}") # Calculate md5sum orig_md5_sum = pod.cal_md5sum(self.pod_obj, file_name) # Take a snapshot snap_yaml = constants.CSI_RBD_SNAPSHOT_YAML if interface == constants.CEPHFILESYSTEM: snap_yaml = constants.CSI_CEPHFS_SNAPSHOT_YAML snap_name = helpers.create_unique_resource_name("test", "snapshot") snap_obj = pvc.create_pvc_snapshot( self.pvc_obj.name, snap_yaml, snap_name, self.pvc_obj.namespace, helpers.default_volumesnapshotclass(interface).name, ) snap_obj.ocp.wait_for_resource( condition="true", resource_name=snap_obj.name, column=constants.STATUS_READYTOUSE, timeout=60, ) teardown_factory(snap_obj) # Same Storage class of the original PVC sc_name = self.pvc_obj.backed_sc # Size should be same as of the original PVC pvc_size = str(self.pvc_obj.size) + "Gi" # Create pvc out of the snapshot # Both, the snapshot and the restore PVC should be in same namespace restore_pvc_name = helpers.create_unique_resource_name( "test", "restore-pvc") restore_pvc_yaml = constants.CSI_RBD_PVC_RESTORE_YAML if interface == constants.CEPHFILESYSTEM: restore_pvc_yaml = constants.CSI_CEPHFS_PVC_RESTORE_YAML restore_pvc_obj = pvc.create_restore_pvc( sc_name=sc_name, snap_name=snap_obj.name, namespace=snap_obj.namespace, size=pvc_size, pvc_name=restore_pvc_name, restore_pvc_yaml=restore_pvc_yaml, ) helpers.wait_for_resource_state(restore_pvc_obj, constants.STATUS_BOUND) restore_pvc_obj.reload() teardown_factory(restore_pvc_obj) # Create and attach pod to the pvc restore_pod_obj = helpers.create_pod( interface_type=interface, pvc_name=restore_pvc_obj.name, namespace=snap_obj.namespace, pod_dict_path=constants.NGINX_POD_YAML, ) # Confirm that the pod is running helpers.wait_for_resource_state(resource=restore_pod_obj, state=constants.STATUS_RUNNING) restore_pod_obj.reload() teardown_factory(restore_pod_obj) # Verify that the file is present on the new pod log.info(f"Checking the existence of {file_name} " f"on restore pod {restore_pod_obj.name}") assert pod.check_file_existence( restore_pod_obj, file_path), f"File {file_name} doesn't exist" log.info(f"File {file_name} exists in {restore_pod_obj.name}") # Verify that the md5sum matches log.info(f"Verifying that md5sum of {file_name} " f"on pod {self.pod_obj.name} matches with md5sum " f"of the same file on restore pod {restore_pod_obj.name}") assert pod.verify_data_integrity( restore_pod_obj, file_name, orig_md5_sum), "Data integrity check failed" log.info("Data integrity check passed, md5sum are same") log.info("Running IO on new pod") # Run IO on new pod restore_pod_obj.run_io(storage_type="fs", size="1G", runtime=20) # Wait for fio to finish restore_pod_obj.get_fio_results() log.info("IO finished o new pod")
def factory(snapshot_factory=snapshot_factory): # Get noobaa pods before execution noobaa_pods = pod.get_noobaa_pods() # Get noobaa PVC before execution noobaa_pvc_obj = pvc.get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get( "volumeName") # Take snapshot db-noobaa-db-0 PVC logger.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC") snap_obj = snapshot_factory( pvc_obj=noobaa_pvc_obj[0], wait=True, snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot", ) logger.info( f"Successfully created snapshot {snap_obj.name} and in Ready state" ) # Restore it to PVC logger.info(f"Restoring snapshot {snap_obj.name} to create new PVC") sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName") pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get( "requests").get("storage")) restore_pvc_obj = pvc.create_restore_pvc( sc_name=sc_name, snap_name=snap_obj.name, namespace=snap_obj.namespace, size=pvc_size, pvc_name=f"{snap_obj.name}-restore", volume_mode=snap_obj.parent_volume_mode, access_mode=snap_obj.parent_access_mode, ) restore_pvc_objs.append(restore_pvc_obj) wait_for_resource_state(restore_pvc_obj, constants.STATUS_BOUND) restore_pvc_obj.reload() logger.info(f"Succeesfuly created PVC {restore_pvc_obj.name} " f"from snapshot {snap_obj.name}") # Scale down the statefulset noobaa-db modify_statefulset_replica_count( statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=0 ), f"Failed to scale down the statefulset {constants.NOOBAA_DB_STATEFULSET}" # Get the noobaa-db PVC pvc_obj = OCP(kind=constants.PVC, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name) # Get the restored noobaa PVC and # change the parameter persistentVolumeReclaimPolicy to Retain restored_noobaa_pvc_obj = pvc.get_pvc_objs( pvc_names=[f"{snap_obj.name}-restore"]) restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get( "spec").get("volumeName")) pv_obj = OCP(kind=constants.PV, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Retain {restored_noobaa_pv_name}") # Delete both PVCs pvc.delete_pvcs( pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]]) # Validate original claim db-noobaa-db-0 removed assert validate_pv_delete( pv_name=noobaa_pv_name ), f"PV not deleted, still exist {noobaa_pv_name}" # Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED, resource_name=restored_noobaa_pv_name) # Edit again restore PV and remove the claimRef section logger.info( f"Remove the claimRef section from PVC {restored_noobaa_pv_name}") params = '[{"op": "remove", "path": "/spec/claimRef"}]' pv_obj.patch(resource_name=restored_noobaa_pv_name, params=params, format_type="json") logger.info( f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}" ) # Validate PV is in Available state pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE, resource_name=restored_noobaa_pv_name) # Edit the yaml db-noobaa-db-0.yaml and change the # setting volumeName to restored PVC noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml) noobaa_pvc_yaml.create() # Validate noobaa PVC is in bound state pvc_obj.wait_for_resource( condition=constants.STATUS_BOUND, resource_name=noobaa_pvc_obj[0].name, timeout=120, ) # Scale up the statefulset again assert modify_statefulset_replica_count( statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=1 ), f"Failed to scale up the statefulset {constants.NOOBAA_DB_STATEFULSET}" # Validate noobaa pod is up and running pod_obj = OCP(kind=constants.POD, namespace=defaults.ROOK_CLUSTER_NAMESPACE) pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, resource_count=len(noobaa_pods), selector=constants.NOOBAA_APP_LABEL, ) # Change the parameter persistentVolumeReclaimPolicy to Delete again params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Delete {restored_noobaa_pv_name}") logger.info( "Changed the parameter persistentVolumeReclaimPolicy to Delete again" )