def measure_create_snapshot_time(self, pvc_name, snap_name, namespace, interface, start_time=None): """ Creation volume snapshot, and measure the creation time Args: pvc_name (str): the PVC name to create a snapshot of snap_name (str): the name of the snapshot to be created interface (str): the interface (rbd / cephfs) to used Returns: int : the snapshot creation time in seconds """ # Find the snapshot yaml according to the interface snap_yaml = constants.CSI_RBD_SNAPSHOT_YAML if interface == constants.CEPHFILESYSTEM: snap_yaml = constants.CSI_CEPHFS_SNAPSHOT_YAML # Create the Snapshot of the PVC self.snap_obj = pvc.create_pvc_snapshot( pvc_name=pvc_name, snap_yaml=snap_yaml, snap_name=snap_name, namespace=namespace, sc_name=helpers.default_volumesnapshotclass(interface).name, ) # Wait until the snapshot is bound and ready to use self.snap_obj.ocp.wait_for_resource( condition="true", resource_name=self.snap_obj.name, column=constants.STATUS_READYTOUSE, timeout=600, ) # Getting the snapshot content name self.snap_content = helpers.get_snapshot_content_obj(self.snap_obj) self.snap_uid = (self.snap_content.data.get("spec").get( "volumeSnapshotRef").get("uid")) log.info(f"The snapshot UID is :{self.snap_uid}") # Measure the snapshot creation time c_time = performance_lib.measure_total_snapshot_creation_time( snap_name, start_time) return c_time
def test_snapshot_at_different_usage_level( self, snapshot_factory, snapshot_restore_factory, pod_factory ): """ Test to take multiple snapshots of same PVC when the PVC usage is at 0%, 20%, 40%, 60%, and 80%, then delete the parent PVC and restore the snapshots to create new PVCs. Delete snapshots and attach the restored PVCs to pods to verify the data. """ snapshots = [] usage_percent = [0, 20, 40, 60, 80] for usage in usage_percent: if usage != 0: for pod_obj in self.pods: log.info(f"Running IO on pod {pod_obj.name} to utilize {usage}%") pod_obj.pvc.filename = f"{pod_obj.name}_{usage}" pod_obj.run_io( storage_type="fs", size=f"{int(self.pvc_size/len(usage_percent))}G", runtime=20, fio_filename=pod_obj.pvc.filename, ) log.info(f"IO started on all pods to utilize {usage}%") for pod_obj in self.pods: # Wait for fio to finish pod_obj.get_fio_results() log.info( f"IO to utilize {usage}% finished on pod " f"{pod_obj.name}" ) # Calculate md5sum md5_sum = pod.cal_md5sum(pod_obj, pod_obj.pvc.filename) if not getattr(pod_obj.pvc, "md5_sum", None): setattr(pod_obj.pvc, "md5_sum", {}) pod_obj.pvc.md5_sum[pod_obj.pvc.filename] = md5_sum # Take snapshot of all PVCs log.info(f"Creating snapshot of all PVCs at {usage}%") for pvc_obj in self.pvcs: log.info(f"Creating snapshot of PVC {pvc_obj.name} at {usage}%") snap_obj = snapshot_factory(pvc_obj, wait=False) # Set a dict containing filename:md5sum for later verification setattr(snap_obj, "md5_sum", deepcopy(getattr(pvc_obj, "md5_sum", {}))) snap_obj.usage_on_mount = get_used_space_on_mount_point( pvc_obj.get_attached_pods()[0] ) snapshots.append(snap_obj) log.info(f"Created snapshot of PVC {pvc_obj.name} at {usage}%") log.info(f"Created snapshot of all PVCs at {usage}%") log.info("Snapshots creation completed.") # Verify snapshots are ready log.info("Verify snapshots are ready") for snapshot in snapshots: snapshot.ocp.wait_for_resource( condition="true", resource_name=snapshot.name, column=constants.STATUS_READYTOUSE, timeout=90, ) # Delete pods log.info("Deleting the pods") for pod_obj in self.pods: pod_obj.delete() pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name) log.info("Deleted all the pods") # Delete parent PVCs log.info("Deleting parent PVCs") for pvc_obj in self.pvcs: pv_obj = pvc_obj.backed_pv_obj pvc_obj.delete() pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name) log.info( f"Deleted PVC {pvc_obj.name}. Verifying whether PV " f"{pv_obj.name} is deleted." ) pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name) log.info( "Deleted parent PVCs before restoring snapshot. " "PVs are also deleted." ) restore_pvc_objs = [] # Create PVCs out of the snapshots log.info("Creating new PVCs from snapshots") for snapshot in snapshots: log.info(f"Creating a PVC from snapshot {snapshot.name}") restore_pvc_obj = snapshot_restore_factory( snapshot_obj=snapshot, size=f"{self.pvc_size}Gi", volume_mode=snapshot.parent_volume_mode, access_mode=snapshot.parent_access_mode, status="", ) log.info( f"Created PVC {restore_pvc_obj.name} from snapshot " f"{snapshot.name}" ) restore_pvc_objs.append(restore_pvc_obj) log.info("Created new PVCs from all the snapshots") # Confirm that the restored PVCs are Bound # Increased wait time to 600 seconds as a workaround for BZ 1899968 # TODO: Revert wait time to 200 seconds once BZ 1899968 is fixed log.info("Verify the restored PVCs are Bound") for pvc_obj in restore_pvc_objs: wait_for_resource_state( resource=pvc_obj, state=constants.STATUS_BOUND, timeout=600 ) pvc_obj.reload() log.info("Verified: Restored PVCs are Bound.") snapcontent_objs = [] # Get VolumeSnapshotContent form VolumeSnapshots and delete # VolumeSnapshots log.info("Deleting snapshots") for snapshot in snapshots: snapcontent_objs.append(get_snapshot_content_obj(snap_obj=snapshot)) snapshot.delete() # Verify volume snapshots are deleted log.info("Verify snapshots are deleted") for snapshot in snapshots: snapshot.ocp.wait_for_delete(resource_name=snapshot.name) log.info("Verified: Snapshots are deleted") # Verify VolumeSnapshotContents are deleted for snapcontent_obj in snapcontent_objs: snapcontent_obj.ocp.wait_for_delete( resource_name=snapcontent_obj.name, timeout=180 ) # Attach the restored PVCs to pods log.info("Attach the restored PVCs to pods") restore_pod_objs = [] for restore_pvc_obj in restore_pvc_objs: interface = ( constants.CEPHFILESYSTEM if (constants.CEPHFS_INTERFACE in restore_pvc_obj.snapshot.parent_sc) else constants.CEPHBLOCKPOOL ) restore_pod_obj = pod_factory( interface=interface, pvc=restore_pvc_obj, status="" ) log.info( f"Attached the PVC {restore_pvc_obj.name} to pod " f"{restore_pod_obj.name}" ) restore_pod_objs.append(restore_pod_obj) # Verify the new pods are running log.info("Verify the new pods are running") for pod_obj in restore_pod_objs: timeout = ( 300 if config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM else 60 ) wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout) log.info("Verified: New pods are running") # Verify md5sum of files log.info("Verifying md5sum of files on all the pods") for restore_pod_obj in restore_pod_objs: log.info( f"Verifying md5sum of these files on pod " f"{restore_pod_obj.name}:" f"{restore_pod_obj.pvc.snapshot.md5_sum}" ) for ( file_name, actual_md5_sum, ) in restore_pod_obj.pvc.snapshot.md5_sum.items(): file_path = pod.get_file_path(restore_pod_obj, file_name) log.info( f"Checking the existence of file {file_name} on pod " f"{restore_pod_obj.name}" ) assert pod.check_file_existence(restore_pod_obj, file_path), ( f"File {file_name} does not exist on pod " f"{restore_pod_obj.name}" ) log.info(f"File {file_name} exists on pod {restore_pod_obj.name}") # Verify that the md5sum matches log.info( f"Verifying md5sum of file {file_name} on pod " f"{restore_pod_obj.name}" ) pod.verify_data_integrity(restore_pod_obj, file_name, actual_md5_sum) log.info( f"Verified md5sum of file {file_name} on pod " f"{restore_pod_obj.name}" ) log.info( f"Verified md5sum of these files on pod " f"{restore_pod_obj.name}:" f"{restore_pod_obj.pvc.snapshot.md5_sum}" ) log.info("md5sum verified") # Verify usage on mount point log.info("Verify usage on new pods") for pod_obj in restore_pod_objs: usage_on_pod = get_used_space_on_mount_point(pod_obj) assert usage_on_pod == pod_obj.pvc.snapshot.usage_on_mount, ( f"Usage on mount point is not the expected value on pod " f"{pod_obj.name}. Usage in percentage {usage_on_pod}. " f"Expected usage in percentage " f"{pod_obj.pvc.snapshot.usage_on_mount}" ) log.info( f"Verified usage on new pod {pod_obj.name}. Usage in " f"percentage {usage_on_pod}. Expected usage in percentage " f"{pod_obj.pvc.snapshot.usage_on_mount}" ) log.info("Verified usage on new pods")
def test_encrypted_rbd_block_pvc_snapshot( self, kms_provider, snapshot_factory, snapshot_restore_factory, pod_factory, kv_version, ): """ Test to take snapshots of encrypted RBD Block VolumeMode PVCs """ log.info( "Check for encrypted device, find initial md5sum value and run IO on all pods" ) for vol_handle, pod_obj in zip(self.vol_handles, self.pod_objs): # Verify whether encrypted device is present inside the pod if pod_obj.exec_sh_cmd_on_pod( command=f"lsblk | grep {vol_handle} | grep crypt"): log.info(f"Encrypted device found in {pod_obj.name}") else: raise ResourceNotFoundError( f"Encrypted device not found in {pod_obj.name}") # Find initial md5sum pod_obj.md5sum_before_io = cal_md5sum( pod_obj=pod_obj, file_name=pod_obj.get_storage_path(storage_type="block"), block=True, ) pod_obj.run_io( storage_type="block", size=f"{self.pvc_size - 1}G", io_direction="write", runtime=60, ) log.info("IO started on all pods") # Wait for IO completion for pod_obj in self.pod_objs: pod_obj.get_fio_results() log.info("IO completed on all pods") snap_objs, snap_handles = ([] for i in range(2)) # Verify md5sum has changed after IO. Create snapshot log.info( "Verify md5sum has changed after IO and create snapshot from all PVCs" ) for pod_obj in self.pod_objs: md5sum_after_io = cal_md5sum( pod_obj=pod_obj, file_name=pod_obj.get_storage_path(storage_type="block"), block=True, ) assert (pod_obj.md5sum_before_io != md5sum_after_io ), f"md5sum has not changed after IO on pod {pod_obj.name}" log.info(f"Creating snapshot of PVC {pod_obj.pvc.name}") snap_obj = snapshot_factory(pod_obj.pvc, wait=False) snap_obj.md5sum = md5sum_after_io snap_objs.append(snap_obj) log.info("Snapshots created") # Verify snapshots are ready and verify if encryption key is created in vault log.info("Verify snapshots are ready") for snap_obj in snap_objs: snap_obj.ocp.wait_for_resource( condition="true", resource_name=snap_obj.name, column=constants.STATUS_READYTOUSE, timeout=180, ) snapshot_content = get_snapshot_content_obj(snap_obj=snap_obj) snap_handle = snapshot_content.get().get("status").get( "snapshotHandle") if kms_provider == constants.VAULT_KMS_PROVIDER: if kms.is_key_present_in_path( key=snap_handle, path=self.kms.vault_backend_path): log.info(f"Vault: Found key for snapshot {snap_obj.name}") else: raise ResourceNotFoundError( f"Vault: Key not found for snapshot {snap_obj.name}") snap_handles.append(snap_handle) # Delete pods log.info("Deleting the pods") for pod_obj in self.pod_objs: pod_obj.delete() pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name) log.info("Deleted all the pods") # Delete parent PVCs to verify snapshot is independent log.info("Deleting parent PVCs") for pvc_obj in self.pvc_objs: pv_obj = pvc_obj.backed_pv_obj pvc_obj.delete() pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name) log.info(f"Deleted PVC {pvc_obj.name}. Verifying whether PV " f"{pv_obj.name} is deleted.") pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name) log.info( "All parent PVCs and PVs are deleted before restoring snapshot.") restore_pvc_objs, restore_vol_handles = ([] for i in range(2)) # Create PVCs out of the snapshots log.info("Creating new PVCs from snapshots") for snap_obj in snap_objs: log.info(f"Creating a PVC from snapshot {snap_obj.name}") restore_pvc_obj = snapshot_restore_factory( snapshot_obj=snap_obj, storageclass=self.sc_obj.name, size=f"{self.pvc_size}Gi", volume_mode=snap_obj.parent_volume_mode, access_mode=snap_obj.parent_access_mode, status="", ) log.info(f"Created PVC {restore_pvc_obj.name} from snapshot " f"{snap_obj.name}") restore_pvc_obj.md5sum = snap_obj.md5sum restore_pvc_objs.append(restore_pvc_obj) log.info("Created new PVCs from all the snapshots") # Confirm that the restored PVCs are Bound log.info("Verify the restored PVCs are Bound") for pvc_obj in restore_pvc_objs: wait_for_resource_state(resource=pvc_obj, state=constants.STATUS_BOUND, timeout=180) pvc_obj.reload() log.info("Verified: Restored PVCs are Bound.") # Attach the restored PVCs to pods. Attach RWX PVC on two pods log.info("Attach the restored PVCs to pods") restore_pod_objs = create_pods( restore_pvc_objs, pod_factory, constants.CEPHBLOCKPOOL, pods_for_rwx=1, status="", ) # Verify the new pods are running log.info("Verify the new pods are running") for pod_obj in restore_pod_objs: timeout = (300 if config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM else 60) wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout) log.info("Verified: New pods are running") # Verify encryption keys are created for restored PVCs in Vault for pvc_obj in restore_pvc_objs: pv_obj = pvc_obj.backed_pv_obj vol_handle = pv_obj.get().get("spec").get("csi").get( "volumeHandle") restore_vol_handles.append(vol_handle) if kms_provider == constants.VAULT_KMS_PROVIDER: if kms.is_key_present_in_path( key=vol_handle, path=self.kms.vault_backend_path): log.info( f"Vault: Found key for restore PVC {pvc_obj.name}") else: raise ResourceNotFoundError( f"Vault: Key not found for restored PVC {pvc_obj.name}" ) # Verify encrypted device is present and md5sum on all pods for vol_handle, pod_obj in zip(restore_vol_handles, restore_pod_objs): if pod_obj.exec_sh_cmd_on_pod( command=f"lsblk | grep {vol_handle} | grep crypt"): log.info(f"Encrypted device found in {pod_obj.name}") else: raise ResourceNotFoundError( f"Encrypted device not found in {pod_obj.name}") log.info(f"Verifying md5sum on pod {pod_obj.name}") verify_data_integrity( pod_obj=pod_obj, file_name=pod_obj.get_storage_path(storage_type="block"), original_md5sum=pod_obj.pvc.md5sum, block=True, ) log.info(f"Verified md5sum on pod {pod_obj.name}") # Run IO on new pods log.info("Starting IO on new pods") for pod_obj in restore_pod_objs: pod_obj.run_io(storage_type="block", size="500M", runtime=15) # Wait for IO completion on new pods log.info("Waiting for IO completion on new pods") for pod_obj in restore_pod_objs: pod_obj.get_fio_results() log.info("IO completed on new pods.") # Delete the restored pods, PVC and snapshots log.info("Deleting pods using restored PVCs") for pod_obj in restore_pod_objs: pod_obj.delete() pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name) log.info("Deleting restored PVCs") for pvc_obj in restore_pvc_objs: pv_obj = pvc_obj.backed_pv_obj pvc_obj.delete() pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name) log.info("Deleting the snapshots") for snap_obj in snap_objs: snapcontent_obj = get_snapshot_content_obj(snap_obj=snap_obj) snap_obj.delete() snapcontent_obj.ocp.wait_for_delete( resource_name=snapcontent_obj.name) if kms_provider == constants.VAULT_KMS_PROVIDER: # Verify if keys for PVCs and snapshots are deleted from Vault if kv_version == "v1" or Version.coerce( config.ENV_DATA["ocs_version"]) >= Version.coerce("4.9"): log.info( "Verify whether the keys for PVCs and snapshots are deleted in vault" ) for key in self.vol_handles + snap_handles + restore_vol_handles: if not kms.is_key_present_in_path( key=key, path=self.kms.vault_backend_path): log.info(f"Vault: Key deleted for {key}") else: raise KMSResourceCleaneupError( f"Vault: Key deletion failed for {key}") log.info("All keys from vault were deleted")
def test_encrypted_pvc_snapshot( self, kv_version, pv_encryption_kms_setup_factory, storageclass_factory, pgsql_factory_fixture, snapshot_factory, snapshot_restore_factory, pgsql_teardown, ): """ 1. Create encrypted storage class 2. Deploy PGSQL workload using created sc 3. Take a snapshot of the pgsql PVC. 4. Create a new PVC out of that snapshot or restore snapshot 5. Attach a new pgsql pod to it. 6. Create pgbench benchmark to new pgsql pod 7. Verify if key is created """ pgsql_teardown log.info("Setting up csi-kms-connection-details configmap") self.vault = pv_encryption_kms_setup_factory(kv_version) log.info("csi-kms-connection-details setup successful") # Create an encryption enabled storageclass for RBD self.sc_obj = storageclass_factory( interface=CEPHBLOCKPOOL, encrypted=True, encryption_kms_id=self.vault.kmsid, ) # Create ceph-csi-kms-token in the tenant namespace self.vault.vault_path_token = self.vault.generate_vault_token() self.vault.create_vault_csi_kms_token(namespace=BMO_NAME) # Deploy PGSQL workload log.info("Deploying pgsql workloads") pgsql = pgsql_factory_fixture(replicas=1, sc_name=self.sc_obj.name) # Get postgres pvc list obj postgres_pvcs_obj = pgsql.get_postgres_pvc() # Take a snapshot of it snapshots, restore_pvc_objs = self.create_snapshot( pgsql, postgres_pvcs_obj, snapshot_factory, snapshot_restore_factory, self.sc_obj.name, ) # Verify encryption keys are created for snapshots in Vault for snap_obj in snapshots: snapshot_content = get_snapshot_content_obj(snap_obj=snap_obj) snap_handle = snapshot_content.get().get("status").get( "snapshotHandle") if kms.is_key_present_in_path(key=snap_handle, path=self.vault.vault_backend_path): log.info(f"Vault: Found key for snapshot {snap_obj.name}") else: raise ResourceNotFoundError( f"Vault: Key not found for snapshot {snap_obj.name}") # Verify encryption keys are created for restored PVCs in Vault for pvc_obj in restore_pvc_objs: pv_obj = pvc_obj.backed_pv_obj vol_handle = pv_obj.get().get("spec").get("csi").get( "volumeHandle") if kms.is_key_present_in_path(key=vol_handle, path=self.vault.vault_backend_path): log.info(f"Vault: Found key for restore PVC {pvc_obj.name}") else: raise ResourceNotFoundError( f"Vault: Key not found for restored PVC {pvc_obj.name}")