def measure_create_snapshot_time(self,
pvc_name,
snap_name,
namespace,
interface,
start_time=None):
"""
Creation volume snapshot, and measure the creation time
Args:
pvc_name (str): the PVC name to create a snapshot of
snap_name (str): the name of the snapshot to be created
interface (str): the interface (rbd / cephfs) to used
Returns:
int : the snapshot creation time in seconds
"""
# Find the snapshot yaml according to the interface
snap_yaml = constants.CSI_RBD_SNAPSHOT_YAML
if interface == constants.CEPHFILESYSTEM:
snap_yaml = constants.CSI_CEPHFS_SNAPSHOT_YAML
# Create the Snapshot of the PVC
self.snap_obj = pvc.create_pvc_snapshot(
pvc_name=pvc_name,
snap_yaml=snap_yaml,
snap_name=snap_name,
namespace=namespace,
sc_name=helpers.default_volumesnapshotclass(interface).name,
)
# Wait until the snapshot is bound and ready to use
self.snap_obj.ocp.wait_for_resource(
condition="true",
resource_name=self.snap_obj.name,
column=constants.STATUS_READYTOUSE,
timeout=600,
)
# Getting the snapshot content name
self.snap_content = helpers.get_snapshot_content_obj(self.snap_obj)
self.snap_uid = (self.snap_content.data.get("spec").get(
"volumeSnapshotRef").get("uid"))
log.info(f"The snapshot UID is :{self.snap_uid}")
# Measure the snapshot creation time
c_time = performance_lib.measure_total_snapshot_creation_time(
snap_name, start_time)
return c_time
def test_snapshot_at_different_usage_level(
self, snapshot_factory, snapshot_restore_factory, pod_factory
):
"""
Test to take multiple snapshots of same PVC when the PVC usage is at
0%, 20%, 40%, 60%, and 80%, then delete the parent PVC and restore the
snapshots to create new PVCs. Delete snapshots and attach the restored
PVCs to pods to verify the data.
"""
snapshots = []
usage_percent = [0, 20, 40, 60, 80]
for usage in usage_percent:
if usage != 0:
for pod_obj in self.pods:
log.info(f"Running IO on pod {pod_obj.name} to utilize {usage}%")
pod_obj.pvc.filename = f"{pod_obj.name}_{usage}"
pod_obj.run_io(
storage_type="fs",
size=f"{int(self.pvc_size/len(usage_percent))}G",
runtime=20,
fio_filename=pod_obj.pvc.filename,
)
log.info(f"IO started on all pods to utilize {usage}%")
for pod_obj in self.pods:
# Wait for fio to finish
pod_obj.get_fio_results()
log.info(
f"IO to utilize {usage}% finished on pod " f"{pod_obj.name}"
)
# Calculate md5sum
md5_sum = pod.cal_md5sum(pod_obj, pod_obj.pvc.filename)
if not getattr(pod_obj.pvc, "md5_sum", None):
setattr(pod_obj.pvc, "md5_sum", {})
pod_obj.pvc.md5_sum[pod_obj.pvc.filename] = md5_sum
# Take snapshot of all PVCs
log.info(f"Creating snapshot of all PVCs at {usage}%")
for pvc_obj in self.pvcs:
log.info(f"Creating snapshot of PVC {pvc_obj.name} at {usage}%")
snap_obj = snapshot_factory(pvc_obj, wait=False)
# Set a dict containing filename:md5sum for later verification
setattr(snap_obj, "md5_sum", deepcopy(getattr(pvc_obj, "md5_sum", {})))
snap_obj.usage_on_mount = get_used_space_on_mount_point(
pvc_obj.get_attached_pods()[0]
)
snapshots.append(snap_obj)
log.info(f"Created snapshot of PVC {pvc_obj.name} at {usage}%")
log.info(f"Created snapshot of all PVCs at {usage}%")
log.info("Snapshots creation completed.")
# Verify snapshots are ready
log.info("Verify snapshots are ready")
for snapshot in snapshots:
snapshot.ocp.wait_for_resource(
condition="true",
resource_name=snapshot.name,
column=constants.STATUS_READYTOUSE,
timeout=90,
)
# Delete pods
log.info("Deleting the pods")
for pod_obj in self.pods:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleted all the pods")
# Delete parent PVCs
log.info("Deleting parent PVCs")
for pvc_obj in self.pvcs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
log.info(
f"Deleted PVC {pvc_obj.name}. Verifying whether PV "
f"{pv_obj.name} is deleted."
)
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info(
"Deleted parent PVCs before restoring snapshot. " "PVs are also deleted."
)
restore_pvc_objs = []
# Create PVCs out of the snapshots
log.info("Creating new PVCs from snapshots")
for snapshot in snapshots:
log.info(f"Creating a PVC from snapshot {snapshot.name}")
restore_pvc_obj = snapshot_restore_factory(
snapshot_obj=snapshot,
size=f"{self.pvc_size}Gi",
volume_mode=snapshot.parent_volume_mode,
access_mode=snapshot.parent_access_mode,
status="",
)
log.info(
f"Created PVC {restore_pvc_obj.name} from snapshot " f"{snapshot.name}"
)
restore_pvc_objs.append(restore_pvc_obj)
log.info("Created new PVCs from all the snapshots")
# Confirm that the restored PVCs are Bound
# Increased wait time to 600 seconds as a workaround for BZ 1899968
# TODO: Revert wait time to 200 seconds once BZ 1899968 is fixed
log.info("Verify the restored PVCs are Bound")
for pvc_obj in restore_pvc_objs:
wait_for_resource_state(
resource=pvc_obj, state=constants.STATUS_BOUND, timeout=600
)
pvc_obj.reload()
log.info("Verified: Restored PVCs are Bound.")
snapcontent_objs = []
# Get VolumeSnapshotContent form VolumeSnapshots and delete
# VolumeSnapshots
log.info("Deleting snapshots")
for snapshot in snapshots:
snapcontent_objs.append(get_snapshot_content_obj(snap_obj=snapshot))
snapshot.delete()
# Verify volume snapshots are deleted
log.info("Verify snapshots are deleted")
for snapshot in snapshots:
snapshot.ocp.wait_for_delete(resource_name=snapshot.name)
log.info("Verified: Snapshots are deleted")
# Verify VolumeSnapshotContents are deleted
for snapcontent_obj in snapcontent_objs:
snapcontent_obj.ocp.wait_for_delete(
resource_name=snapcontent_obj.name, timeout=180
)
# Attach the restored PVCs to pods
log.info("Attach the restored PVCs to pods")
restore_pod_objs = []
for restore_pvc_obj in restore_pvc_objs:
interface = (
constants.CEPHFILESYSTEM
if (constants.CEPHFS_INTERFACE in restore_pvc_obj.snapshot.parent_sc)
else constants.CEPHBLOCKPOOL
)
restore_pod_obj = pod_factory(
interface=interface, pvc=restore_pvc_obj, status=""
)
log.info(
f"Attached the PVC {restore_pvc_obj.name} to pod "
f"{restore_pod_obj.name}"
)
restore_pod_objs.append(restore_pod_obj)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in restore_pod_objs:
timeout = (
300
if config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM
else 60
)
wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout)
log.info("Verified: New pods are running")
# Verify md5sum of files
log.info("Verifying md5sum of files on all the pods")
for restore_pod_obj in restore_pod_objs:
log.info(
f"Verifying md5sum of these files on pod "
f"{restore_pod_obj.name}:"
f"{restore_pod_obj.pvc.snapshot.md5_sum}"
)
for (
file_name,
actual_md5_sum,
) in restore_pod_obj.pvc.snapshot.md5_sum.items():
file_path = pod.get_file_path(restore_pod_obj, file_name)
log.info(
f"Checking the existence of file {file_name} on pod "
f"{restore_pod_obj.name}"
)
assert pod.check_file_existence(restore_pod_obj, file_path), (
f"File {file_name} does not exist on pod " f"{restore_pod_obj.name}"
)
log.info(f"File {file_name} exists on pod {restore_pod_obj.name}")
# Verify that the md5sum matches
log.info(
f"Verifying md5sum of file {file_name} on pod "
f"{restore_pod_obj.name}"
)
pod.verify_data_integrity(restore_pod_obj, file_name, actual_md5_sum)
log.info(
f"Verified md5sum of file {file_name} on pod "
f"{restore_pod_obj.name}"
)
log.info(
f"Verified md5sum of these files on pod "
f"{restore_pod_obj.name}:"
f"{restore_pod_obj.pvc.snapshot.md5_sum}"
)
log.info("md5sum verified")
# Verify usage on mount point
log.info("Verify usage on new pods")
for pod_obj in restore_pod_objs:
usage_on_pod = get_used_space_on_mount_point(pod_obj)
assert usage_on_pod == pod_obj.pvc.snapshot.usage_on_mount, (
f"Usage on mount point is not the expected value on pod "
f"{pod_obj.name}. Usage in percentage {usage_on_pod}. "
f"Expected usage in percentage "
f"{pod_obj.pvc.snapshot.usage_on_mount}"
)
log.info(
f"Verified usage on new pod {pod_obj.name}. Usage in "
f"percentage {usage_on_pod}. Expected usage in percentage "
f"{pod_obj.pvc.snapshot.usage_on_mount}"
)
log.info("Verified usage on new pods")
def test_encrypted_rbd_block_pvc_snapshot(
self,
kms_provider,
snapshot_factory,
snapshot_restore_factory,
pod_factory,
kv_version,
):
"""
Test to take snapshots of encrypted RBD Block VolumeMode PVCs
"""
log.info(
"Check for encrypted device, find initial md5sum value and run IO on all pods"
)
for vol_handle, pod_obj in zip(self.vol_handles, self.pod_objs):
# Verify whether encrypted device is present inside the pod
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
# Find initial md5sum
pod_obj.md5sum_before_io = cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
pod_obj.run_io(
storage_type="block",
size=f"{self.pvc_size - 1}G",
io_direction="write",
runtime=60,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in self.pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
snap_objs, snap_handles = ([] for i in range(2))
# Verify md5sum has changed after IO. Create snapshot
log.info(
"Verify md5sum has changed after IO and create snapshot from all PVCs"
)
for pod_obj in self.pod_objs:
md5sum_after_io = cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
assert (pod_obj.md5sum_before_io != md5sum_after_io
), f"md5sum has not changed after IO on pod {pod_obj.name}"
log.info(f"Creating snapshot of PVC {pod_obj.pvc.name}")
snap_obj = snapshot_factory(pod_obj.pvc, wait=False)
snap_obj.md5sum = md5sum_after_io
snap_objs.append(snap_obj)
log.info("Snapshots created")
# Verify snapshots are ready and verify if encryption key is created in vault
log.info("Verify snapshots are ready")
for snap_obj in snap_objs:
snap_obj.ocp.wait_for_resource(
condition="true",
resource_name=snap_obj.name,
column=constants.STATUS_READYTOUSE,
timeout=180,
)
snapshot_content = get_snapshot_content_obj(snap_obj=snap_obj)
snap_handle = snapshot_content.get().get("status").get(
"snapshotHandle")
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=snap_handle, path=self.kms.vault_backend_path):
log.info(f"Vault: Found key for snapshot {snap_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for snapshot {snap_obj.name}")
snap_handles.append(snap_handle)
# Delete pods
log.info("Deleting the pods")
for pod_obj in self.pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleted all the pods")
# Delete parent PVCs to verify snapshot is independent
log.info("Deleting parent PVCs")
for pvc_obj in self.pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
log.info(f"Deleted PVC {pvc_obj.name}. Verifying whether PV "
f"{pv_obj.name} is deleted.")
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info(
"All parent PVCs and PVs are deleted before restoring snapshot.")
restore_pvc_objs, restore_vol_handles = ([] for i in range(2))
# Create PVCs out of the snapshots
log.info("Creating new PVCs from snapshots")
for snap_obj in snap_objs:
log.info(f"Creating a PVC from snapshot {snap_obj.name}")
restore_pvc_obj = snapshot_restore_factory(
snapshot_obj=snap_obj,
storageclass=self.sc_obj.name,
size=f"{self.pvc_size}Gi",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
status="",
)
log.info(f"Created PVC {restore_pvc_obj.name} from snapshot "
f"{snap_obj.name}")
restore_pvc_obj.md5sum = snap_obj.md5sum
restore_pvc_objs.append(restore_pvc_obj)
log.info("Created new PVCs from all the snapshots")
# Confirm that the restored PVCs are Bound
log.info("Verify the restored PVCs are Bound")
for pvc_obj in restore_pvc_objs:
wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=180)
pvc_obj.reload()
log.info("Verified: Restored PVCs are Bound.")
# Attach the restored PVCs to pods. Attach RWX PVC on two pods
log.info("Attach the restored PVCs to pods")
restore_pod_objs = create_pods(
restore_pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status="",
)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in restore_pod_objs:
timeout = (300 if config.ENV_DATA["platform"]
== constants.IBMCLOUD_PLATFORM else 60)
wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout)
log.info("Verified: New pods are running")
# Verify encryption keys are created for restored PVCs in Vault
for pvc_obj in restore_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
restore_vol_handles.append(vol_handle)
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(
f"Vault: Found key for restore PVC {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for restored PVC {pvc_obj.name}"
)
# Verify encrypted device is present and md5sum on all pods
for vol_handle, pod_obj in zip(restore_vol_handles, restore_pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
log.info(f"Verifying md5sum on pod {pod_obj.name}")
verify_data_integrity(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
original_md5sum=pod_obj.pvc.md5sum,
block=True,
)
log.info(f"Verified md5sum on pod {pod_obj.name}")
# Run IO on new pods
log.info("Starting IO on new pods")
for pod_obj in restore_pod_objs:
pod_obj.run_io(storage_type="block", size="500M", runtime=15)
# Wait for IO completion on new pods
log.info("Waiting for IO completion on new pods")
for pod_obj in restore_pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on new pods.")
# Delete the restored pods, PVC and snapshots
log.info("Deleting pods using restored PVCs")
for pod_obj in restore_pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleting restored PVCs")
for pvc_obj in restore_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info("Deleting the snapshots")
for snap_obj in snap_objs:
snapcontent_obj = get_snapshot_content_obj(snap_obj=snap_obj)
snap_obj.delete()
snapcontent_obj.ocp.wait_for_delete(
resource_name=snapcontent_obj.name)
if kms_provider == constants.VAULT_KMS_PROVIDER:
# Verify if keys for PVCs and snapshots are deleted from Vault
if kv_version == "v1" or Version.coerce(
config.ENV_DATA["ocs_version"]) >= Version.coerce("4.9"):
log.info(
"Verify whether the keys for PVCs and snapshots are deleted in vault"
)
for key in self.vol_handles + snap_handles + restore_vol_handles:
if not kms.is_key_present_in_path(
key=key, path=self.kms.vault_backend_path):
log.info(f"Vault: Key deleted for {key}")
else:
raise KMSResourceCleaneupError(
f"Vault: Key deletion failed for {key}")
log.info("All keys from vault were deleted")
def test_encrypted_pvc_snapshot(
self,
kv_version,
pv_encryption_kms_setup_factory,
storageclass_factory,
pgsql_factory_fixture,
snapshot_factory,
snapshot_restore_factory,
pgsql_teardown,
):
"""
1. Create encrypted storage class
2. Deploy PGSQL workload using created sc
3. Take a snapshot of the pgsql PVC.
4. Create a new PVC out of that snapshot or restore snapshot
5. Attach a new pgsql pod to it.
6. Create pgbench benchmark to new pgsql pod
7. Verify if key is created
"""
pgsql_teardown
log.info("Setting up csi-kms-connection-details configmap")
self.vault = pv_encryption_kms_setup_factory(kv_version)
log.info("csi-kms-connection-details setup successful")
# Create an encryption enabled storageclass for RBD
self.sc_obj = storageclass_factory(
interface=CEPHBLOCKPOOL,
encrypted=True,
encryption_kms_id=self.vault.kmsid,
)
# Create ceph-csi-kms-token in the tenant namespace
self.vault.vault_path_token = self.vault.generate_vault_token()
self.vault.create_vault_csi_kms_token(namespace=BMO_NAME)
# Deploy PGSQL workload
log.info("Deploying pgsql workloads")
pgsql = pgsql_factory_fixture(replicas=1, sc_name=self.sc_obj.name)
# Get postgres pvc list obj
postgres_pvcs_obj = pgsql.get_postgres_pvc()
# Take a snapshot of it
snapshots, restore_pvc_objs = self.create_snapshot(
pgsql,
postgres_pvcs_obj,
snapshot_factory,
snapshot_restore_factory,
self.sc_obj.name,
)
# Verify encryption keys are created for snapshots in Vault
for snap_obj in snapshots:
snapshot_content = get_snapshot_content_obj(snap_obj=snap_obj)
snap_handle = snapshot_content.get().get("status").get(
"snapshotHandle")
if kms.is_key_present_in_path(key=snap_handle,
path=self.vault.vault_backend_path):
log.info(f"Vault: Found key for snapshot {snap_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for snapshot {snap_obj.name}")
# Verify encryption keys are created for restored PVCs in Vault
for pvc_obj in restore_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
if kms.is_key_present_in_path(key=vol_handle,
path=self.vault.vault_backend_path):
log.info(f"Vault: Found key for restore PVC {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for restored PVC {pvc_obj.name}")