def test_rbd_pv_encryption_vaulttenantsa(
self,
project_factory,
storageclass_factory,
multi_pvc_factory,
pod_factory,
kv_version,
):
"""
Test to verify creation and deletion of encrypted RBD PVC using vaulttenantsa method
"""
# Create a project
proj_obj = project_factory()
# Create an encryption enabled storageclass for RBD
sc_obj = storageclass_factory(
interface=constants.CEPHBLOCKPOOL,
encrypted=True,
encryption_kms_id=self.kms.kmsid,
)
# Create serviceaccount in the tenant namespace
self.kms.create_tenant_sa(namespace=proj_obj.namespace)
# Create role in Vault
self.kms.create_vault_kube_auth_role(namespace=proj_obj.namespace)
# Create RBD PVCs with volume mode Block
pvc_size = 5
pvc_objs = multi_pvc_factory(
interface=constants.CEPHBLOCKPOOL,
project=proj_obj,
storageclass=sc_obj,
size=pvc_size,
access_modes=[
f"{constants.ACCESS_MODE_RWX}-Block",
f"{constants.ACCESS_MODE_RWO}-Block",
],
status=constants.STATUS_BOUND,
num_of_pvc=3,
wait_each=False,
)
# Create pods
pod_objs = create_pods(
pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status=constants.STATUS_RUNNING,
)
# Verify if the key is created in Vault
vol_handles = []
for pvc_obj in pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
vol_handles.append(vol_handle)
# Check if encryption key is created in Vault
if kms.is_key_present_in_path(key=vol_handle,
path=self.kms.vault_backend_path):
log.info(f"Vault: Found key for {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for {pvc_obj.name}")
# Verify whether encrypted device is present inside the pod and run IO
for vol_handle, pod_obj in zip(vol_handles, pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
pod_obj.run_io(
storage_type="block",
size=f"{pvc_size - 1}G",
io_direction="write",
runtime=60,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
# Delete the pod
for pod_obj in pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
# Delete the PVC
for pvc_obj in pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
# Verify whether the key is deleted in Vault
for vol_handle in vol_handles:
if not kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(f"Vault: Key deleted for {vol_handle}")
else:
raise KMSResourceCleaneupError(
f"Vault: Key deletion failed for {vol_handle}")
def setup(
self,
kv_version,
kms_provider,
use_vault_namespace,
pv_encryption_kms_setup_factory,
project_factory,
multi_pvc_factory,
pod_factory,
storageclass_factory,
):
"""
Setup csi-kms-connection-details configmap and create resources for the test
"""
log.info("Setting up csi-kms-connection-details configmap")
self.kms = pv_encryption_kms_setup_factory(kv_version,
use_vault_namespace)
log.info("csi-kms-connection-details setup successful")
# Create a project
self.proj_obj = project_factory()
# Create an encryption enabled storageclass for RBD
self.sc_obj = storageclass_factory(
interface=constants.CEPHBLOCKPOOL,
encrypted=True,
encryption_kms_id=self.kms.kmsid,
)
if kms_provider == constants.VAULT_KMS_PROVIDER:
# Create ceph-csi-kms-token in the tenant namespace
self.kms.vault_path_token = self.kms.generate_vault_token()
self.kms.create_vault_csi_kms_token(
namespace=self.proj_obj.namespace)
# Create PVC and Pods
self.pvc_size = 1
self.pvc_objs = multi_pvc_factory(
interface=constants.CEPHBLOCKPOOL,
project=self.proj_obj,
storageclass=self.sc_obj,
size=self.pvc_size,
access_modes=[
f"{constants.ACCESS_MODE_RWX}-Block",
f"{constants.ACCESS_MODE_RWO}-Block",
],
status=constants.STATUS_BOUND,
num_of_pvc=2,
wait_each=False,
)
self.pod_objs = helpers.create_pods(
self.pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status=constants.STATUS_RUNNING,
)
# Verify if the key is created in Vault
self.vol_handles = []
for pvc_obj in self.pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
self.vol_handles.append(vol_handle)
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(f"Vault: Found key for {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for {pvc_obj.name}")
def test_pvc_to_pvc_clone(self, kv_version, kms_provider, pod_factory):
"""
Test to create a clone from an existing encrypted RBD PVC.
Verify that the cloned PVC is encrypted and all the data is preserved.
"""
log.info("Checking for encrypted device and running IO on all pods")
for vol_handle, pod_obj in zip(self.vol_handles, self.pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
log.info(f"File created during IO {pod_obj.name}")
pod_obj.run_io(
storage_type="block",
size="500M",
io_direction="write",
runtime=60,
end_fsync=1,
direct=1,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in self.pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
cloned_pvc_objs, cloned_vol_handles = ([] for i in range(2))
# Calculate the md5sum value and create clones of exisiting PVCs
log.info("Calculate the md5sum after IO and create clone of all PVCs")
for pod_obj in self.pod_objs:
pod_obj.md5sum_after_io = pod.cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
cloned_pvc_obj = pvc.create_pvc_clone(
self.sc_obj.name,
pod_obj.pvc.name,
constants.CSI_RBD_PVC_CLONE_YAML,
self.proj_obj.namespace,
volume_mode=constants.VOLUME_MODE_BLOCK,
access_mode=pod_obj.pvc.access_mode,
)
helpers.wait_for_resource_state(cloned_pvc_obj,
constants.STATUS_BOUND)
cloned_pvc_obj.reload()
cloned_pvc_obj.md5sum = pod_obj.md5sum_after_io
cloned_pvc_objs.append(cloned_pvc_obj)
log.info("Clone of all PVCs created")
# Create and attach pod to the pvc
cloned_pod_objs = helpers.create_pods(
cloned_pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status="",
)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in cloned_pod_objs:
helpers.wait_for_resource_state(pod_obj, constants.STATUS_RUNNING)
pod_obj.reload()
log.info("Verified: New pods are running")
# Verify encryption keys are created for cloned PVCs in Vault
for pvc_obj in cloned_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
cloned_vol_handles.append(vol_handle)
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(
f"Vault: Found key for restore PVC {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for restored PVC {pvc_obj.name}"
)
# Verify encrypted device is present and md5sum on all pods
for vol_handle, pod_obj in zip(cloned_vol_handles, cloned_pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
log.info(f"Verifying md5sum on pod {pod_obj.name}")
pod.verify_data_integrity(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
original_md5sum=pod_obj.pvc.md5sum,
block=True,
)
log.info(f"Verified md5sum on pod {pod_obj.name}")
# Run IO on new pods
log.info("Starting IO on new pods")
for pod_obj in cloned_pod_objs:
pod_obj.run_io(storage_type="block", size="100M", runtime=10)
# Wait for IO completion on new pods
log.info("Waiting for IO completion on new pods")
for pod_obj in cloned_pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on new pods.")
# Delete the restored pods, PVC and snapshots
log.info("Deleting all pods")
for pod_obj in cloned_pod_objs + self.pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleting all PVCs")
for pvc_obj in cloned_pvc_objs + self.pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
if kms_provider == constants.VAULT_KMS_PROVIDER:
# Verify if the keys for parent and cloned PVCs are deleted from Vault
if kv_version == "v1" or Version.coerce(
config.ENV_DATA["ocs_version"]) >= Version.coerce("4.9"):
log.info(
"Verify whether the keys for cloned PVCs are deleted from vault"
)
for key in cloned_vol_handles + self.vol_handles:
if not kms.is_key_present_in_path(
key=key, path=self.kms.vault_backend_path):
log.info(f"Vault: Key deleted for {key}")
else:
raise KMSResourceCleaneupError(
f"Vault: Key deletion failed for {key}")
log.info("All keys from vault were deleted")
def test_encrypted_rbd_block_pvc_snapshot(
self,
kms_provider,
snapshot_factory,
snapshot_restore_factory,
pod_factory,
kv_version,
):
"""
Test to take snapshots of encrypted RBD Block VolumeMode PVCs
"""
log.info(
"Check for encrypted device, find initial md5sum value and run IO on all pods"
)
for vol_handle, pod_obj in zip(self.vol_handles, self.pod_objs):
# Verify whether encrypted device is present inside the pod
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
# Find initial md5sum
pod_obj.md5sum_before_io = cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
pod_obj.run_io(
storage_type="block",
size=f"{self.pvc_size - 1}G",
io_direction="write",
runtime=60,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in self.pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
snap_objs, snap_handles = ([] for i in range(2))
# Verify md5sum has changed after IO. Create snapshot
log.info(
"Verify md5sum has changed after IO and create snapshot from all PVCs"
)
for pod_obj in self.pod_objs:
md5sum_after_io = cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
assert (pod_obj.md5sum_before_io != md5sum_after_io
), f"md5sum has not changed after IO on pod {pod_obj.name}"
log.info(f"Creating snapshot of PVC {pod_obj.pvc.name}")
snap_obj = snapshot_factory(pod_obj.pvc, wait=False)
snap_obj.md5sum = md5sum_after_io
snap_objs.append(snap_obj)
log.info("Snapshots created")
# Verify snapshots are ready and verify if encryption key is created in vault
log.info("Verify snapshots are ready")
for snap_obj in snap_objs:
snap_obj.ocp.wait_for_resource(
condition="true",
resource_name=snap_obj.name,
column=constants.STATUS_READYTOUSE,
timeout=180,
)
snapshot_content = get_snapshot_content_obj(snap_obj=snap_obj)
snap_handle = snapshot_content.get().get("status").get(
"snapshotHandle")
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=snap_handle, path=self.kms.vault_backend_path):
log.info(f"Vault: Found key for snapshot {snap_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for snapshot {snap_obj.name}")
snap_handles.append(snap_handle)
# Delete pods
log.info("Deleting the pods")
for pod_obj in self.pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleted all the pods")
# Delete parent PVCs to verify snapshot is independent
log.info("Deleting parent PVCs")
for pvc_obj in self.pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
log.info(f"Deleted PVC {pvc_obj.name}. Verifying whether PV "
f"{pv_obj.name} is deleted.")
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info(
"All parent PVCs and PVs are deleted before restoring snapshot.")
restore_pvc_objs, restore_vol_handles = ([] for i in range(2))
# Create PVCs out of the snapshots
log.info("Creating new PVCs from snapshots")
for snap_obj in snap_objs:
log.info(f"Creating a PVC from snapshot {snap_obj.name}")
restore_pvc_obj = snapshot_restore_factory(
snapshot_obj=snap_obj,
storageclass=self.sc_obj.name,
size=f"{self.pvc_size}Gi",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
status="",
)
log.info(f"Created PVC {restore_pvc_obj.name} from snapshot "
f"{snap_obj.name}")
restore_pvc_obj.md5sum = snap_obj.md5sum
restore_pvc_objs.append(restore_pvc_obj)
log.info("Created new PVCs from all the snapshots")
# Confirm that the restored PVCs are Bound
log.info("Verify the restored PVCs are Bound")
for pvc_obj in restore_pvc_objs:
wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=180)
pvc_obj.reload()
log.info("Verified: Restored PVCs are Bound.")
# Attach the restored PVCs to pods. Attach RWX PVC on two pods
log.info("Attach the restored PVCs to pods")
restore_pod_objs = create_pods(
restore_pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status="",
)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in restore_pod_objs:
timeout = (300 if config.ENV_DATA["platform"]
== constants.IBMCLOUD_PLATFORM else 60)
wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout)
log.info("Verified: New pods are running")
# Verify encryption keys are created for restored PVCs in Vault
for pvc_obj in restore_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
restore_vol_handles.append(vol_handle)
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(
f"Vault: Found key for restore PVC {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for restored PVC {pvc_obj.name}"
)
# Verify encrypted device is present and md5sum on all pods
for vol_handle, pod_obj in zip(restore_vol_handles, restore_pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
log.info(f"Verifying md5sum on pod {pod_obj.name}")
verify_data_integrity(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
original_md5sum=pod_obj.pvc.md5sum,
block=True,
)
log.info(f"Verified md5sum on pod {pod_obj.name}")
# Run IO on new pods
log.info("Starting IO on new pods")
for pod_obj in restore_pod_objs:
pod_obj.run_io(storage_type="block", size="500M", runtime=15)
# Wait for IO completion on new pods
log.info("Waiting for IO completion on new pods")
for pod_obj in restore_pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on new pods.")
# Delete the restored pods, PVC and snapshots
log.info("Deleting pods using restored PVCs")
for pod_obj in restore_pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleting restored PVCs")
for pvc_obj in restore_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info("Deleting the snapshots")
for snap_obj in snap_objs:
snapcontent_obj = get_snapshot_content_obj(snap_obj=snap_obj)
snap_obj.delete()
snapcontent_obj.ocp.wait_for_delete(
resource_name=snapcontent_obj.name)
if kms_provider == constants.VAULT_KMS_PROVIDER:
# Verify if keys for PVCs and snapshots are deleted from Vault
if kv_version == "v1" or Version.coerce(
config.ENV_DATA["ocs_version"]) >= Version.coerce("4.9"):
log.info(
"Verify whether the keys for PVCs and snapshots are deleted in vault"
)
for key in self.vol_handles + snap_handles + restore_vol_handles:
if not kms.is_key_present_in_path(
key=key, path=self.kms.vault_backend_path):
log.info(f"Vault: Key deleted for {key}")
else:
raise KMSResourceCleaneupError(
f"Vault: Key deletion failed for {key}")
log.info("All keys from vault were deleted")
def test_encrypted_pvc_clone(
self,
kv_version,
pv_encryption_kms_setup_factory,
storageclass_factory,
pgsql_factory_fixture,
pvc_clone_factory,
pgsql_teardown,
):
"""
1. Create encrypted storage class
2. Deploy PGSQL workload using created sc
3. Create multiple clone of same PVC when the PVC usage is different
4. Attach a new pgsql pod to it.
5. Create pgbench benchmark to new pgsql pod
6. Verify if key is created for cloned pvc
"""
pgsql_teardown
log.info("Setting up csi-kms-connection-details configmap")
self.vault = pv_encryption_kms_setup_factory(kv_version)
log.info("csi-kms-connection-details setup successful")
# Create an encryption enabled storageclass for RBD
self.sc_obj = storageclass_factory(
interface=CEPHBLOCKPOOL,
encrypted=True,
encryption_kms_id=self.vault.kmsid,
)
# Create ceph-csi-kms-token in the tenant namespace
self.vault.vault_path_token = self.vault.generate_vault_token()
self.vault.create_vault_csi_kms_token(namespace=BMO_NAME)
# Deploy PGSQL workload
log.info("Deploying pgsql workloads")
pgsql = pgsql_factory_fixture(replicas=1, sc_name=self.sc_obj.name)
# Get postgres pvcs obj list
postgres_pvcs_obj = pgsql.get_postgres_pvc()
# Get postgres pods obj list
postgres_pods_obj = pgsql.get_postgres_pods()
cloned_pvcs = self.create_cloned_pvc_and_verify_data(
pgsql,
postgres_pvcs_obj,
postgres_pods_obj,
pvc_clone_factory,
self.sc_obj.name,
)
# Verify encryption keys are created for cloned PVCs in Vault
for pvc_obj in cloned_pvcs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
if kms.is_key_present_in_path(key=vol_handle,
path=self.vault.vault_backend_path):
log.info(f"Vault: Found key for restore PVC {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for restored PVC {pvc_obj.name}")
def test_encrypted_pvc_snapshot(
self,
kv_version,
pv_encryption_kms_setup_factory,
storageclass_factory,
pgsql_factory_fixture,
snapshot_factory,
snapshot_restore_factory,
pgsql_teardown,
):
"""
1. Create encrypted storage class
2. Deploy PGSQL workload using created sc
3. Take a snapshot of the pgsql PVC.
4. Create a new PVC out of that snapshot or restore snapshot
5. Attach a new pgsql pod to it.
6. Create pgbench benchmark to new pgsql pod
7. Verify if key is created
"""
pgsql_teardown
log.info("Setting up csi-kms-connection-details configmap")
self.vault = pv_encryption_kms_setup_factory(kv_version)
log.info("csi-kms-connection-details setup successful")
# Create an encryption enabled storageclass for RBD
self.sc_obj = storageclass_factory(
interface=CEPHBLOCKPOOL,
encrypted=True,
encryption_kms_id=self.vault.kmsid,
)
# Create ceph-csi-kms-token in the tenant namespace
self.vault.vault_path_token = self.vault.generate_vault_token()
self.vault.create_vault_csi_kms_token(namespace=BMO_NAME)
# Deploy PGSQL workload
log.info("Deploying pgsql workloads")
pgsql = pgsql_factory_fixture(replicas=1, sc_name=self.sc_obj.name)
# Get postgres pvc list obj
postgres_pvcs_obj = pgsql.get_postgres_pvc()
# Take a snapshot of it
snapshots, restore_pvc_objs = self.create_snapshot(
pgsql,
postgres_pvcs_obj,
snapshot_factory,
snapshot_restore_factory,
self.sc_obj.name,
)
# Verify encryption keys are created for snapshots in Vault
for snap_obj in snapshots:
snapshot_content = get_snapshot_content_obj(snap_obj=snap_obj)
snap_handle = snapshot_content.get().get("status").get(
"snapshotHandle")
if kms.is_key_present_in_path(key=snap_handle,
path=self.vault.vault_backend_path):
log.info(f"Vault: Found key for snapshot {snap_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for snapshot {snap_obj.name}")
# Verify encryption keys are created for restored PVCs in Vault
for pvc_obj in restore_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
if kms.is_key_present_in_path(key=vol_handle,
path=self.vault.vault_backend_path):
log.info(f"Vault: Found key for restore PVC {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for restored PVC {pvc_obj.name}")