def remove_vault_policy(self):
"""
Cleanup the policy we used
"""
cmd = f"vault policy delete {self.vault_policy_name} "
subprocess.check_output(shlex.split(cmd))
# Check if policy still exists
cmd = "vault policy list --format=json"
out = subprocess.check_output(shlex.split(cmd))
json_out = json.loads(out)
if self.vault_policy_name in json_out:
raise KMSResourceCleaneupError(
f"Policy {self.vault_policy_name} not deleted")
logger.info(f"Vault policy {self.vault_policy_name} deleted")
def remove_vault_backend_path(self):
"""
remove vault path
"""
cmd = f"vault secrets disable {self.vault_backend_path}"
subprocess.check_output(shlex.split(cmd))
# Check if path doesn't appear in the list
cmd = "vault secrets list --format=json"
out = subprocess.check_output(shlex.split(cmd))
json_out = json.loads(out)
for path in json_out.keys():
if self.vault_backend_path in path:
raise KMSResourceCleaneupError(
f"Path {self.vault_backend_path} not deleted")
logger.info(f"Vault path {self.vault_backend_path} deleted")
def remove_vault_namespace(self):
"""
Cleanup the namespace
Raises:
KMSResourceCleanupError: If namespace deletion fails
"""
# Unset namespace from environment
# else delete will look for namespace within namespace
os.environ.pop("VAULT_NAMESPACE")
cmd = f"vault namespace delete {self.vault_namespace}/"
subprocess.check_output(shlex.split(cmd))
if self.vault_namespace_exists(self.vault_namespace):
raise KMSResourceCleaneupError(
f"Namespace {self.vault_namespace} deletion failed")
logger.info(f"Vault namespace {self.vault_namespace} deleted")
def remove_kmsid(kmsid):
"""
This function will remove all the details for the given kmsid from the csi-kms-connection-details configmap
Args:
kmsid (str) : kmsid to be remove_kmsid
Raises:
KMSResourceCleaneupError: If the kmsid entry is not deleted
"""
ocp_obj = ocp.OCP()
patch = f'\'[{{"op": "remove", "path": "/data/{kmsid}"}}]\''
patch_cmd = (
f"patch -n {constants.OPENSHIFT_STORAGE_NAMESPACE} cm "
f"{constants.VAULT_KMS_CSI_CONNECTION_DETAILS} --type json -p " + patch
)
ocp_obj.exec_oc_cmd(command=patch_cmd)
kmsid_list = get_encryption_kmsid()
if kmsid in kmsid_list:
raise KMSResourceCleaneupError(f"KMS ID {kmsid} deletion failed")
logger.info(f"KMS ID {kmsid} deleted")
def test_rbd_pv_encryption_vaulttenantsa(
self,
project_factory,
storageclass_factory,
multi_pvc_factory,
pod_factory,
kv_version,
):
"""
Test to verify creation and deletion of encrypted RBD PVC using vaulttenantsa method
"""
# Create a project
proj_obj = project_factory()
# Create an encryption enabled storageclass for RBD
sc_obj = storageclass_factory(
interface=constants.CEPHBLOCKPOOL,
encrypted=True,
encryption_kms_id=self.kms.kmsid,
)
# Create serviceaccount in the tenant namespace
self.kms.create_tenant_sa(namespace=proj_obj.namespace)
# Create role in Vault
self.kms.create_vault_kube_auth_role(namespace=proj_obj.namespace)
# Create RBD PVCs with volume mode Block
pvc_size = 5
pvc_objs = multi_pvc_factory(
interface=constants.CEPHBLOCKPOOL,
project=proj_obj,
storageclass=sc_obj,
size=pvc_size,
access_modes=[
f"{constants.ACCESS_MODE_RWX}-Block",
f"{constants.ACCESS_MODE_RWO}-Block",
],
status=constants.STATUS_BOUND,
num_of_pvc=3,
wait_each=False,
)
# Create pods
pod_objs = create_pods(
pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status=constants.STATUS_RUNNING,
)
# Verify if the key is created in Vault
vol_handles = []
for pvc_obj in pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
vol_handles.append(vol_handle)
# Check if encryption key is created in Vault
if kms.is_key_present_in_path(key=vol_handle,
path=self.kms.vault_backend_path):
log.info(f"Vault: Found key for {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for {pvc_obj.name}")
# Verify whether encrypted device is present inside the pod and run IO
for vol_handle, pod_obj in zip(vol_handles, pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
pod_obj.run_io(
storage_type="block",
size=f"{pvc_size - 1}G",
io_direction="write",
runtime=60,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
# Delete the pod
for pod_obj in pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
# Delete the PVC
for pvc_obj in pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
# Verify whether the key is deleted in Vault
for vol_handle in vol_handles:
if not kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(f"Vault: Key deleted for {vol_handle}")
else:
raise KMSResourceCleaneupError(
f"Vault: Key deletion failed for {vol_handle}")
def test_pvc_to_pvc_clone(self, kv_version, kms_provider, pod_factory):
"""
Test to create a clone from an existing encrypted RBD PVC.
Verify that the cloned PVC is encrypted and all the data is preserved.
"""
log.info("Checking for encrypted device and running IO on all pods")
for vol_handle, pod_obj in zip(self.vol_handles, self.pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
log.info(f"File created during IO {pod_obj.name}")
pod_obj.run_io(
storage_type="block",
size="500M",
io_direction="write",
runtime=60,
end_fsync=1,
direct=1,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in self.pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
cloned_pvc_objs, cloned_vol_handles = ([] for i in range(2))
# Calculate the md5sum value and create clones of exisiting PVCs
log.info("Calculate the md5sum after IO and create clone of all PVCs")
for pod_obj in self.pod_objs:
pod_obj.md5sum_after_io = pod.cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
cloned_pvc_obj = pvc.create_pvc_clone(
self.sc_obj.name,
pod_obj.pvc.name,
constants.CSI_RBD_PVC_CLONE_YAML,
self.proj_obj.namespace,
volume_mode=constants.VOLUME_MODE_BLOCK,
access_mode=pod_obj.pvc.access_mode,
)
helpers.wait_for_resource_state(cloned_pvc_obj,
constants.STATUS_BOUND)
cloned_pvc_obj.reload()
cloned_pvc_obj.md5sum = pod_obj.md5sum_after_io
cloned_pvc_objs.append(cloned_pvc_obj)
log.info("Clone of all PVCs created")
# Create and attach pod to the pvc
cloned_pod_objs = helpers.create_pods(
cloned_pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status="",
)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in cloned_pod_objs:
helpers.wait_for_resource_state(pod_obj, constants.STATUS_RUNNING)
pod_obj.reload()
log.info("Verified: New pods are running")
# Verify encryption keys are created for cloned PVCs in Vault
for pvc_obj in cloned_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
cloned_vol_handles.append(vol_handle)
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(
f"Vault: Found key for restore PVC {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for restored PVC {pvc_obj.name}"
)
# Verify encrypted device is present and md5sum on all pods
for vol_handle, pod_obj in zip(cloned_vol_handles, cloned_pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
log.info(f"Verifying md5sum on pod {pod_obj.name}")
pod.verify_data_integrity(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
original_md5sum=pod_obj.pvc.md5sum,
block=True,
)
log.info(f"Verified md5sum on pod {pod_obj.name}")
# Run IO on new pods
log.info("Starting IO on new pods")
for pod_obj in cloned_pod_objs:
pod_obj.run_io(storage_type="block", size="100M", runtime=10)
# Wait for IO completion on new pods
log.info("Waiting for IO completion on new pods")
for pod_obj in cloned_pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on new pods.")
# Delete the restored pods, PVC and snapshots
log.info("Deleting all pods")
for pod_obj in cloned_pod_objs + self.pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleting all PVCs")
for pvc_obj in cloned_pvc_objs + self.pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
if kms_provider == constants.VAULT_KMS_PROVIDER:
# Verify if the keys for parent and cloned PVCs are deleted from Vault
if kv_version == "v1" or Version.coerce(
config.ENV_DATA["ocs_version"]) >= Version.coerce("4.9"):
log.info(
"Verify whether the keys for cloned PVCs are deleted from vault"
)
for key in cloned_vol_handles + self.vol_handles:
if not kms.is_key_present_in_path(
key=key, path=self.kms.vault_backend_path):
log.info(f"Vault: Key deleted for {key}")
else:
raise KMSResourceCleaneupError(
f"Vault: Key deletion failed for {key}")
log.info("All keys from vault were deleted")
def test_encrypted_rbd_block_pvc_snapshot(
self,
kms_provider,
snapshot_factory,
snapshot_restore_factory,
pod_factory,
kv_version,
):
"""
Test to take snapshots of encrypted RBD Block VolumeMode PVCs
"""
log.info(
"Check for encrypted device, find initial md5sum value and run IO on all pods"
)
for vol_handle, pod_obj in zip(self.vol_handles, self.pod_objs):
# Verify whether encrypted device is present inside the pod
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
# Find initial md5sum
pod_obj.md5sum_before_io = cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
pod_obj.run_io(
storage_type="block",
size=f"{self.pvc_size - 1}G",
io_direction="write",
runtime=60,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in self.pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
snap_objs, snap_handles = ([] for i in range(2))
# Verify md5sum has changed after IO. Create snapshot
log.info(
"Verify md5sum has changed after IO and create snapshot from all PVCs"
)
for pod_obj in self.pod_objs:
md5sum_after_io = cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
assert (pod_obj.md5sum_before_io != md5sum_after_io
), f"md5sum has not changed after IO on pod {pod_obj.name}"
log.info(f"Creating snapshot of PVC {pod_obj.pvc.name}")
snap_obj = snapshot_factory(pod_obj.pvc, wait=False)
snap_obj.md5sum = md5sum_after_io
snap_objs.append(snap_obj)
log.info("Snapshots created")
# Verify snapshots are ready and verify if encryption key is created in vault
log.info("Verify snapshots are ready")
for snap_obj in snap_objs:
snap_obj.ocp.wait_for_resource(
condition="true",
resource_name=snap_obj.name,
column=constants.STATUS_READYTOUSE,
timeout=180,
)
snapshot_content = get_snapshot_content_obj(snap_obj=snap_obj)
snap_handle = snapshot_content.get().get("status").get(
"snapshotHandle")
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=snap_handle, path=self.kms.vault_backend_path):
log.info(f"Vault: Found key for snapshot {snap_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for snapshot {snap_obj.name}")
snap_handles.append(snap_handle)
# Delete pods
log.info("Deleting the pods")
for pod_obj in self.pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleted all the pods")
# Delete parent PVCs to verify snapshot is independent
log.info("Deleting parent PVCs")
for pvc_obj in self.pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
log.info(f"Deleted PVC {pvc_obj.name}. Verifying whether PV "
f"{pv_obj.name} is deleted.")
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info(
"All parent PVCs and PVs are deleted before restoring snapshot.")
restore_pvc_objs, restore_vol_handles = ([] for i in range(2))
# Create PVCs out of the snapshots
log.info("Creating new PVCs from snapshots")
for snap_obj in snap_objs:
log.info(f"Creating a PVC from snapshot {snap_obj.name}")
restore_pvc_obj = snapshot_restore_factory(
snapshot_obj=snap_obj,
storageclass=self.sc_obj.name,
size=f"{self.pvc_size}Gi",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
status="",
)
log.info(f"Created PVC {restore_pvc_obj.name} from snapshot "
f"{snap_obj.name}")
restore_pvc_obj.md5sum = snap_obj.md5sum
restore_pvc_objs.append(restore_pvc_obj)
log.info("Created new PVCs from all the snapshots")
# Confirm that the restored PVCs are Bound
log.info("Verify the restored PVCs are Bound")
for pvc_obj in restore_pvc_objs:
wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=180)
pvc_obj.reload()
log.info("Verified: Restored PVCs are Bound.")
# Attach the restored PVCs to pods. Attach RWX PVC on two pods
log.info("Attach the restored PVCs to pods")
restore_pod_objs = create_pods(
restore_pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status="",
)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in restore_pod_objs:
timeout = (300 if config.ENV_DATA["platform"]
== constants.IBMCLOUD_PLATFORM else 60)
wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout)
log.info("Verified: New pods are running")
# Verify encryption keys are created for restored PVCs in Vault
for pvc_obj in restore_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
restore_vol_handles.append(vol_handle)
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(
f"Vault: Found key for restore PVC {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for restored PVC {pvc_obj.name}"
)
# Verify encrypted device is present and md5sum on all pods
for vol_handle, pod_obj in zip(restore_vol_handles, restore_pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
log.info(f"Verifying md5sum on pod {pod_obj.name}")
verify_data_integrity(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
original_md5sum=pod_obj.pvc.md5sum,
block=True,
)
log.info(f"Verified md5sum on pod {pod_obj.name}")
# Run IO on new pods
log.info("Starting IO on new pods")
for pod_obj in restore_pod_objs:
pod_obj.run_io(storage_type="block", size="500M", runtime=15)
# Wait for IO completion on new pods
log.info("Waiting for IO completion on new pods")
for pod_obj in restore_pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on new pods.")
# Delete the restored pods, PVC and snapshots
log.info("Deleting pods using restored PVCs")
for pod_obj in restore_pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleting restored PVCs")
for pvc_obj in restore_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info("Deleting the snapshots")
for snap_obj in snap_objs:
snapcontent_obj = get_snapshot_content_obj(snap_obj=snap_obj)
snap_obj.delete()
snapcontent_obj.ocp.wait_for_delete(
resource_name=snapcontent_obj.name)
if kms_provider == constants.VAULT_KMS_PROVIDER:
# Verify if keys for PVCs and snapshots are deleted from Vault
if kv_version == "v1" or Version.coerce(
config.ENV_DATA["ocs_version"]) >= Version.coerce("4.9"):
log.info(
"Verify whether the keys for PVCs and snapshots are deleted in vault"
)
for key in self.vol_handles + snap_handles + restore_vol_handles:
if not kms.is_key_present_in_path(
key=key, path=self.kms.vault_backend_path):
log.info(f"Vault: Key deleted for {key}")
else:
raise KMSResourceCleaneupError(
f"Vault: Key deletion failed for {key}")
log.info("All keys from vault were deleted")