def test_monitoring_delete_pvc(self):
"""
Test case to validate whether delete pvcs+configmap and recovery of a
node where monitoring pods running has no functional impact
"""
# Get 'cluster-monitoring-config' configmap
ocp_configmap = ocp.OCP(
namespace=constants.MONITORING_NAMESPACE, kind="configmap"
)
configmap_dict = ocp_configmap.get(resource_name="cluster-monitoring-config")
dir_configmap = tempfile.mkdtemp(prefix="configmap_")
yaml_file = f"{dir_configmap}/configmap.yaml"
templating.dump_data_to_temp_yaml(configmap_dict, yaml_file)
# Get prometheus and alertmanager pods
prometheus_alertmanager_pods = pod.get_all_pods(
namespace=defaults.OCS_MONITORING_NAMESPACE,
selector=["prometheus", "alertmanager"],
)
# Get all pvc on monitoring namespace
pvc_objs_list = pvc.get_all_pvc_objs(namespace=constants.MONITORING_NAMESPACE)
# Delete configmap
ocp_configmap.delete(resource_name="cluster-monitoring-config")
# Delete all pvcs on monitoring namespace
pvc.delete_pvcs(pvc_objs=pvc_objs_list)
# Check all the prometheus and alertmanager pods are up
for pod_obj in prometheus_alertmanager_pods:
wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING, timeout=180
)
# Create configmap
ocp_configmap.create(yaml_file=dir_configmap)
# Check all the PVCs are up
for pvc_obj in pvc_objs_list:
wait_for_resource_state(
resource=pvc_obj, state=constants.STATUS_BOUND, timeout=180
)
# Check all the prometheus and alertmanager pods are up
# and pvc are mounted on monitoring pods
for pod_obj in prometheus_alertmanager_pods:
wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING, timeout=180
)
mount_point = pod_obj.exec_cmd_on_pod(
command="df -kh",
out_yaml_format=False,
)
assert "/dev/rbd" in mount_point, f"pvc is not mounted on pod {pod.name}"
log.info("Verified all pvc are mounted on monitoring pods")
# Validate the prometheus health is ok
assert prometheus_health_check(), "Prometheus cluster health is not OK"
def finalizer():
# Delete pods
for pod_obj in cls_ref.pod_objs:
pod_obj.delete()
# Delete newly created PVCs
delete_pvcs(cls_ref.pvc_objs_new)
def create_pvc_delete(self, multi_pvc_factory, project=None):
"""
Creates and deletes all types of PVCs
"""
# Create rbd pvcs
pvc_objs_rbd = create_pvcs(multi_pvc_factory=multi_pvc_factory,
interface='CephBlockPool',
project=project,
status="",
storageclass=None)
# Create cephfs pvcs
pvc_objs_cephfs = create_pvcs(multi_pvc_factory=multi_pvc_factory,
interface='CephFileSystem',
project=project,
status="",
storageclass=None)
all_pvc_to_delete = pvc_objs_rbd + pvc_objs_cephfs
# Check pvc status
for pvc_obj in all_pvc_to_delete:
helpers.wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=300)
# Start deleting PVC
delete_pvcs(all_pvc_to_delete)
# Check PVCs are deleted
for pvc_obj in all_pvc_to_delete:
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
logger.info("All PVCs are deleted as expected")
def cleanup(
self,
kafka_namespace=constants.AMQ_NAMESPACE,
tiller_namespace=AMQ_BENCHMARK_NAMESPACE,
):
"""
Clean up function,
will start to delete from amq cluster operator
then amq-connector, persistent, bridge, at the end it will delete the created namespace
Args:
kafka_namespace (str): Created namespace for amq
tiller_namespace (str): Created namespace for benchmark
"""
ocs_pvc_obj = get_all_pvc_objs(namespace=kafka_namespace)
if self.amq_is_setup:
if self.messaging:
self.consumer_pod.delete()
self.producer_pod.delete()
self.kafka_user.delete()
self.kafka_topic.delete()
if self.benchmark:
# Delete the helm app
try:
purge_cmd = f"linux-amd64/helm delete benchmark --purge --tiller-namespace {tiller_namespace}"
run(purge_cmd, shell=True, cwd=self.dir, check=True)
except (CommandFailed, CalledProcessError) as cf:
log.error("Failed to delete help app")
raise cf
# Delete the pods and namespace created
self.sa_tiller.delete()
self.crb_tiller.delete()
run_cmd(f"oc delete project {tiller_namespace}")
self.ns_obj.wait_for_delete(resource_name=tiller_namespace)
self.kafka_connect.delete()
self.kafka_bridge.delete()
self.kafka_persistent.delete()
delete_pvcs(pvc_objs=ocs_pvc_obj)
cmd = f"oc delete -f {self.strimzi_kafka_operator}"
log.info(f"Executing cmd: {cmd}")
run(cmd, shell=True, check=True, cwd=self.dir)
for pvc in ocs_pvc_obj:
logging.info(pvc.name)
validate_pv_delete(pvc.backed_pv)
run_cmd(f"oc delete project {kafka_namespace}")
self.ns_obj.wait_for_delete(resource_name=kafka_namespace, timeout=90)
# Reset namespace to default
switch_to_default_rook_cluster_project()
def test_bulk_pvc_creation_after_deletion_performance(
self, teardown_factory):
"""
Measuring PVC creation time of bulk of 75% of initial PVC bulk (120) in the same
rate after deleting ( serial deletion) 75% of the initial PVCs.
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
Returns:
"""
initial_number_of_pvcs = 120
number_of_pvcs = math.ceil(initial_number_of_pvcs * 0.75)
log.info(f"Start creating new {initial_number_of_pvcs} PVCs in a bulk")
pvc_objs, _ = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
number_of_pvc=initial_number_of_pvcs,
size=self.pvc_size,
burst=True,
)
for pvc_obj in pvc_objs:
teardown_factory(pvc_obj)
with ThreadPoolExecutor() as executor:
for pvc_obj in pvc_objs:
executor.submit(helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND)
executor.submit(pvc_obj.reload)
log.info("Deleting 75% of the PVCs - 90 PVCs")
assert pvc.delete_pvcs(pvc_objs[:number_of_pvcs],
True), "Deletion of 75% of PVCs failed"
log.info("Re-creating the 90 PVCs")
pvc_objs, _ = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
number_of_pvc=number_of_pvcs,
size=self.pvc_size,
burst=True,
)
start_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="end")
total = end_time - start_time
total_time = total.total_seconds()
logging.info(
f"Deletion time of {number_of_pvcs} is {total_time} seconds.")
if total_time > 50:
raise ex.PerformanceException(
f"{number_of_pvcs} PVCs creation (after initial deletion of "
f"75% of PVCs) time is {total_time} and greater than 50 seconds."
)
logging.info(
f"{number_of_pvcs} PVCs creation time took less than a 50 seconds")
def teardown(self):
"""
Delete pvc and config map created
"""
assert ocp.delete(resource_name='cluster-monitoring-config')
pvc_obj_list = get_list_pvc_objs_created_on_monitoring_pods()
assert delete_pvcs(pvc_obj_list)
def destroy_ocs(self):
"""
Uninstall ODF Managed Service addon via rosa cli.
"""
cluster_namespace = config.ENV_DATA["cluster_namespace"]
# Deleting PVCs
rbd_pvcs = [
p for p in pvc.get_all_pvcs_in_storageclass(
constants.CEPHBLOCKPOOL_SC)
if not (p.data["metadata"]["namespace"] == cluster_namespace
and p.data["metadata"]["labels"]["app"] == "noobaa")
]
pvc.delete_pvcs(rbd_pvcs)
cephfs_pvcs = pvc.get_all_pvcs_in_storageclass(
constants.CEPHFILESYSTEM_SC)
pvc.delete_pvcs(cephfs_pvcs)
rosa.delete_odf_addon(self.cluster_name)
def test_multiple_pvc_creation_after_deletion_performance(
self, teardown_factory
):
"""
Measuring PVC creation time of 75% of initial PVCs (120) in the same
rate after deleting 75% of the initial PVCs
"""
initial_number_of_pvcs = 120
number_of_pvcs = math.ceil(initial_number_of_pvcs * 0.75)
log.info('Start creating new 120 PVCs')
pvc_objs = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
number_of_pvc=initial_number_of_pvcs,
size=self.pvc_size,
)
for pvc_obj in pvc_objs:
teardown_factory(pvc_obj)
with ThreadPoolExecutor() as executor:
for pvc_obj in pvc_objs:
executor.submit(
helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND
)
executor.submit(pvc_obj.reload)
log.info('Deleting 75% of the PVCs - 90 PVCs')
assert pvc.delete_pvcs(pvc_objs[:number_of_pvcs], True), (
"Deletion of 75% of PVCs failed"
)
log.info('Re-creating the 90 PVCs')
pvc_objs = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
number_of_pvc=number_of_pvcs,
size=self.pvc_size,
)
start_time = helpers.get_start_creation_time(
self.interface, pvc_objs[0].name
)
end_time = helpers.get_end_creation_time(
self.interface, pvc_objs[number_of_pvcs - 1].name,
)
total = end_time - start_time
total_time = total.total_seconds()
if total_time > 45:
raise ex.PerformanceException(
f"{number_of_pvcs} PVCs creation (after initial deletion of "
f"75%) time is {total_time} and greater than 45 seconds"
)
logging.info(
f"{number_of_pvcs} PVCs creation time took less than a 45 seconds"
)
def teardown():
# Delete created app pods and pvcs
assert pod.delete_pods(pod_objs)
assert pvc.delete_pvcs(pvc_objs)
# Switch to default project
ret = ocp.switch_to_default_rook_cluster_project()
assert ret, 'Failed to switch to default rook cluster project'
# Delete created projects
for prj in namespace_list:
prj.delete(resource_name=prj.namespace)
def teardown(self):
"""
Delete PVCs
Delete project
"""
# Delete newly created PVCs
assert delete_pvcs(self.pvc_objs_new), 'Failed to delete PVCs'
log.info(f'Newly created {self.number_of_pvc} PVCs are now deleted.')
# Switch to default project
ret = ocp.switch_to_default_rook_cluster_project()
assert ret, 'Failed to switch to default rook cluster project'
# Delete project created for the test case
self.project_obj.delete(resource_name=self.namespace)
def teardown():
"""
Tearing down the environment
"""
global RBD_SECRET_OBJ, CEPHFS_SECRET_OBJ
log.info("Deleting PVC")
assert pvc.delete_pvcs(PVC_OBJS)
log.info("Deleting CEPH BLOCK POOL")
assert helpers.delete_cephblockpools(POOL_OBJS)
log.info("Deleting RBD Secret")
RBD_SECRET_OBJ.delete()
log.info("Deleting CEPHFS Secret")
CEPHFS_SECRET_OBJ.delete()
log.info("Deleting RBD Storageclass")
assert helpers.delete_storageclasses(SC_RBD_OBJS)
log.info("Deleting CephFS Storageclass")
assert helpers.delete_storageclasses([SC_CEPHFS_OBJ])
def teardown():
"""
Tearing down the environment
"""
global RBD_SECRET_OBJ, CEPHFS_SECRET_OBJ
log.info("Deleting PVC")
assert pvc.delete_pvcs(PVC_OBJS)
log.info("Deleting CEPH BLOCK POOL")
assert helpers.delete_cephblockpool()
log.info("Deleting RBD Secret")
RBD_SECRET_OBJ.delete()
log.info("Deleting CEPHFS Secret")
CEPHFS_SECRET_OBJ.delete()
log.info("Deleting CEPH FILESYSTEM")
assert helpers.delete_all_cephfilesystem()
log.info("Deleting Storageclass")
assert helpers.delete_all_storageclass()
def test_bulk_pvc_creation_after_deletion_performance(
self, teardown_factory):
"""
Measuring PVC creation time of bulk of 75% of initial PVC bulk (120) in the same
rate after deleting ( serial deletion) 75% of the initial PVCs
and sends results to the Elastic Search DB
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
Returns:
"""
initial_number_of_pvcs = 120
number_of_pvcs = math.ceil(initial_number_of_pvcs * 0.75)
log.info(f"Start creating new {initial_number_of_pvcs} PVCs in a bulk")
pvc_objs, _ = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=self.namespace,
number_of_pvc=initial_number_of_pvcs,
size=self.pvc_size,
burst=True,
)
for pvc_obj in pvc_objs:
teardown_factory(pvc_obj)
with ThreadPoolExecutor() as executor:
for pvc_obj in pvc_objs:
executor.submit(helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND)
executor.submit(pvc_obj.reload)
log.info("Deleting 75% of the PVCs - 90 PVCs")
assert pvc.delete_pvcs(pvc_objs[:number_of_pvcs],
True), "Deletion of 75% of PVCs failed"
log.info("Re-creating the 90 PVCs")
pvc_objs, _ = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=self.namespace,
number_of_pvc=number_of_pvcs,
size=self.pvc_size,
burst=True,
)
start_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="end")
total = end_time - start_time
total_time = total.total_seconds()
logging.info(
f"Creation after deletion time of {number_of_pvcs} is {total_time} seconds."
)
for pvc_obj in pvc_objs:
teardown_factory(pvc_obj)
with ThreadPoolExecutor() as executor:
for pvc_obj in pvc_objs:
executor.submit(helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND)
executor.submit(pvc_obj.reload)
if total_time > 50:
raise ex.PerformanceException(
f"{number_of_pvcs} PVCs creation (after initial deletion of "
f"75% of PVCs) time is {total_time} and greater than 50 seconds."
)
logging.info(
f"{number_of_pvcs} PVCs creation time took less than a 50 seconds")
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_pvc_creation_after_deletion_measurement",
))
full_results.add_key("interface", self.interface)
full_results.add_key("number_of_pvcs", number_of_pvcs)
full_results.add_key("pvc_size", self.pvc_size)
full_results.add_key("creation_after_deletion_time", total_time)
# Write the test results into the ES server
full_results.es_write()
def teardown(self):
pvc_objs = get_all_pvc_objs(namespace="openshift-storage")
pvcs = [pvc_obj for pvc_obj in pvc_objs if "test-pvc" in pvc_obj.name]
delete_pvcs(pvc_objs=pvcs)
def destroy_ocs(self):
"""
Handle OCS destruction. Remove storage classes, PVCs, Storage
Cluster, Openshift-storage namespace, LocalVolume, unlabel
worker-storage nodes, delete ocs CRDs, etc.
"""
cluster_namespace = config.ENV_DATA["cluster_namespace"]
# https://access.redhat.com/documentation/en-us/red_hat_openshift_container_storage/4.5/html/deploying_openshift_container_storage_using_bare_metal_infrastructure/assembly_uninstalling-openshift-container-storage_rhocs
# Section 3.1 Step 1
# Deleting PVCs
rbd_pvcs = [
p for p in pvc.get_all_pvcs_in_storageclass(
constants.CEPHBLOCKPOOL_SC)
if not (p.data["metadata"]["namespace"] == cluster_namespace
and p.data["metadata"]["labels"]["app"] == "noobaa")
]
pvc.delete_pvcs(rbd_pvcs)
cephfs_pvcs = pvc.get_all_pvcs_in_storageclass(
constants.CEPHFILESYSTEM_SC)
# Section 3.1 Step 2
# Section 3.3 Step 1
# Removing OpenShift Container Platform registry from OpenShift Container Storage
registry_conf_name = "configs.imageregistry.operator.openshift.io"
registry_conf = ocp.OCP().exec_oc_cmd(
f"get {registry_conf_name} -o yaml")
if registry_conf["items"][0]["spec"].get("storage", dict()).get("pvc"):
patch = dict(spec=dict(storage=dict(emptyDir=dict(), pvc=None)))
ocp.OCP().exec_oc_cmd(
f"patch {registry_conf_name} cluster --type merge "
f"-p '{json.dumps(patch)}'")
# Section 3.3 Step 2
pvc.delete_pvcs(cephfs_pvcs)
# Section 3.1 Step 3
try:
ocp.OCP().exec_oc_cmd(
f"delete -n {cluster_namespace} storagecluster --all --wait=true"
)
except (CommandFailed, subprocess.TimeoutExpired):
pass
# Section 3.1 Step 4
ocp.OCP().exec_oc_cmd("project default")
ocp.OCP().exec_oc_cmd(
f"delete project {cluster_namespace} --wait=true --timeout=5m")
tried = 0
leftovers = True
while tried < 5:
# We need to loop here until the project can't be found
try:
ocp.OCP().exec_oc_cmd(
f"get project {cluster_namespace}",
out_yaml_format=False,
)
except CommandFailed:
leftovers = False
break
time.sleep(60)
tried += 1
if leftovers:
# https://access.redhat.com/documentation/en-us/red_hat_openshift_container_storage/4.5/html/troubleshooting_openshift_container_storage/troubleshooting-and-deleting-remaining-resources-during-uninstall_rhocs
leftover_types = [
"cephfilesystem.ceph.rook.io",
"cephobjectstore.ceph.rook.io",
"cephobjectstoreuser.ceph.rook.io",
"storagecluster.ocs.openshift.io",
]
patch = dict(metadata=dict(finalizers=None))
for obj_type in leftover_types:
try:
objs = ocp.OCP(kind=obj_type).get()
except CommandFailed:
continue
for obj in objs["items"]:
name = obj["metadata"]["name"]
ocp.OCP().exec_oc_cmd(
f"oc patch -n {cluster_namespace} {obj_type} {name} --type=merge -p '{json.dumps(patch)}'"
)
# Section 3.1 Step 5
nodes = ocp.OCP().exec_oc_cmd(
"get node -l cluster.ocs.openshift.io/openshift-storage= -o yaml")
for node in nodes["items"]:
node_name = node["metadata"]["name"]
ocp.OCP().exec_oc_cmd(
f"debug node/{node_name} -- chroot /host rm -rfv /var/lib/rook"
)
# Section 3.1 Step 6
ocp.OCP().exec_oc_cmd(
"delete storageclass openshift-storage.noobaa.io --wait=true --timeout=5m"
)
# Section 3.1 Step 7
ocp.OCP().exec_oc_cmd(
"label nodes --all cluster.ocs.openshift.io/openshift-storage-")
ocp.OCP().exec_oc_cmd("label nodes --all topology.rook.io/rack-")
# Section 3.1 Step 8
pvs = ocp.OCP(kind="PersistentVolume").get()
for pv in pvs["items"]:
pv_name = pv["metadata"]["name"]
if pv_name.startswith("ocs-storagecluster-ceph"):
ocp.OCP().exec_oc_cmd(f"oc delete pv {pv_name}")
# Section 3.1 Step 9
# Note that the below process differs from the documentation slightly.
# Instead of deleting all CRDs at once and calling the job done, we
# iterate over a list of them, noting which ones don't delete fully and
# applying the standard workaround of removing the finalizers from any
# CRs and also the CRD. Finally, the documentation leaves out a few
# CRDs that we've seen in deployed clusters.
crd_types = [
"backingstores.noobaa.io",
"bucketclasses.noobaa.io",
"cephblockpools.ceph.rook.io",
"cephclients.ceph.rook.io",
"cephclusters.ceph.rook.io",
"cephfilesystems.ceph.rook.io",
"cephnfses.ceph.rook.io",
"cephobjectrealms.ceph.rook.io", # not in doc
"cephobjectstores.ceph.rook.io",
"cephobjectstoreusers.ceph.rook.io",
"cephobjectzonegroups.ceph.rook.io", # not in doc
"cephobjectzones.ceph.rook.io", # not in doc
"cephrbdmirrors.ceph.rook.io", # not in doc
"noobaas.noobaa.io",
"ocsinitializations.ocs.openshift.io",
"storageclusterinitializations.ocs.openshift.io",
"storageclusters.ocs.openshift.io",
]
cr_patch = json.dumps(dict(finalizers=None))
crd_patch = json.dumps(dict(metadata=dict(finalizers=None)))
for crd_type in crd_types:
try:
ocp.OCP().exec_oc_cmd(
f"delete crd {crd_type} --wait=true --timeout=30s",
out_yaml_format=False,
)
except CommandFailed:
pass
crs = []
try:
crs = ocp.OCP(kind=crd_type).get(all_namespaces=True)["items"]
except CommandFailed:
continue
for cr in crs:
cr_md = cr["metadata"]
ocp.OCP().exec_oc_cmd(
f"patch -n {cr_md['namespace']} {crd_type} {cr_md['name']} --type=merge -p '{cr_patch}'"
)
try:
crs = ocp.OCP(kind=crd_type).get(all_namespaces=True)["items"]
except CommandFailed:
continue
ocp.OCP().exec_oc_cmd(
f"patch crd {crd_type} --type=merge -p '{crd_patch}'")
# End sections from above documentation
ocp.OCP().exec_oc_cmd(
f"delete catalogsource {constants.OPERATOR_CATALOG_SOURCE_NAME} "
f"-n {constants.MARKETPLACE_NAMESPACE}")
storageclasses = ocp.OCP(kind="StorageClass").get(all_namespaces=True)
for sc in storageclasses["items"]:
if sc["provisioner"].startswith("openshift-storage."):
sc_name = sc["metadata"]["name"]
ocp.OCP().exec_oc_cmd(f"delete storageclass {sc_name}")
volumesnapclasses = ocp.OCP(kind="VolumeSnapshotClass").get(
all_namespaces=True)
for vsc in volumesnapclasses["items"]:
if vsc["driver"].startswith("openshift-storage."):
vsc_name = vsc["metadata"]["name"]
ocp.OCP().exec_oc_cmd(f"delete volumesnapshotclass {vsc_name}")
self.destroy_lso()
def finalizer():
# Delete newly created PVCs
assert delete_pvcs(cls_ref.pvc_objs_new), 'Failed to delete PVCs'
log.info(f'Newly created {cls_ref.num_of_pvcs} PVCs are now deleted.')
def uninstall_cluster_logging():
"""
Function to uninstall cluster-logging from the cluster
Deletes the project "openshift-logging" and "openshift-operators-redhat"
"""
# Validating the pods before deleting the instance
pod_list = get_all_pods(namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
for pod in pod_list:
logger.info(
f"Pods running in the openshift-logging namespace {pod.name}")
# Excluding cluster-logging-operator from pod_list and getting pod names
pod_names_list = [
pod.name for pod in pod_list
if not pod.name.startswith("cluster-logging-operator")
]
pvc_objs = get_all_pvc_objs(
namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
# Fetch image uuid associated with PVCs to be deleted
pvc_uuid_map = {}
for pvc_obj in pvc_objs:
pvc_uuid_map[pvc_obj.name] = pvc_obj.image_uuid
# Checking for used space
cbp_name = default_ceph_block_pool()
used_space_before_deletion = fetch_used_size(cbp_name)
logger.info(
f"Used space before deletion of cluster logging {used_space_before_deletion}"
)
# Deleting the clusterlogging instance
clusterlogging_obj = ocp.OCP(
kind=constants.CLUSTER_LOGGING,
namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
assert clusterlogging_obj.delete(resource_name="instance")
check_pod_vanished(pod_names_list)
for pvc_obj in pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
assert delete_pvcs(pvc_objs=pvc_objs), "PVCs deletion failed"
for pvc_obj in pvc_objs:
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name, timeout=300)
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name, timeout=300)
logger.info("Verified: PVCs are deleted.")
logger.info("Verified: PV are deleted")
for pvc_name, uuid in pvc_uuid_map.items():
rbd = verify_volume_deleted_in_backend(
interface=constants.CEPHBLOCKPOOL,
image_uuid=uuid,
pool_name=cbp_name)
assert rbd, f"Volume associated with PVC {pvc_name} still exists " f"in backend"
# Checking for used space after PVC deletion
used_space_after_deletion = fetch_used_size(cbp_name, exp_val=30)
logger.info(
f"Used space after deletion of cluster logging {used_space_after_deletion}"
)
if used_space_after_deletion < used_space_before_deletion:
logger.info("Expected !!! Space has reclaimed")
else:
logger.warning(
"Unexpected !! No space reclaimed after deletion of PVC")
# Deleting the RBAC permission set
rbac_role = ocp.OCP(
kind=constants.ROLE,
namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE)
rbac_role.delete(yaml_file=constants.EO_RBAC_YAML)
openshift_logging_namespace = ocp.OCP(
kind=constants.NAMESPACES,
namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
openshift_operators_redhat_namespace = ocp.OCP(
kind=constants.NAMESPACES,
namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE,
)
if openshift_operators_redhat_namespace.get():
assert openshift_operators_redhat_namespace.delete(
resource_name=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE)
logger.info(
"The project openshift-opertors-redhat got deleted successfully")
if openshift_logging_namespace.get():
assert openshift_logging_namespace.delete(
resource_name=constants.OPENSHIFT_LOGGING_NAMESPACE)
logger.info("The namespace openshift-logging got deleted successfully")
def test_noobaa_db_backup_and_recovery(
self,
pvc_factory,
pod_factory,
snapshot_factory,
bucket_factory,
rgw_bucket_factory,
):
"""
Test case to verify noobaa backup and recovery
1. Take snapshot db-noobaa-db-0 PVC and retore it to PVC
2. Scale down the statefulset noobaa-db
3. Get the yaml of the current PVC, db-noobaa-db-0 and
change the parameter persistentVolumeReclaimPolicy to Retain for restored PVC
4. Delete both PVCs, the PV for the original claim db-noobaa-db-0 will be removed.
The PV for claim db-noobaa-db-0-snapshot-restore will move to ‘Released’
5. Edit again restore PV and remove the claimRef section.
The volume will transition to Available.
6. Edit the yaml db-noobaa-db-0.yaml and change the setting volumeName to restored PVC.
7. Scale up the stateful set again and the pod should be running
"""
# Initialise variable
self.noobaa_db_sst_name = "noobaa-db-pg"
# Get noobaa pods before execution
noobaa_pods = get_noobaa_pods()
# Get noobaa PVC before execution
noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"])
noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get(
"volumeName")
# Take snapshot db-noobaa-db-0 PVC
log.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC")
snap_obj = snapshot_factory(
pvc_obj=noobaa_pvc_obj[0],
wait=True,
snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot",
)
log.info(
f"Successfully created snapshot {snap_obj.name} and in Ready state"
)
# Restore it to PVC
log.info(f"Restoring snapshot {snap_obj.name} to create new PVC")
sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName")
pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get(
"requests").get("storage"))
self.restore_pvc_obj = create_restore_pvc(
sc_name=sc_name,
snap_name=snap_obj.name,
namespace=snap_obj.namespace,
size=pvc_size,
pvc_name=f"{snap_obj.name}-restore",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
)
wait_for_resource_state(self.restore_pvc_obj, constants.STATUS_BOUND)
self.restore_pvc_obj.reload()
log.info(f"Succeesfuly created PVC {self.restore_pvc_obj.name} "
f"from snapshot {snap_obj.name}")
# Scale down the statefulset noobaa-db
modify_statefulset_replica_count(
statefulset_name=self.noobaa_db_sst_name, replica_count=0
), f"Failed to scale down the statefulset {self.noobaa_db_sst_name}"
# Get the noobaa-db PVC
pvc_obj = OCP(kind=constants.PVC,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name)
# Get the restored noobaa PVC and
# change the parameter persistentVolumeReclaimPolicy to Retain
restored_noobaa_pvc_obj = get_pvc_objs(
pvc_names=[f"{snap_obj.name}-restore"])
restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get(
"spec").get("volumeName"))
pv_obj = OCP(kind=constants.PV,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Retain {restored_noobaa_pv_name}")
# Delete both PVCs
delete_pvcs(pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]])
# Validate original claim db-noobaa-db-0 removed
assert validate_pv_delete(
pv_name=noobaa_pv_name
), f"PV not deleted, still exist {noobaa_pv_name}"
# Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state
pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED,
resource_name=restored_noobaa_pv_name)
# Edit again restore PV and remove the claimRef section
log.info(
f"Remove the claimRef section from PVC {restored_noobaa_pv_name}")
params = '[{"op": "remove", "path": "/spec/claimRef"}]'
pv_obj.patch(resource_name=restored_noobaa_pv_name,
params=params,
format_type="json")
log.info(
f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}"
)
# Validate PV is in Available state
pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE,
resource_name=restored_noobaa_pv_name)
# Edit the yaml db-noobaa-db-0.yaml and change the
# setting volumeName to restored PVC
noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name
noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml)
noobaa_pvc_yaml.create()
# Validate noobaa PVC is in bound state
pvc_obj.wait_for_resource(
condition=constants.STATUS_BOUND,
resource_name=noobaa_pvc_obj[0].name,
timeout=120,
)
# Scale up the statefulset again
assert modify_statefulset_replica_count(
statefulset_name=self.noobaa_db_sst_name, replica_count=1
), f"Failed to scale up the statefulset {self.noobaa_db_sst_name}"
# Validate noobaa pod is up and running
pod_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(noobaa_pods),
selector=constants.NOOBAA_APP_LABEL,
)
# Change the parameter persistentVolumeReclaimPolicy to Delete again
params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Delete {restored_noobaa_pv_name}")
log.info(
"Changed the parameter persistentVolumeReclaimPolicy to Delete again"
)
# Verify all storage pods are running
wait_for_storage_pods()
# Creating Resources
log.info("Creating Resources using sanity helpers")
self.sanity_helpers.create_resources(pvc_factory, pod_factory,
bucket_factory,
rgw_bucket_factory)
# Deleting Resources
self.sanity_helpers.delete_resources()
# Verify everything running fine
log.info(
"Verifying All resources are Running and matches expected result")
self.sanity_helpers.health_check(tries=120)
def test_multiple_sc_comp_rep_data_deletion(self, storageclass_factory,
pvc_factory, pod_factory):
"""
This test function does below,
*. Creates 2 Storage Class with creating new rbd pool
*. Creates PVCs using new Storage Class
*. Mount PVC to an app pod
*. Run IO on an app pod
*. Delete the pods and pvc
*. Verify that the data is deleted
"""
log.info("Creating storageclasses with compression and replica3")
interface_type = constants.CEPHBLOCKPOOL
sc_obj1 = storageclass_factory(
interface=interface_type,
new_rbd_pool=True,
replica=3,
compression="aggressive",
)
log.info("Creating storageclasses with compression and replica2")
sc_obj2 = storageclass_factory(
interface=interface_type,
new_rbd_pool=True,
replica=2,
compression="aggressive",
)
sc_obj_list = [sc_obj1, sc_obj2]
pod_obj_list = []
pvc_obj_list = []
log.info("Creating PVCs and PODs")
for sc_obj in sc_obj_list:
pvc_obj = pvc_factory(interface=interface_type,
storageclass=sc_obj)
pvc_obj_list.append(pvc_obj)
pod_obj_list.append(
pod_factory(interface=interface_type, pvc=pvc_obj))
log.info("Running IO on pods")
for pod_obj in pod_obj_list:
pod_obj.run_io("fs", size="1G")
for pod_obj in pod_obj_list:
get_fio_rw_iops(pod_obj)
log.info("deleting PODs and PVCs")
delete_pods(pod_obj_list, wait=True)
delete_pvcs(pvc_obj_list, concurrent=True)
log.info("Wait for 15 seconds for all data to delete")
sleep(15)
log.info("Checking stats after deleting PODs and PVCs")
for sc_obj in sc_obj_list:
pvc_list = get_all_pvcs_in_storageclass(sc_obj.name)
if len(pvc_list) == 0:
cbp_name = sc_obj.get()["parameters"]["pool"]
ceph_pool_byte_used = get_byte_used_by_pool(cbp_name)
log.info(
f"pool {cbp_name} has {ceph_pool_byte_used} bytes used")
if ceph_pool_byte_used > MAX_BYTES_IN_POOL_AFTER_DATA_DELETE:
raise PoolDataNotErased(
f"Pool {cbp_name} has {ceph_pool_byte_used} bytes which were not deleted"
)
else:
raise PvcNotDeleted(f"PVC {pvc_list} were not deleted")
def factory(snapshot_factory=snapshot_factory):
# Get noobaa pods before execution
noobaa_pods = pod.get_noobaa_pods()
# Get noobaa PVC before execution
noobaa_pvc_obj = pvc.get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"])
noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get(
"volumeName")
# Take snapshot db-noobaa-db-0 PVC
logger.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC")
snap_obj = snapshot_factory(
pvc_obj=noobaa_pvc_obj[0],
wait=True,
snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot",
)
logger.info(
f"Successfully created snapshot {snap_obj.name} and in Ready state"
)
# Restore it to PVC
logger.info(f"Restoring snapshot {snap_obj.name} to create new PVC")
sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName")
pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get(
"requests").get("storage"))
restore_pvc_obj = pvc.create_restore_pvc(
sc_name=sc_name,
snap_name=snap_obj.name,
namespace=snap_obj.namespace,
size=pvc_size,
pvc_name=f"{snap_obj.name}-restore",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
)
restore_pvc_objs.append(restore_pvc_obj)
wait_for_resource_state(restore_pvc_obj, constants.STATUS_BOUND)
restore_pvc_obj.reload()
logger.info(f"Succeesfuly created PVC {restore_pvc_obj.name} "
f"from snapshot {snap_obj.name}")
# Scale down the statefulset noobaa-db
modify_statefulset_replica_count(
statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=0
), f"Failed to scale down the statefulset {constants.NOOBAA_DB_STATEFULSET}"
# Get the noobaa-db PVC
pvc_obj = OCP(kind=constants.PVC,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name)
# Get the restored noobaa PVC and
# change the parameter persistentVolumeReclaimPolicy to Retain
restored_noobaa_pvc_obj = pvc.get_pvc_objs(
pvc_names=[f"{snap_obj.name}-restore"])
restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get(
"spec").get("volumeName"))
pv_obj = OCP(kind=constants.PV,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Retain {restored_noobaa_pv_name}")
# Delete both PVCs
pvc.delete_pvcs(
pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]])
# Validate original claim db-noobaa-db-0 removed
assert validate_pv_delete(
pv_name=noobaa_pv_name
), f"PV not deleted, still exist {noobaa_pv_name}"
# Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state
pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED,
resource_name=restored_noobaa_pv_name)
# Edit again restore PV and remove the claimRef section
logger.info(
f"Remove the claimRef section from PVC {restored_noobaa_pv_name}")
params = '[{"op": "remove", "path": "/spec/claimRef"}]'
pv_obj.patch(resource_name=restored_noobaa_pv_name,
params=params,
format_type="json")
logger.info(
f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}"
)
# Validate PV is in Available state
pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE,
resource_name=restored_noobaa_pv_name)
# Edit the yaml db-noobaa-db-0.yaml and change the
# setting volumeName to restored PVC
noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name
noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml)
noobaa_pvc_yaml.create()
# Validate noobaa PVC is in bound state
pvc_obj.wait_for_resource(
condition=constants.STATUS_BOUND,
resource_name=noobaa_pvc_obj[0].name,
timeout=120,
)
# Scale up the statefulset again
assert modify_statefulset_replica_count(
statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=1
), f"Failed to scale up the statefulset {constants.NOOBAA_DB_STATEFULSET}"
# Validate noobaa pod is up and running
pod_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(noobaa_pods),
selector=constants.NOOBAA_APP_LABEL,
)
# Change the parameter persistentVolumeReclaimPolicy to Delete again
params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Delete {restored_noobaa_pv_name}")
logger.info(
"Changed the parameter persistentVolumeReclaimPolicy to Delete again"
)
def test_bulk_pvc_creation_after_deletion_performance(
self, interface_iterate, storageclass_factory
):
"""
Measuring PVC creation time of bulk of 75% of initial PVC bulk (120) in the same
rate after deleting ( serial deletion) 75% of the initial PVCs
and sends results to the Elastic Search DB
"""
self.interface = interface_iterate
self.sc_obj = storageclass_factory(self.interface)
initial_number_of_pvcs = 120
number_of_pvcs = math.ceil(initial_number_of_pvcs * 0.75)
# Getting the test start time
self.test_start_time = self.get_time()
log.info(f"Start creating new {initial_number_of_pvcs} PVCs in a bulk")
self.pvc_bulk_create_and_wait_for_bound(initial_number_of_pvcs)
log.info(f"Deleting 75% of the PVCs - {number_of_pvcs} PVCs")
assert pvc.delete_pvcs(
self.pvc_objs[:number_of_pvcs], True
), "Deletion of 75% of PVCs failed"
# save the list of pvcs which not deleted, for the teardown phase
original_pvcs = self.pvc_objs[number_of_pvcs:]
log.info(f"Re-creating the {number_of_pvcs} PVCs")
csi_bulk_start_time = self.get_time(time_format="csi")
self.pvc_bulk_create_and_wait_for_bound(number_of_pvcs)
# Get the bulk recraation time - total time.
total_time = self.get_bulk_creation_time()
log.info(
f"Creation after deletion time of {number_of_pvcs} is {total_time} seconds."
)
if total_time > 50:
raise ex.PerformanceException(
f"{number_of_pvcs} PVCs creation (after initial deletion of "
f"75% of PVCs) time is {total_time} and greater than 50 seconds."
)
log.info(f"{number_of_pvcs} PVCs creation time took less than a 50 seconds")
csi_creation_times = performance_lib.csi_bulk_pvc_time_measure(
self.interface, self.pvc_objs, "create", csi_bulk_start_time
)
# Getting the end time of the test
self.test_end_time = self.get_time()
# update the list of pvcs for the teardown process
self.pvc_objs += original_pvcs
# Produce ES report
self.results_path = os.path.join(
"/",
*self.results_path,
"test_bulk_pvc_creation_after_deletion_performance",
)
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_pvc_creation_after_deletion_measurement",
)
)
# Add the test time to the ES report
full_results.add_key(
"test_time", {"start": self.test_start_time, "end": self.test_end_time}
)
full_results.add_key("number_of_pvcs", number_of_pvcs)
full_results.add_key("creation_after_deletion_time", total_time)
full_results.add_key("creation_after_deletion_csi_time", csi_creation_times)
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (2 - according to the parameters)
self.write_result_to_file(res_link)
