def test_respin_mcg_pod_and_check_data_integrity(self, mcg_obj, cld_mgr,
awscli_pod,
ns_resource_factory,
bucket_factory, mcg_pod):
"""
Test Write to ns bucket using MCG RPC and read directly from AWS.
Respin one of mcg pods when data are uploaded.
"""
logger.info("Create the namespace resource and verify health")
resource = ns_resource_factory()
target_bucket_name = resource[0]
ns_resource_name = resource[1]
s3_creds = {
"access_key_id": cld_mgr.aws_client.access_key,
"access_key": cld_mgr.aws_client.secret_key,
"endpoint": constants.MCG_NS_AWS_ENDPOINT,
"region": self.DEFAULT_REGION,
}
logger.info(
"Create the namespace bucket on top of the namespace resource")
rand_ns_bucket = bucket_factory(
amount=1,
interface="mcg-namespace",
write_ns_resource=ns_resource_name,
read_ns_resources=[ns_resource_name],
)[0].name
logger.info("Upload files to NS bucket")
self.write_files_to_pod_and_upload(mcg_obj,
awscli_pod,
bucket_to_write=rand_ns_bucket,
amount=3)
logger.info(f"Respin mcg resource {mcg_pod}")
noobaa_pods = pod.get_noobaa_pods()
pod_obj = [pod for pod in noobaa_pods
if pod.name.startswith(mcg_pod)][0]
pod_obj.delete(force=True)
logger.info("Wait for noobaa pods to come up")
assert pod_obj.ocp.wait_for_resource(
condition="Running",
selector="app=noobaa",
resource_count=len(noobaa_pods),
timeout=1000,
)
logger.info("Wait for noobaa health to be OK")
ceph_cluster_obj = CephCluster()
ceph_cluster_obj.wait_for_noobaa_health_ok()
logger.info("Read files directly from AWS")
self.download_files(mcg_obj,
awscli_pod,
bucket_to_read=target_bucket_name,
s3_creds=s3_creds)
logger.info("Compare between uploaded files and downloaded files")
assert self.compare_dirs(awscli_pod, amount=3)
def test_rgw_host_node_failure(
self, nodes, node_restart_teardown, mcg_obj, bucket_factory
):
"""
Test case to fail node where RGW and Noobaa-db-0 hosting
and verify new pod spuns on healthy node
"""
# Get rgw pods
rgw_pod_obj = get_rgw_pods()
# Get nooba pods
noobaa_pod_obj = get_noobaa_pods()
# Get the node where noobaa-db hosted
for noobaa_pod in noobaa_pod_obj:
if noobaa_pod.name == "noobaa-db-0":
noobaa_pod_node = get_pod_node(noobaa_pod)
for rgw_pod in rgw_pod_obj:
pod_node = rgw_pod.get().get("spec").get("nodeName")
if pod_node == noobaa_pod_node.name:
# Stop the node
log.info(
f"Stopping node {pod_node} where"
f" rgw pod {rgw_pod.name} and noobaa-db-0 hosted"
)
node_obj = get_node_objs(node_names=[pod_node])
nodes.stop_nodes(node_obj)
# Validate old rgw pod went terminating state
wait_for_resource_state(
resource=rgw_pod, state=constants.STATUS_TERMINATING, timeout=720
)
# Validate new rgw pod spun
ocp_obj = OCP(
kind=constants.POD, namespace=defaults.ROOK_CLUSTER_NAMESPACE
)
ocp_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(rgw_pod_obj),
selector=constants.RGW_APP_LABEL,
)
# Create OBC and read wnd write
self.create_obc_creation(bucket_factory, mcg_obj, "Object-key-1")
# Start the node
nodes.start_nodes(node_obj)
# Create OBC and read wnd write
self.create_obc_creation(bucket_factory, mcg_obj, "Object-key-2")
# Verify cluster health
self.sanity_helpers.health_check()
# Verify all storage pods are running
wait_for_storage_pods()
def recover_mcg():
"""
Recovery procedure for noobaa by re-spinning the pods after mon recovery
"""
logger.info("Re-spinning noobaa pods")
for noobaa_pod in get_noobaa_pods():
noobaa_pod.delete()
for noobaa_pod in get_noobaa_pods():
wait_for_resource_state(
resource=noobaa_pod, state=constants.STATUS_RUNNING, timeout=600
)
if config.ENV_DATA["platform"].lower() in constants.ON_PREM_PLATFORMS:
logger.info("Re-spinning RGW pods")
for rgw_pod in get_rgw_pods():
rgw_pod.delete()
for rgw_pod in get_rgw_pods():
wait_for_resource_state(
resource=rgw_pod, state=constants.STATUS_RUNNING, timeout=600
)
def scan_cluster(self):
"""
Get accurate info on current state of pods
"""
self._ceph_pods = pod.get_all_pods(self._namespace)
# TODO: Workaround for BZ1748325:
mons = pod.get_mon_pods(self.mon_selector, self.namespace)
for mon in mons:
if mon.ocp.get_resource_status(
mon.name) == constant.STATUS_RUNNING:
self.mons.append(mon)
# TODO: End of workaround for BZ1748325
self.mdss = pod.get_mds_pods(self.mds_selector, self.namespace)
self.mgrs = pod.get_mgr_pods(self.mgr_selector, self.namespace)
self.osds = pod.get_osd_pods(self.osd_selector, self.namespace)
self.noobaas = pod.get_noobaa_pods(self.noobaa_selector,
self.namespace)
self.rgws = pod.get_rgw_pods()
self.toolbox = pod.get_ceph_tools_pod()
# set port attrib on mon pods
self.mons = list(map(self.set_port, self.mons))
self.cluster.reload()
if self.cephfs:
self.cephfs.reload()
else:
try:
self.cephfs_config = self.CEPHFS.get().get('items')[0]
self.cephfs = ocs.OCS(**self.cephfs_config)
self.cephfs.reload()
except IndexError as e:
logging.warning(e)
logging.warning("No CephFS found")
self.mon_count = len(self.mons)
self.mds_count = len(self.mdss)
self.mgr_count = len(self.mgrs)
self.osd_count = len(self.osds)
self.noobaa_count = len(self.noobaas)
self.rgw_count = len(self.rgws)
def test_respin_mcg_pod_and_check_data_integrity_crd(
self,
mcg_obj,
cld_mgr,
awscli_pod,
namespace_store_factory,
bucket_factory,
mcg_pod,
):
"""
Test Write to ns bucket using CRDs and read directly from AWS.
Respin one of mcg pods when data are uploaded.
"""
logger.info("Create the namespace resources and verify health")
nss_tup = ("oc", {"aws": [(1, self.DEFAULT_REGION)]})
ns_store = namespace_store_factory(*nss_tup)[0]
logger.info(
"Create the namespace bucket on top of the namespace stores")
bucketclass_dict = {
"interface": "OC",
"namespace_policy_dict": {
"type": "Single",
"namespacestores": [ns_store],
},
}
logger.info(
"Create the namespace bucket on top of the namespace resource")
ns_bucket = bucket_factory(
amount=1,
interface=bucketclass_dict["interface"],
bucketclass=bucketclass_dict,
)[0].name
s3_creds = {
"access_key_id": cld_mgr.aws_client.access_key,
"access_key": cld_mgr.aws_client.secret_key,
"endpoint": constants.MCG_NS_AWS_ENDPOINT,
"region": self.DEFAULT_REGION,
}
logger.info("Upload files to NS bucket")
self.write_files_to_pod_and_upload(mcg_obj,
awscli_pod,
bucket_to_write=ns_bucket,
amount=3)
logger.info(f"Respin mcg resource {mcg_pod}")
noobaa_pods = pod.get_noobaa_pods()
pod_obj = [pod for pod in noobaa_pods
if pod.name.startswith(mcg_pod)][0]
pod_obj.delete(force=True)
logger.info("Wait for noobaa pods to come up")
assert pod_obj.ocp.wait_for_resource(
condition="Running",
selector="app=noobaa",
resource_count=len(noobaa_pods),
timeout=1000,
)
logger.info("Wait for noobaa health to be OK")
ceph_cluster_obj = CephCluster()
ceph_cluster_obj.wait_for_noobaa_health_ok()
logger.info("Read files directly from AWS")
self.download_files(mcg_obj,
awscli_pod,
bucket_to_read=ns_store.uls_name,
s3_creds=s3_creds)
logger.info("Compare between uploaded files and downloaded files")
assert self.compare_dirs(awscli_pod, amount=3)
def test_noobaa_sts_host_node_failure(
self,
noobaa_sts,
respin_noobaa_operator,
mcg_obj,
bucket_factory,
nodes,
node_restart_teardown,
):
"""
Test case to fail node where NooBaa Statefulset pod (noobaa-core, noobaa-db)
is hosted and verify the pod is rescheduled on a healthy node
"""
executor = ThreadPoolExecutor(max_workers=1)
pod_obj = OCP(kind=constants.POD,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
# Get noobaa statefulset pod and node where it is hosted
noobaa_sts_pod = get_noobaa_pods(
noobaa_label=self.labels_map[noobaa_sts])[0]
noobaa_sts_pod_node = get_pod_node(noobaa_sts_pod)
log.info(
f"{noobaa_sts_pod.name} is running on {noobaa_sts_pod_node.name}")
# Get the NooBaa operator pod and node where it is hosted
# Check if NooBaa operator and statefulset pod are hosted on same node
noobaa_operator_pod = get_noobaa_pods(noobaa_label=self.labels_map[
constants.NOOBAA_OPERATOR_DEPLOYMENT])[0]
noobaa_operator_pod_node = get_pod_node(noobaa_operator_pod)
log.info(
f"{noobaa_operator_pod.name} is running on {noobaa_operator_pod_node.name}"
)
if noobaa_sts_pod_node.name == noobaa_operator_pod_node.name:
operator_on_same_node = True
log.info(
f"{noobaa_sts_pod.name} and {noobaa_operator_pod.name} are running on same node."
)
else:
operator_on_same_node = False
log.info(
f"{noobaa_sts_pod.name} and {noobaa_operator_pod.name} are running on different node."
)
# Stop the node
log.info(
f"Stopping {noobaa_sts_pod_node.name} where {noobaa_sts_pod.name} is hosted"
)
stop_thread = executor.submit(nodes.stop_nodes,
nodes=[noobaa_sts_pod_node])
node.wait_for_nodes_status(node_names=[noobaa_sts_pod_node.name],
status=constants.NODE_NOT_READY)
# Disrupt NooBaa operator
if respin_noobaa_operator:
noobaa_operator_pod.delete(force=True)
# Check result of 'stop_thread'
stop_thread.result()
# Wait for NooBaa operator pod to reach terminating state if on same node
# and not respun
if operator_on_same_node and not respin_noobaa_operator:
wait_for_resource_state(
resource=noobaa_operator_pod,
state=constants.STATUS_TERMINATING,
timeout=360,
)
# Wait for NooBaa operator pod to reach running state
pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
selector=self.labels_map[constants.NOOBAA_OPERATOR_DEPLOYMENT],
resource_count=1,
)
# Verify NooBaa statefulset pod reschedules on another node
try:
for pod_list in TimeoutSampler(
60,
3,
get_noobaa_pods,
noobaa_label=self.labels_map[noobaa_sts],
):
if len(pod_list) == 1:
pod_node = get_pod_node(pod_list[0])
if pod_node.name != noobaa_sts_pod_node.name:
log.info(
f"{pod_list[0].name} has been rescheduled on {pod_node.name}"
)
break
log.info(
f"Waiting for {noobaa_sts_pod.name} pod to be rescheduled"
)
except TimeoutExpiredError:
raise TimeoutExpiredError(
f"{noobaa_sts_pod.name} pod not rescheduled within 60 seconds")
# Wait for rescheduled pod to reach Running state.
# For noobaa-db pod which is attached to a PV it may take more time (~8 minutes)
# until the new pod can attach to the PV
pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
selector=self.labels_map[noobaa_sts],
resource_count=1,
timeout=800
if noobaa_sts == constants.NOOBAA_DB_STATEFULSET else 60,
sleep=30 if noobaa_sts == constants.NOOBAA_DB_STATEFULSET else 3,
)
# Start the node
log.info(
f"Starting {noobaa_sts_pod_node.name} where {noobaa_sts_pod.name} was hosted"
)
nodes.start_nodes(nodes=[noobaa_sts_pod_node])
node.wait_for_nodes_status(node_names=[noobaa_sts_pod_node.name],
status=constants.NODE_READY)
log.info("Wait for all pods to be in running state")
wait_for_pods_to_be_running(timeout=300)
# Check cluster health
self.sanity_helpers.health_check()
# Creates bucket then writes, reads and deletes objects
self.sanity_helpers.obc_put_obj_create_delete(mcg_obj, bucket_factory)
def test_noobaa_rebuild(self, bucket_factory):
"""
Test case to verify noobaa rebuild. Verifies KCS: https://access.redhat.com/solutions/5948631
1. Stop the noobaa-operator by setting the replicas of noobaa-operator deployment to 0.
2. Delete the noobaa deployments/statefulsets.
3. Delete the PVC db-noobaa-db-0.
4. Patch existing backingstores and bucketclasses to remove finalizer
5. Delete the backingstores/bucketclass.
6. Delete the noobaa secrets.
7. Restart noobaa-operator by setting the replicas back to 1.
8. Monitor the pods in openshift-storage for noobaa pods to be Running.
"""
dep_ocp = OCP(kind=constants.DEPLOYMENT,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
state_ocp = OCP(kind=constants.STATEFULSET,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"])
# Scale down noobaa operator
logger.info(
f"Scaling down {constants.NOOBAA_OPERATOR_DEPLOYMENT} deployment to replica: 0"
)
dep_ocp.exec_oc_cmd(
f"scale deployment {constants.NOOBAA_OPERATOR_DEPLOYMENT} --replicas=0"
)
# Delete noobaa deployments and statefulsets
logger.info("Deleting noobaa deployments and statefulsets")
dep_ocp.delete(resource_name=constants.NOOBAA_ENDPOINT_DEPLOYMENT)
state_ocp.delete(resource_name=constants.NOOBAA_DB_STATEFULSET)
state_ocp.delete(resource_name=constants.NOOBAA_CORE_STATEFULSET)
# Delete noobaa-db pvc
pvc_obj = OCP(kind=constants.PVC,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
logger.info("Deleting noobaa-db pvc")
pvc_obj.delete(resource_name=noobaa_pvc_obj[0].name, wait=True)
pvc_obj.wait_for_delete(resource_name=noobaa_pvc_obj[0].name,
timeout=300)
# Patch and delete existing backingstores
params = '{"metadata": {"finalizers":null}}'
bs_obj = OCP(kind=constants.BACKINGSTORE,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
for bs in bs_obj.get()["items"]:
assert bs_obj.patch(
resource_name=bs["metadata"]["name"],
params=params,
format_type="merge",
), "Failed to change the parameter in backingstore"
logger.info(f"Deleting backingstore: {bs['metadata']['name']}")
bs_obj.delete(resource_name=bs["metadata"]["name"])
# Patch and delete existing bucketclass
bc_obj = OCP(kind=constants.BUCKETCLASS,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
for bc in bc_obj.get()["items"]:
assert bc_obj.patch(
resource_name=bc["metadata"]["name"],
params=params,
format_type="merge",
), "Failed to change the parameter in bucketclass"
logger.info(f"Deleting bucketclass: {bc['metadata']['name']}")
bc_obj.delete(resource_name=bc["metadata"]["name"])
# Delete noobaa secrets
logger.info("Deleting noobaa related secrets")
dep_ocp.exec_oc_cmd(
"delete secrets noobaa-admin noobaa-endpoints noobaa-operator noobaa-server"
)
# Scale back noobaa-operator deployment
logger.info(
f"Scaling back {constants.NOOBAA_OPERATOR_DEPLOYMENT} deployment to replica: 1"
)
dep_ocp.exec_oc_cmd(
f"scale deployment {constants.NOOBAA_OPERATOR_DEPLOYMENT} --replicas=1"
)
# Wait and validate noobaa PVC is in bound state
pvc_obj.wait_for_resource(
condition=constants.STATUS_BOUND,
resource_name=noobaa_pvc_obj[0].name,
timeout=600,
sleep=120,
)
# Validate noobaa pods are up and running
pod_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
noobaa_pods = get_noobaa_pods()
pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(noobaa_pods),
selector=constants.NOOBAA_APP_LABEL,
timeout=900,
)
# Verify everything running fine
logger.info(
"Verifying all resources are Running and matches expected result")
self.sanity_helpers.health_check(tries=120)
# Verify default backingstore/bucketclass
default_bs = OCP(kind=constants.BACKINGSTORE,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE).get(
resource_name=DEFAULT_NOOBAA_BACKINGSTORE)
default_bc = OCP(kind=constants.BUCKETCLASS,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE).get(
resource_name=DEFAULT_NOOBAA_BUCKETCLASS)
assert (default_bs["status"]["phase"] == default_bc["status"]["phase"]
== constants.STATUS_READY
), "Failed: Default bs/bc are not in ready state"
# Create OBCs
logger.info("Creating OBCs after noobaa rebuild")
bucket_factory(amount=3, interface="OC", verify_health=True)
def test_rgw_host_node_failure(self, nodes, node_restart_teardown,
node_drain_teardown, mcg_obj,
bucket_factory):
"""
Test case to fail node where RGW and the NooBaa DB are hosted
and verify the new pods spin on a healthy node
"""
# Get nooba pods
noobaa_pod_obj = get_noobaa_pods()
# Get the node where noobaa-db hosted
noobaa_pod_node = None
for noobaa_pod in noobaa_pod_obj:
if noobaa_pod.name in [
constants.NB_DB_NAME_46_AND_BELOW,
constants.NB_DB_NAME_47_AND_ABOVE,
]:
noobaa_pod_node = get_pod_node(noobaa_pod)
if noobaa_pod_node is None:
assert False, "Could not find the NooBaa DB pod"
# Validate if RGW pod and noobaa-db are hosted on same node
# If not, make sure both pods are hosted on same node
log.info("Validate if RGW pod and noobaa-db are hosted on same node")
rgw_pod_obj = get_rgw_pods()
rgw_pod_node_list = [
rgw_pod.get().get("spec").get("nodeName")
for rgw_pod in rgw_pod_obj
]
if not list(
set(rgw_pod_node_list).intersection(
noobaa_pod_node.name.split())):
log.info("Unschedule other two nodes such that RGW "
"pod moves to node where NooBaa DB pod hosted")
worker_node_list = get_worker_nodes()
node_names = list(
set(worker_node_list) - set(noobaa_pod_node.name.split()))
unschedule_nodes(node_names=node_names)
ocp_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
rgw_pod_obj[0].delete()
ocp_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(rgw_pod_obj),
selector=constants.RGW_APP_LABEL,
timeout=300,
sleep=5,
)
log.info("Schedule those nodes again")
schedule_nodes(node_names=node_names)
# Check the ceph health OK
ceph_health_check(tries=90, delay=15)
# Verify all storage pods are running
wait_for_storage_pods()
# Check again the rgw pod move to node where NooBaa DB pod hosted
rgw_pod_obj_list = get_rgw_pods()
rgw_pod_node_list = [
get_pod_node(rgw_pod_obj) for rgw_pod_obj in rgw_pod_obj_list
]
value = [
True if rgw_pod_node == noobaa_pod_node.name else False
for rgw_pod_node in rgw_pod_node_list
]
assert value, ("RGW Pod didn't move to node where NooBaa DB pod"
" hosted even after cordoned and uncordoned nodes"
f"RGW pod hosted: {rgw_pod_node_list}"
f"NooBaa DB pod hosted: {noobaa_pod_node.name}")
log.info(
"RGW and noobaa-db are hosted on same node start the test execution"
)
rgw_pod_obj = get_rgw_pods()
for rgw_pod in rgw_pod_obj:
pod_node = rgw_pod.get().get("spec").get("nodeName")
if pod_node == noobaa_pod_node.name:
# Stop the node
log.info(f"Stopping node {pod_node} where"
f" rgw pod {rgw_pod.name} and NooBaa DB are hosted")
node_obj = get_node_objs(node_names=[pod_node])
nodes.stop_nodes(node_obj)
# Validate old rgw pod went terminating state
wait_for_resource_state(resource=rgw_pod,
state=constants.STATUS_TERMINATING,
timeout=720)
# Validate new rgw pod spun
ocp_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
ocp_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(rgw_pod_obj),
selector=constants.RGW_APP_LABEL,
)
# Start the node
nodes.start_nodes(node_obj)
# Check the ceph health OK
ceph_health_check(tries=90, delay=15)
# Verify all storage pods are running
wait_for_storage_pods()
# Create OBC and read wnd write
self.create_obc_creation(bucket_factory, mcg_obj,
"Object-key-2")
# Verify cluster health
self.sanity_helpers.health_check()
def test_noobaa_db_backup_and_recovery(
self,
pvc_factory,
pod_factory,
snapshot_factory,
bucket_factory,
rgw_bucket_factory,
):
"""
Test case to verify noobaa backup and recovery
1. Take snapshot db-noobaa-db-0 PVC and retore it to PVC
2. Scale down the statefulset noobaa-db
3. Get the yaml of the current PVC, db-noobaa-db-0 and
change the parameter persistentVolumeReclaimPolicy to Retain for restored PVC
4. Delete both PVCs, the PV for the original claim db-noobaa-db-0 will be removed.
The PV for claim db-noobaa-db-0-snapshot-restore will move to ‘Released’
5. Edit again restore PV and remove the claimRef section.
The volume will transition to Available.
6. Edit the yaml db-noobaa-db-0.yaml and change the setting volumeName to restored PVC.
7. Scale up the stateful set again and the pod should be running
"""
# Initialise variable
self.noobaa_db_sst_name = "noobaa-db-pg"
# Get noobaa pods before execution
noobaa_pods = get_noobaa_pods()
# Get noobaa PVC before execution
noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"])
noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get(
"volumeName")
# Take snapshot db-noobaa-db-0 PVC
log.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC")
snap_obj = snapshot_factory(
pvc_obj=noobaa_pvc_obj[0],
wait=True,
snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot",
)
log.info(
f"Successfully created snapshot {snap_obj.name} and in Ready state"
)
# Restore it to PVC
log.info(f"Restoring snapshot {snap_obj.name} to create new PVC")
sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName")
pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get(
"requests").get("storage"))
self.restore_pvc_obj = create_restore_pvc(
sc_name=sc_name,
snap_name=snap_obj.name,
namespace=snap_obj.namespace,
size=pvc_size,
pvc_name=f"{snap_obj.name}-restore",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
)
wait_for_resource_state(self.restore_pvc_obj, constants.STATUS_BOUND)
self.restore_pvc_obj.reload()
log.info(f"Succeesfuly created PVC {self.restore_pvc_obj.name} "
f"from snapshot {snap_obj.name}")
# Scale down the statefulset noobaa-db
modify_statefulset_replica_count(
statefulset_name=self.noobaa_db_sst_name, replica_count=0
), f"Failed to scale down the statefulset {self.noobaa_db_sst_name}"
# Get the noobaa-db PVC
pvc_obj = OCP(kind=constants.PVC,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name)
# Get the restored noobaa PVC and
# change the parameter persistentVolumeReclaimPolicy to Retain
restored_noobaa_pvc_obj = get_pvc_objs(
pvc_names=[f"{snap_obj.name}-restore"])
restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get(
"spec").get("volumeName"))
pv_obj = OCP(kind=constants.PV,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Retain {restored_noobaa_pv_name}")
# Delete both PVCs
delete_pvcs(pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]])
# Validate original claim db-noobaa-db-0 removed
assert validate_pv_delete(
pv_name=noobaa_pv_name
), f"PV not deleted, still exist {noobaa_pv_name}"
# Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state
pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED,
resource_name=restored_noobaa_pv_name)
# Edit again restore PV and remove the claimRef section
log.info(
f"Remove the claimRef section from PVC {restored_noobaa_pv_name}")
params = '[{"op": "remove", "path": "/spec/claimRef"}]'
pv_obj.patch(resource_name=restored_noobaa_pv_name,
params=params,
format_type="json")
log.info(
f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}"
)
# Validate PV is in Available state
pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE,
resource_name=restored_noobaa_pv_name)
# Edit the yaml db-noobaa-db-0.yaml and change the
# setting volumeName to restored PVC
noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name
noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml)
noobaa_pvc_yaml.create()
# Validate noobaa PVC is in bound state
pvc_obj.wait_for_resource(
condition=constants.STATUS_BOUND,
resource_name=noobaa_pvc_obj[0].name,
timeout=120,
)
# Scale up the statefulset again
assert modify_statefulset_replica_count(
statefulset_name=self.noobaa_db_sst_name, replica_count=1
), f"Failed to scale up the statefulset {self.noobaa_db_sst_name}"
# Validate noobaa pod is up and running
pod_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(noobaa_pods),
selector=constants.NOOBAA_APP_LABEL,
)
# Change the parameter persistentVolumeReclaimPolicy to Delete again
params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Delete {restored_noobaa_pv_name}")
log.info(
"Changed the parameter persistentVolumeReclaimPolicy to Delete again"
)
# Verify all storage pods are running
wait_for_storage_pods()
# Creating Resources
log.info("Creating Resources using sanity helpers")
self.sanity_helpers.create_resources(pvc_factory, pod_factory,
bucket_factory,
rgw_bucket_factory)
# Deleting Resources
self.sanity_helpers.delete_resources()
# Verify everything running fine
log.info(
"Verifying All resources are Running and matches expected result")
self.sanity_helpers.health_check(tries=120)
def factory(snapshot_factory=snapshot_factory):
# Get noobaa pods before execution
noobaa_pods = pod.get_noobaa_pods()
# Get noobaa PVC before execution
noobaa_pvc_obj = pvc.get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"])
noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get(
"volumeName")
# Take snapshot db-noobaa-db-0 PVC
logger.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC")
snap_obj = snapshot_factory(
pvc_obj=noobaa_pvc_obj[0],
wait=True,
snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot",
)
logger.info(
f"Successfully created snapshot {snap_obj.name} and in Ready state"
)
# Restore it to PVC
logger.info(f"Restoring snapshot {snap_obj.name} to create new PVC")
sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName")
pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get(
"requests").get("storage"))
restore_pvc_obj = pvc.create_restore_pvc(
sc_name=sc_name,
snap_name=snap_obj.name,
namespace=snap_obj.namespace,
size=pvc_size,
pvc_name=f"{snap_obj.name}-restore",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
)
restore_pvc_objs.append(restore_pvc_obj)
wait_for_resource_state(restore_pvc_obj, constants.STATUS_BOUND)
restore_pvc_obj.reload()
logger.info(f"Succeesfuly created PVC {restore_pvc_obj.name} "
f"from snapshot {snap_obj.name}")
# Scale down the statefulset noobaa-db
modify_statefulset_replica_count(
statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=0
), f"Failed to scale down the statefulset {constants.NOOBAA_DB_STATEFULSET}"
# Get the noobaa-db PVC
pvc_obj = OCP(kind=constants.PVC,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name)
# Get the restored noobaa PVC and
# change the parameter persistentVolumeReclaimPolicy to Retain
restored_noobaa_pvc_obj = pvc.get_pvc_objs(
pvc_names=[f"{snap_obj.name}-restore"])
restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get(
"spec").get("volumeName"))
pv_obj = OCP(kind=constants.PV,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Retain {restored_noobaa_pv_name}")
# Delete both PVCs
pvc.delete_pvcs(
pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]])
# Validate original claim db-noobaa-db-0 removed
assert validate_pv_delete(
pv_name=noobaa_pv_name
), f"PV not deleted, still exist {noobaa_pv_name}"
# Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state
pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED,
resource_name=restored_noobaa_pv_name)
# Edit again restore PV and remove the claimRef section
logger.info(
f"Remove the claimRef section from PVC {restored_noobaa_pv_name}")
params = '[{"op": "remove", "path": "/spec/claimRef"}]'
pv_obj.patch(resource_name=restored_noobaa_pv_name,
params=params,
format_type="json")
logger.info(
f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}"
)
# Validate PV is in Available state
pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE,
resource_name=restored_noobaa_pv_name)
# Edit the yaml db-noobaa-db-0.yaml and change the
# setting volumeName to restored PVC
noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name
noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml)
noobaa_pvc_yaml.create()
# Validate noobaa PVC is in bound state
pvc_obj.wait_for_resource(
condition=constants.STATUS_BOUND,
resource_name=noobaa_pvc_obj[0].name,
timeout=120,
)
# Scale up the statefulset again
assert modify_statefulset_replica_count(
statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=1
), f"Failed to scale up the statefulset {constants.NOOBAA_DB_STATEFULSET}"
# Validate noobaa pod is up and running
pod_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(noobaa_pods),
selector=constants.NOOBAA_APP_LABEL,
)
# Change the parameter persistentVolumeReclaimPolicy to Delete again
params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Delete {restored_noobaa_pv_name}")
logger.info(
"Changed the parameter persistentVolumeReclaimPolicy to Delete again"
)
