def check_automated_recovery_from_drain_node(nodes):
"""
1) Drain one worker node.
2) Delete the OSD pods associated with the node.
3) The new OSD pods with the same ids that come up, should be in a Pending state.
4) Schedule the worker node.
5) The OSD pods associated with the node, should back into a Running state, and come up
on the same node.
"""
osd_node_name = random.choice(get_osd_running_nodes())
old_osd_pod_ids = get_node_osd_ids(osd_node_name)
log.info(f"osd pod ids: {old_osd_pod_ids}")
node_osd_pods = get_osd_pods_having_ids(old_osd_pod_ids)
unschedule_nodes([osd_node_name])
log.info(f"Successfully unschedule the node: {osd_node_name}")
log.info("Delete the node osd pods")
delete_pods(node_osd_pods)
new_osd_pods = wait_for_osd_pods_having_ids(osd_ids=old_osd_pod_ids)
new_osd_pod_names = [p.name for p in new_osd_pods]
wnodes = get_worker_nodes()
if len(wnodes) <= 3:
expected_pods_status = constants.STATUS_PENDING
else:
expected_pods_status = constants.STATUS_RUNNING
log.info(
f"Verify the new osd pods {new_osd_pod_names} go into a {expected_pods_status} state"
)
res = wait_for_pods_to_be_in_statuses(
[expected_pods_status],
new_osd_pod_names,
raise_pod_not_found_error=True,
)
assert res, f"Not all the node osd pods are in a {expected_pods_status} state"
log.info(f"Wait for the node: {osd_node_name} to be scheduled")
schedule_nodes([osd_node_name])
log.info(f"Successfully scheduled the node {osd_node_name}")
if len(wnodes) <= 3:
assert wait_for_osd_ids_come_up_on_node(osd_node_name, old_osd_pod_ids)
log.info(
f"the osd ids {old_osd_pod_ids} Successfully come up on the node {osd_node_name}"
)
def test_rgw_host_node_failure(self, nodes, node_restart_teardown,
node_drain_teardown, mcg_obj,
bucket_factory):
"""
Test case to fail node where RGW and the NooBaa DB are hosted
and verify the new pods spin on a healthy node
"""
# Get nooba pods
noobaa_pod_obj = get_noobaa_pods()
# Get the node where noobaa-db hosted
noobaa_pod_node = None
for noobaa_pod in noobaa_pod_obj:
if noobaa_pod.name in [
constants.NB_DB_NAME_46_AND_BELOW,
constants.NB_DB_NAME_47_AND_ABOVE,
]:
noobaa_pod_node = get_pod_node(noobaa_pod)
if noobaa_pod_node is None:
assert False, "Could not find the NooBaa DB pod"
# Validate if RGW pod and noobaa-db are hosted on same node
# If not, make sure both pods are hosted on same node
log.info("Validate if RGW pod and noobaa-db are hosted on same node")
rgw_pod_obj = get_rgw_pods()
rgw_pod_node_list = [
rgw_pod.get().get("spec").get("nodeName")
for rgw_pod in rgw_pod_obj
]
if not list(
set(rgw_pod_node_list).intersection(
noobaa_pod_node.name.split())):
log.info("Unschedule other two nodes such that RGW "
"pod moves to node where NooBaa DB pod hosted")
worker_node_list = get_worker_nodes()
node_names = list(
set(worker_node_list) - set(noobaa_pod_node.name.split()))
unschedule_nodes(node_names=node_names)
ocp_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
rgw_pod_obj[0].delete()
ocp_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(rgw_pod_obj),
selector=constants.RGW_APP_LABEL,
timeout=300,
sleep=5,
)
log.info("Schedule those nodes again")
schedule_nodes(node_names=node_names)
# Check the ceph health OK
ceph_health_check(tries=90, delay=15)
# Verify all storage pods are running
wait_for_storage_pods()
# Check again the rgw pod move to node where NooBaa DB pod hosted
rgw_pod_obj_list = get_rgw_pods()
rgw_pod_node_list = [
get_pod_node(rgw_pod_obj) for rgw_pod_obj in rgw_pod_obj_list
]
value = [
True if rgw_pod_node == noobaa_pod_node.name else False
for rgw_pod_node in rgw_pod_node_list
]
assert value, ("RGW Pod didn't move to node where NooBaa DB pod"
" hosted even after cordoned and uncordoned nodes"
f"RGW pod hosted: {rgw_pod_node_list}"
f"NooBaa DB pod hosted: {noobaa_pod_node.name}")
log.info(
"RGW and noobaa-db are hosted on same node start the test execution"
)
rgw_pod_obj = get_rgw_pods()
for rgw_pod in rgw_pod_obj:
pod_node = rgw_pod.get().get("spec").get("nodeName")
if pod_node == noobaa_pod_node.name:
# Stop the node
log.info(f"Stopping node {pod_node} where"
f" rgw pod {rgw_pod.name} and NooBaa DB are hosted")
node_obj = get_node_objs(node_names=[pod_node])
nodes.stop_nodes(node_obj)
# Validate old rgw pod went terminating state
wait_for_resource_state(resource=rgw_pod,
state=constants.STATUS_TERMINATING,
timeout=720)
# Validate new rgw pod spun
ocp_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
ocp_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(rgw_pod_obj),
selector=constants.RGW_APP_LABEL,
)
# Start the node
nodes.start_nodes(node_obj)
# Check the ceph health OK
ceph_health_check(tries=90, delay=15)
# Verify all storage pods are running
wait_for_storage_pods()
# Create OBC and read wnd write
self.create_obc_creation(bucket_factory, mcg_obj,
"Object-key-2")
# Verify cluster health
self.sanity_helpers.health_check()
def measure_stop_ceph_mon(measurement_dir, create_mon_quorum_loss, threading_lock):
"""
Downscales Ceph Monitor deployment, measures the time when it was
downscaled and monitors alerts that were triggered during this event.
Returns:
dict: Contains information about `start` and `stop` time for stopping
Ceph Monitor pod
"""
oc = ocp.OCP(
kind=constants.DEPLOYMENT,
namespace=config.ENV_DATA["cluster_namespace"],
threading_lock=threading_lock,
)
mon_deployments = oc.get(selector=constants.MON_APP_LABEL)["items"]
mons = [deployment["metadata"]["name"] for deployment in mon_deployments]
# get monitor deployments to stop,
# if mon quorum to be lost split_index will be 1
# else leave even number of monitors
split_index = (
1 if create_mon_quorum_loss else len(mons) // 2 if len(mons) > 3 else 2
)
mons_to_stop = mons[split_index:]
logger.info(f"Monitors to stop: {mons_to_stop}")
logger.info(f"Monitors left to run: {mons[:split_index]}")
# run_time of operation
run_time = 60 * 14
def stop_mon():
"""
Downscale Ceph Monitor deployments for 14 minutes. First 15 minutes
the alert CephMonQuorumAtRisk should be in 'Pending'. After 15 minutes
the alert turns into 'Firing' state.
This configuration of monitoring can be observed in ceph-mixins which
are used in the project:
https://github.com/ceph/ceph-mixins/blob/d22afe8c0da34490cb77e52a202eefcf4f62a869/config.libsonnet#L16
`Firing` state shouldn't actually happen because monitor should be
automatically redeployed shortly after 10 minutes.
Returns:
str: Names of downscaled deployments
"""
nonlocal oc
nonlocal mons_to_stop
for mon in mons_to_stop:
logger.info(f"Downscaling deployment {mon} to 0")
oc.exec_oc_cmd(f"scale --replicas=0 deployment/{mon}")
logger.info(f"Waiting for {run_time} seconds")
time.sleep(run_time)
return mons_to_stop
test_file = os.path.join(
measurement_dir, f"measure_stop_ceph_mon_{split_index}.json"
)
if config.ENV_DATA["platform"].lower() in constants.MANAGED_SERVICE_PLATFORMS:
# It seems that it takes longer to propagate incidents to PagerDuty.
# Adding 6 extra minutes so that alert is actually triggered and
# unscheduling worker nodes so that monitor is not replaced
worker_node_names = [
node.name for node in get_nodes(node_type=constants.WORKER_MACHINE)
]
unschedule_nodes(worker_node_names)
measured_op = measure_operation(stop_mon, test_file, minimal_time=60 * 20)
schedule_nodes(worker_node_names)
else:
measured_op = measure_operation(stop_mon, test_file)
# expected minimal downtime of a mon inflicted by this fixture
measured_op["min_downtime"] = run_time - (60 * 2)
# get new list of monitors to make sure that new monitors were deployed
mon_deployments = oc.get(selector=constants.MON_APP_LABEL)["items"]
mons = [deployment["metadata"]["name"] for deployment in mon_deployments]
# check that downscaled monitors are removed as OCS should redeploy them
# but only when we are running this for the first time
check_old_mons_deleted = all(mon not in mons for mon in mons_to_stop)
if measured_op["first_run"] and not check_old_mons_deleted:
for mon in mons_to_stop:
logger.info(f"Upscaling deployment {mon} back to 1")
oc.exec_oc_cmd(f"scale --replicas=1 deployment/{mon}")
if (
not split_index == 1
and config.ENV_DATA["platform"].lower()
not in constants.MANAGED_SERVICE_PLATFORMS
):
msg = f"Downscaled monitors {mons_to_stop} were not replaced"
assert check_old_mons_deleted, msg
# wait for ceph to return into HEALTH_OK state after mon deployment
# is returned back to normal
ceph_health_check(tries=20, delay=15)
return measured_op
def test_nodereplacement_proactive(self, pvc_factory, pod_factory,
dc_pod_factory):
"""
Knip-894 Node Replacement proactive
"""
# Get worker nodes
worker_node_list = get_worker_nodes()
log.info(f"Current available worker nodes are {worker_node_list}")
osd_pods_obj = pod.get_osd_pods()
osd_node_name = pod.get_pod_node(random.choice(osd_pods_obj)).name
log.info(f"Selected OSD is {osd_node_name}")
log.info("Creating dc pod backed with rbd pvc and running io in bg")
for worker_node in worker_node_list:
if worker_node != osd_node_name:
rbd_dc_pod = dc_pod_factory(interface=constants.CEPHBLOCKPOOL,
node_name=worker_node,
size=20)
pod.run_io_in_bg(rbd_dc_pod,
expect_to_fail=False,
fedora_dc=True)
log.info("Creating dc pod backed with cephfs pvc and running io in bg")
for worker_node in worker_node_list:
if worker_node != osd_node_name:
cephfs_dc_pod = dc_pod_factory(
interface=constants.CEPHFILESYSTEM,
node_name=worker_node,
size=20)
pod.run_io_in_bg(cephfs_dc_pod,
expect_to_fail=False,
fedora_dc=True)
# Unscheduling node
node.unschedule_nodes([osd_node_name])
# Draining Node
node.drain_nodes([osd_node_name])
log.info("Getting machine name from specified node name")
machine_name = machine.get_machine_from_node_name(osd_node_name)
log.info(f"Node {osd_node_name} associated machine is {machine_name}")
log.info(
f"Deleting machine {machine_name} and waiting for new machine to come up"
)
machine.delete_machine_and_check_state_of_new_spinned_machine(
machine_name)
new_machine_list = machine.get_machines()
for machines in new_machine_list:
# Trimming is done to get just machine name
# eg:- machine_name:- prsurve-40-ocs-43-kbrvf-worker-us-east-2b-nlgkr
# After trimming:- prsurve-40-ocs-43-kbrvf-worker-us-east-2b
if re.match(machines.name[:-6], machine_name):
new_machine_name = machines.name
machineset_name = machine.get_machineset_from_machine_name(
new_machine_name)
log.info("Waiting for new worker node to be in ready state")
machine.wait_for_new_node_to_be_ready(machineset_name)
new_node_name = node.get_node_from_machine_name(new_machine_name)
log.info("Adding ocs label to newly created worker node")
node_obj = ocp.OCP(kind='node')
node_obj.add_label(resource_name=new_node_name,
label=constants.OPERATOR_NODE_LABEL)
log.info(
f"Successfully labeled {new_node_name} with OCS storage label")
# Creating Resources
log.info("Creating Resources using sanity helpers")
self.sanity_helpers.create_resources(pvc_factory, pod_factory)
# Deleting Resources
self.sanity_helpers.delete_resources()
# Verify everything running fine
log.info(
"Verifying All resources are Running and matches expected result")
self.sanity_helpers.health_check()