def test_reboot_standby_controller(no_simplex):
active, standby = system_helper.get_active_standby_controllers()
LOG.tc_step("'sudo reboot -f' from {}".format(standby))
host_helper.reboot_hosts(standby,
wait_for_offline=True,
wait_for_reboot_finish=True,
force_reboot=True)
system_helper.wait_for_hosts_states(standby,
timeout=360,
check_interval=30,
availability=['available'])
kube_helper.wait_for_pods_healthy(check_interval=30, all_namespaces=True)
def verify_():
LOG.fixture_step(
"({}) Verify system alarms, applications and pods status after "
"test {} ended...".format(scope, scope))
alarm_res, new_alarms = check_helper.check_alarms(
before_alarms=before_alarms, fail_ok=True)
post_apps_status = container_helper.get_apps(
field=('application', 'status'), application=prev_applied_apps)
LOG.info('prev bad pods: {}'.format(prev_bad_pods))
exclude = False
prev_name = None
if prev_bad_pods:
exclude = True
prev_name = prev_bad_pods
kube_helper.wait_for_pods_healthy(timeout=300,
all_namespaces=True,
name=prev_name,
exclude=exclude,
strict=True)
# check no new bad application
LOG.info("Check applications status...")
new_bad_apps = [
item for item in post_apps_status if item[1] != AppStatus.APPLIED
]
if new_bad_apps:
applying_apps = [
item[0] for item in post_apps_status
if item[1] == AppStatus.APPLYING
]
if applying_apps:
LOG.info('Wait for {} to apply'.format(applying_apps))
container_helper.wait_for_apps_status(apps=applying_apps,
status=AppStatus.APPLIED,
check_interval=20,
timeout=600)
new_bad_apps = [
item for item in new_bad_apps if item[0] not in applying_apps
]
assert not new_bad_apps, "Applications no longer applied after test " \
"{}: {}".format(scope, new_bad_apps)
# check no new alarms
assert alarm_res, "New alarm(s) appeared within test {}: {}".format(
scope, new_alarms)
def test_openstack_pods_healthy():
"""
Check the health of the OpenStack pods: healthy, running or in completed state
"""
application_status = container_helper.get_apps(
application="stx-openstack")[0]
assert application_status == "applied", "System status is not in state applied"
command_health = kube_helper.wait_for_pods_healthy(
namespace="stx-openstack")
assert command_health is None, "Check PODs health has failed"
def check_containers(prev_bad_pods, check_app):
host_helper._wait_for_openstack_cli_enable()
res_app = True
if check_app:
res_app = container_helper.wait_for_apps_status(apps='stx-openstack', status=AppStatus.APPLIED)[0]
prev_bad_pods = None if not prev_bad_pods else prev_bad_pods
res_pods = kube_helper.wait_for_pods_healthy(check_interval=10, name=prev_bad_pods, exclude=True,
all_namespaces=True)
assert (res_app and res_pods), "Application status or pods status check failed after modify and unlock host"
def test_kube_system_services():
"""
Check pods status and kube-system services are displayed.
"""
# Check PODs Health
command_health = kube_helper.wait_for_pods_healthy(
namespace="stx-openstack")
assert command_health is None, "Check PODs health has failed"
# Check Kube System Services
services_to_check = [
'ingress', 'ingress-error-pages', 'ingress-exporter', 'kube-dns',
'tiller-deploy'
]
services_to_check.sort()
services_list = kube_helper.get_resources(field="NAME",
namespace="kube-system",
resource_type="service")
services_list.sort()
for i in services_list:
i = ''.join(i)
assert services_to_check == services_list, "One or more services are missing from the list"
def test_detect_failed_compute(no_simplex, no_duplex):
con_ssh = ssh.ControllerClient.get_active_controller()
active_controller = system_helper.get_active_controller_name()
compute_host = system_helper.get_computes(
administrative=HostAdminState.UNLOCKED,
operational=HostOperState.ENABLED,
availability=HostAvailState.AVAILABLE)[0]
compute_su_prompt = r'.*compute\-([0-9]){1,}\:/home/sysadmin#'
cmd_get_offset = ("ntpq -p | grep {} -A1 | "
"tail -1 | awk '{{print$8}}'".format(active_controller))
cmd_magic_keys_enable = ("echo 1 > /proc/sys/kernel/sysrq")
cmd_get_start_date = ("python -c \"import datetime; "
"print str(datetime.datetime.now())[:-3]\"")
cmd_get_end_date = ("cat /var/log/mtcAgent.log | "
"grep \"{} MNFA new candidate\" | "
"tail -1 | awk '{{print$1}}'".format(compute_host))
cmd_trigger_reboot = ("echo b > /proc/sysrq-trigger")
res = list()
for i in range(20):
LOG.tc_step("Start of iter {}".format(i))
st = str()
offset = float()
with host_helper.ssh_to_host(compute_host) as node_ssh:
offset = float(
node_ssh.exec_cmd(cmd=cmd_get_offset,
get_exit_code=False)[1]) / 1000
node_ssh.send_sudo(cmd="su")
node_ssh.expect(compute_su_prompt)
node_ssh.send_sudo(cmd=cmd_magic_keys_enable)
node_ssh.expect(compute_su_prompt)
st = node_ssh.exec_cmd(cmd=cmd_get_start_date,
get_exit_code=False,
blob=compute_su_prompt)[1]
node_ssh.exec_sudo_cmd(cmd_trigger_reboot, get_exit_code=False)
system_helper.wait_for_hosts_states(
compute_host,
check_interval=20,
availability=HostAvailState.AVAILABLE)
pods_health = kube_helper.wait_for_pods_healthy(
check_interval=20, timeout=HostTimeout.REBOOT)
assert pods_health is True, "Check PODs health has failed"
st_date = datetime.datetime.fromtimestamp(
datetime.datetime.strptime(st, '%Y-%m-%d %H:%M:%S.%f').timestamp()
- offset)
et = con_ssh.exec_cmd(cmd=cmd_get_end_date, get_exit_code=False)[1]
et_date = datetime.datetime.strptime(et, '%Y-%m-%dT%H:%M:%S.%f')
diff = et_date - st_date
LOG.info("\noffset = {}\nstart time = {}\nend time = {}".format(
offset, st, et))
LOG.info("\ndiff = {}".format(diff))
res.append(diff)
def calc_avg(lst):
rtrn_sum = datetime.timedelta()
for i in lst:
LOG.info("Iter {}: {}".format(lst.index(i), i))
rtrn_sum += i
return rtrn_sum / len(lst)
final_res = calc_avg(res)
LOG.info("Avg time is : {}".format(final_res))
def check_openstack_pods_healthy(host, timeout):
with host_helper.ssh_to_host(hostname=host) as con_ssh:
kube_helper.wait_for_pods_healthy(namespace='stx-openstack',
con_ssh=con_ssh, timeout=timeout)