def test_reboot_standby_controller(no_simplex): active, standby = system_helper.get_active_standby_controllers() LOG.tc_step("'sudo reboot -f' from {}".format(standby)) host_helper.reboot_hosts(standby, wait_for_offline=True, wait_for_reboot_finish=True, force_reboot=True) system_helper.wait_for_hosts_states(standby, timeout=360, check_interval=30, availability=['available']) kube_helper.wait_for_pods_healthy(check_interval=30, all_namespaces=True)
def verify_(): LOG.fixture_step( "({}) Verify system alarms, applications and pods status after " "test {} ended...".format(scope, scope)) alarm_res, new_alarms = check_helper.check_alarms( before_alarms=before_alarms, fail_ok=True) post_apps_status = container_helper.get_apps( field=('application', 'status'), application=prev_applied_apps) LOG.info('prev bad pods: {}'.format(prev_bad_pods)) exclude = False prev_name = None if prev_bad_pods: exclude = True prev_name = prev_bad_pods kube_helper.wait_for_pods_healthy(timeout=300, all_namespaces=True, name=prev_name, exclude=exclude, strict=True) # check no new bad application LOG.info("Check applications status...") new_bad_apps = [ item for item in post_apps_status if item[1] != AppStatus.APPLIED ] if new_bad_apps: applying_apps = [ item[0] for item in post_apps_status if item[1] == AppStatus.APPLYING ] if applying_apps: LOG.info('Wait for {} to apply'.format(applying_apps)) container_helper.wait_for_apps_status(apps=applying_apps, status=AppStatus.APPLIED, check_interval=20, timeout=600) new_bad_apps = [ item for item in new_bad_apps if item[0] not in applying_apps ] assert not new_bad_apps, "Applications no longer applied after test " \ "{}: {}".format(scope, new_bad_apps) # check no new alarms assert alarm_res, "New alarm(s) appeared within test {}: {}".format( scope, new_alarms)
def test_openstack_pods_healthy(): """ Check the health of the OpenStack pods: healthy, running or in completed state """ application_status = container_helper.get_apps( application="stx-openstack")[0] assert application_status == "applied", "System status is not in state applied" command_health = kube_helper.wait_for_pods_healthy( namespace="stx-openstack") assert command_health is None, "Check PODs health has failed"
def check_containers(prev_bad_pods, check_app): host_helper._wait_for_openstack_cli_enable() res_app = True if check_app: res_app = container_helper.wait_for_apps_status(apps='stx-openstack', status=AppStatus.APPLIED)[0] prev_bad_pods = None if not prev_bad_pods else prev_bad_pods res_pods = kube_helper.wait_for_pods_healthy(check_interval=10, name=prev_bad_pods, exclude=True, all_namespaces=True) assert (res_app and res_pods), "Application status or pods status check failed after modify and unlock host"
def test_kube_system_services(): """ Check pods status and kube-system services are displayed. """ # Check PODs Health command_health = kube_helper.wait_for_pods_healthy( namespace="stx-openstack") assert command_health is None, "Check PODs health has failed" # Check Kube System Services services_to_check = [ 'ingress', 'ingress-error-pages', 'ingress-exporter', 'kube-dns', 'tiller-deploy' ] services_to_check.sort() services_list = kube_helper.get_resources(field="NAME", namespace="kube-system", resource_type="service") services_list.sort() for i in services_list: i = ''.join(i) assert services_to_check == services_list, "One or more services are missing from the list"
def test_detect_failed_compute(no_simplex, no_duplex): con_ssh = ssh.ControllerClient.get_active_controller() active_controller = system_helper.get_active_controller_name() compute_host = system_helper.get_computes( administrative=HostAdminState.UNLOCKED, operational=HostOperState.ENABLED, availability=HostAvailState.AVAILABLE)[0] compute_su_prompt = r'.*compute\-([0-9]){1,}\:/home/sysadmin#' cmd_get_offset = ("ntpq -p | grep {} -A1 | " "tail -1 | awk '{{print$8}}'".format(active_controller)) cmd_magic_keys_enable = ("echo 1 > /proc/sys/kernel/sysrq") cmd_get_start_date = ("python -c \"import datetime; " "print str(datetime.datetime.now())[:-3]\"") cmd_get_end_date = ("cat /var/log/mtcAgent.log | " "grep \"{} MNFA new candidate\" | " "tail -1 | awk '{{print$1}}'".format(compute_host)) cmd_trigger_reboot = ("echo b > /proc/sysrq-trigger") res = list() for i in range(20): LOG.tc_step("Start of iter {}".format(i)) st = str() offset = float() with host_helper.ssh_to_host(compute_host) as node_ssh: offset = float( node_ssh.exec_cmd(cmd=cmd_get_offset, get_exit_code=False)[1]) / 1000 node_ssh.send_sudo(cmd="su") node_ssh.expect(compute_su_prompt) node_ssh.send_sudo(cmd=cmd_magic_keys_enable) node_ssh.expect(compute_su_prompt) st = node_ssh.exec_cmd(cmd=cmd_get_start_date, get_exit_code=False, blob=compute_su_prompt)[1] node_ssh.exec_sudo_cmd(cmd_trigger_reboot, get_exit_code=False) system_helper.wait_for_hosts_states( compute_host, check_interval=20, availability=HostAvailState.AVAILABLE) pods_health = kube_helper.wait_for_pods_healthy( check_interval=20, timeout=HostTimeout.REBOOT) assert pods_health is True, "Check PODs health has failed" st_date = datetime.datetime.fromtimestamp( datetime.datetime.strptime(st, '%Y-%m-%d %H:%M:%S.%f').timestamp() - offset) et = con_ssh.exec_cmd(cmd=cmd_get_end_date, get_exit_code=False)[1] et_date = datetime.datetime.strptime(et, '%Y-%m-%dT%H:%M:%S.%f') diff = et_date - st_date LOG.info("\noffset = {}\nstart time = {}\nend time = {}".format( offset, st, et)) LOG.info("\ndiff = {}".format(diff)) res.append(diff) def calc_avg(lst): rtrn_sum = datetime.timedelta() for i in lst: LOG.info("Iter {}: {}".format(lst.index(i), i)) rtrn_sum += i return rtrn_sum / len(lst) final_res = calc_avg(res) LOG.info("Avg time is : {}".format(final_res))
def check_openstack_pods_healthy(host, timeout): with host_helper.ssh_to_host(hostname=host) as con_ssh: kube_helper.wait_for_pods_healthy(namespace='stx-openstack', con_ssh=con_ssh, timeout=timeout)