def install_non_active_node(node_name, lab): """ Install the non-active controller node, usually it is controller-1, the second controller on a non-AIO SX system. Args: node_name: - the name of the host/node, usually 'controller-1' lab: - lab to test """ boot_interfaces = lab['boot_device_dict'] LOG.tc_step("Restoring {}".format(node_name)) install_helper.open_vlm_console_thread(node_name, boot_interface=boot_interfaces, vlm_power_on=True) LOG.info( "Verifying {} is Locked, Disabled and Online ...".format(node_name)) system_helper.wait_for_hosts_states(node_name, administrative=HostAdminState.LOCKED, operational=HostOperState.DISABLED, availability=HostAvailState.ONLINE) LOG.info("Unlocking {} ...".format(node_name)) rc, output = host_helper.unlock_host(node_name, available_only=False) assert rc == 0 or rc == 4, "Host {} failed to unlock: rc = {}, msg: {}".format( node_name, rc, output) if rc == 4: LOG.warn('{} now is in degraded status'.format(node_name)) LOG.info('{} is installed'.format(node_name))
def test_unlock_hosts(): active = system_helper.get_active_controller_name() standby = 'controller-1' if active == 'controller-0' else 'controller-0' system_helper.wait_for_hosts_states([standby, 'compute-1'], availability='available') LOG.tc_step("Lock hosts.") host_helper.lock_host(standby) host_helper.lock_host('compute-1') LOG.tc_step("Unlock hosts") res = host_helper.unlock_hosts([standby, 'compute-1']) LOG.tc_step("Show results") LOG.info("Unlock hosts result: {}".format(res))
def test_reboot_standby_controller(no_simplex): active, standby = system_helper.get_active_standby_controllers() LOG.tc_step("'sudo reboot -f' from {}".format(standby)) host_helper.reboot_hosts(standby, wait_for_offline=True, wait_for_reboot_finish=True, force_reboot=True) system_helper.wait_for_hosts_states(standby, timeout=360, check_interval=30, availability=['available']) kube_helper.wait_for_pods_healthy(check_interval=30, all_namespaces=True)
def check_host_state(host, expected_host_state): """ Return the state that the host enters after the triggered event and configured sensor action.""" con_ssh = ControllerClient.get_active_controller() return system_helper.wait_for_hosts_states( host, timeout=90, check_interval=10, con_ssh=con_ssh, availability=['{}'.format(expected_host_state)])
def test_detect_failed_compute(no_simplex, no_duplex): con_ssh = ssh.ControllerClient.get_active_controller() active_controller = system_helper.get_active_controller_name() compute_host = system_helper.get_computes( administrative=HostAdminState.UNLOCKED, operational=HostOperState.ENABLED, availability=HostAvailState.AVAILABLE)[0] compute_su_prompt = r'.*compute\-([0-9]){1,}\:/home/sysadmin#' cmd_get_offset = ("ntpq -p | grep {} -A1 | " "tail -1 | awk '{{print$8}}'".format(active_controller)) cmd_magic_keys_enable = ("echo 1 > /proc/sys/kernel/sysrq") cmd_get_start_date = ("python -c \"import datetime; " "print str(datetime.datetime.now())[:-3]\"") cmd_get_end_date = ("cat /var/log/mtcAgent.log | " "grep \"{} MNFA new candidate\" | " "tail -1 | awk '{{print$1}}'".format(compute_host)) cmd_trigger_reboot = ("echo b > /proc/sysrq-trigger") res = list() for i in range(20): LOG.tc_step("Start of iter {}".format(i)) st = str() offset = float() with host_helper.ssh_to_host(compute_host) as node_ssh: offset = float( node_ssh.exec_cmd(cmd=cmd_get_offset, get_exit_code=False)[1]) / 1000 node_ssh.send_sudo(cmd="su") node_ssh.expect(compute_su_prompt) node_ssh.send_sudo(cmd=cmd_magic_keys_enable) node_ssh.expect(compute_su_prompt) st = node_ssh.exec_cmd(cmd=cmd_get_start_date, get_exit_code=False, blob=compute_su_prompt)[1] node_ssh.exec_sudo_cmd(cmd_trigger_reboot, get_exit_code=False) system_helper.wait_for_hosts_states( compute_host, check_interval=20, availability=HostAvailState.AVAILABLE) pods_health = kube_helper.wait_for_pods_healthy( check_interval=20, timeout=HostTimeout.REBOOT) assert pods_health is True, "Check PODs health has failed" st_date = datetime.datetime.fromtimestamp( datetime.datetime.strptime(st, '%Y-%m-%d %H:%M:%S.%f').timestamp() - offset) et = con_ssh.exec_cmd(cmd=cmd_get_end_date, get_exit_code=False)[1] et_date = datetime.datetime.strptime(et, '%Y-%m-%dT%H:%M:%S.%f') diff = et_date - st_date LOG.info("\noffset = {}\nstart time = {}\nend time = {}".format( offset, st, et)) LOG.info("\ndiff = {}".format(diff)) res.append(diff) def calc_avg(lst): rtrn_sum = datetime.timedelta() for i in lst: LOG.info("Iter {}: {}".format(lst.index(i), i)) rtrn_sum += i return rtrn_sum / len(lst) final_res = calc_avg(res) LOG.info("Avg time is : {}".format(final_res))
def test_install_cloned_image(install_clone_setup): controller1 = 'controller-1' lab = InstallVars.get_install_var('LAB') install_output_dir = ProjVar.get_var('LOG_DIR') controller0_node = lab['controller-0'] hostnames = install_clone_setup['hostnames'] system_mode = install_clone_setup['system_mode'] lab_name = lab['name'] LOG.info("Starting install-clone on AIO lab {} .... ".format(lab_name)) LOG.tc_step("Booting controller-0 ... ") if controller0_node.telnet_conn is None: controller0_node.telnet_conn = install_helper.open_telnet_session( controller0_node, install_output_dir) try: controller0_node.telnet_conn.login() except: LOG.info("Telnet Login failed. Attempting to reset password") try: controller0_node.telnet_conn.login(reset=True) except: if controller0_node.telnet_conn: controller0_node.telnet_conn.close() controller0_node.telnet_conn = None if controller0_node.telnet_conn: install_helper.wipe_disk_hosts(hostnames) # power off hosts LOG.tc_step("Powring off system hosts ... ") install_helper.power_off_host(hostnames) install_helper.boot_controller(boot_usb=True, small_footprint=True, clone_install=True) # establish telnet connection with controller LOG.tc_step( "Establishing telnet connection with controller-0 after install-clone ..." ) node_name_in_ini = '{}.*\~\$ '.format(controller0_node.host_name) normalized_name = re.sub(r'([^\d])0*(\d+)', r'\1\2', node_name_in_ini) # controller_prompt = Prompt.TIS_NODE_PROMPT_BASE.format(lab['name'].split('_')[0]) \ # + '|' + Prompt.CONTROLLER_0 \ # + '|{}'.format(node_name_in_ini) \ # + '|{}'.format(normalized_name) if controller0_node.telnet_conn: controller0_node.telnet_conn.close() output_dir = ProjVar.get_var('LOG_DIR') controller0_node.telnet_conn = install_helper.open_telnet_session( controller0_node, output_dir) controller0_node.telnet_conn.login() controller0_node.telnet_conn.exec_cmd("xterm") LOG.tc_step("Verify install-clone status ....") install_helper.check_clone_status( tel_net_session=controller0_node.telnet_conn) LOG.info("Source Keystone user admin environment ...") #controller0_node.telnet_conn.exec_cmd("cd; source /etc/platform/openrc") LOG.tc_step("Checking controller-0 hardware ....") install_helper.check_cloned_hardware_status('controller-0') if system_mode == 'duplex': LOG.tc_step("Booting controller-1 ... ") boot_interfaces = lab['boot_device_dict'] install_helper.open_vlm_console_thread('controller-1', boot_interface=boot_interfaces, vlm_power_on=True, wait_for_thread=True) LOG.info("waiting for {} to boot ...".format(controller1)) LOG.info("Verifying {} is Locked, Disabled and Online ...".format( controller1)) system_helper.wait_for_hosts_states( controller1, check_interval=20, use_telnet=True, con_telnet=controller0_node.telnet_conn, administrative=HostAdminState.LOCKED, operational=HostOperState.DISABLED, availability=HostAvailState.ONLINE) LOG.info("Unlocking {} ...".format(controller1)) rc, output = host_helper.unlock_host( controller1, use_telnet=True, con_telnet=controller0_node.telnet_conn) assert rc == 0, "Host {} unlock failed: {}".format(controller1, output) LOG.info("Host {} unlocked successfully ...".format(controller1)) LOG.info("Host controller-1 booted successfully... ") LOG.tc_step("Checking controller-1 hardware ....") install_helper.check_cloned_hardware_status(controller1) # LOG.tc_step("Customizing the cloned system ....") LOG.info("Changing the OAM IP configuration ... ") install_helper.update_oam_for_cloned_system(system_mode=system_mode) LOG.tc_step("Downloading lab specific license, config and scripts ....") software_version = system_helper.get_sw_version() load_path = BuildServerPath.LATEST_HOST_BUILD_PATHS[software_version] install_helper.download_lab_config_files( lab, install_clone_setup['build_server'], load_path) LOG.tc_step("Running lab cleanup to removed source attributes ....") install_helper.run_setup_script(script='lab_cleanup') LOG.tc_step( "Running lab setup script to upadate cloned system attributes ....") rc, output = install_helper.run_lab_setup() assert rc == 0, "Lab setup run failed: {}".format(output) time.sleep(30) LOG.tc_step( "Checking config status of controller-0 and perform lock/unlock if necessary..." ) if system_helper.get_host_values( 'controller-0', 'config_status')[0] == 'Config out-of-date': host_helper.lock_unlock_controllers() LOG.tc_step("Verifying system health after restore ...") system_helper.wait_for_all_alarms_gone(timeout=300) rc, failed = system_helper.get_system_health_query() assert rc == 0, "System health not OK: {}".format(failed)
def test_modify_mtu_oam_interface(mtu_range): """ of the 2016-04-04 sysinv_test_plan.pdf 20) Change the MTU value of the OAM interface using CLI Verify that MTU on oam interfaces on both standby and active controller can be modified by cli Args: mtu_range (str): A string that contain the mtu want to be tested Setup: - Nothing Test Steps: - lock standby controller - modify the imtu value of the controller - unlock the controller - revert and oam mtu of the controller and check system is still healthy - swact the controller - lock the controller - modify the imtu value of the controller - unlock the controller - check the controllers have expected mtu - revert the oam mtu of the controller and check system is still healthy Teardown: - Nothing """ is_sx = system_helper.is_aio_simplex() origin_active, origin_standby = system_helper.get_active_standby_controllers() if not origin_standby and not is_sx: skip("Standby controller unavailable. Cannot lock controller.") mtu = __get_mtu_to_mod(providernet_name='-ext', mtu_range=mtu_range) first_host = origin_active if is_sx else origin_standby max_mtu, cur_mtu, nic_name = get_max_allowed_mtus(host=first_host, network_type='oam') LOG.info('OK, the max MTU for {} is {}'.format(nic_name, max_mtu)) expecting_pass = not max_mtu or mtu <= max_mtu if not expecting_pass: LOG.warn('Expecting to fail in changing MTU: changing to:{}, max-mtu:{}'.format(mtu, max_mtu)) oam_attributes = host_helper.get_host_interfaces(host=first_host, field='attributes', name='oam', strict=False) # sample attributes: [MTU=9216,AE_MODE=802.3ad] pre_oam_mtu = int(oam_attributes[0].split(',')[0].split('=')[1]) is_stx_openstack_applied = container_helper.is_stx_openstack_deployed(applied_only=True) if not is_sx: HostsToRecover.add(origin_standby) prev_bad_pods = kube_helper.get_unhealthy_pods(all_namespaces=True) LOG.tc_step("Modify {} oam interface MTU from {} to {} on standby controller, and " "ensure it's applied successfully after unlock".format(origin_standby, pre_oam_mtu, mtu)) if mtu == cur_mtu: LOG.info('Setting to same MTU: from:{} to:{}'.format(mtu, cur_mtu)) code, res = host_helper.modify_mtu_on_interfaces(origin_standby, mtu_val=mtu, network_type='oam', lock_unlock=True, fail_ok=True) LOG.tc_step("Revert OAM MTU to original value: {}".format(pre_oam_mtu)) code_revert, res_revert = host_helper.modify_mtu_on_interfaces(origin_standby, mtu_val=pre_oam_mtu, network_type='oam', lock_unlock=True, fail_ok=True) if 0 == code: assert expecting_pass, "OAM MTU is not modified successfully. Result: {}".format(res) else: assert not expecting_pass, "OAM MTU WAS modified unexpectedly. Result: {}".format(res) assert 0 == code_revert, "OAM MTU is not reverted successfully. Result: {}".format(res_revert) LOG.tc_step("Check openstack cli, application and pods status after modify and revert {} oam mtu". format(origin_standby)) check_containers(prev_bad_pods, check_app=is_stx_openstack_applied) LOG.tc_step("Ensure standby controller is in available state and attempt to swact active controller to {}". format(origin_standby)) system_helper.wait_for_hosts_states(origin_active, availability=['available']) host_helper.swact_host(fail_ok=False) host_helper.wait_for_webservice_up(origin_standby) prev_bad_pods = kube_helper.get_unhealthy_pods(all_namespaces=True) HostsToRecover.add(origin_active) LOG.tc_step("Modify {} oam interface MTU to: {}, and " "ensure it's applied successfully after unlock".format(origin_active, mtu)) code, res = host_helper.modify_mtu_on_interfaces(origin_active, mtu_val=mtu, network_type='oam', lock_unlock=True, fail_ok=True) LOG.tc_step("Revert OAM MTU to original value: {}".format(pre_oam_mtu)) code_revert, res_revert = host_helper.modify_mtu_on_interfaces(origin_active, mtu_val=pre_oam_mtu, network_type='oam', lock_unlock=True, fail_ok=True) if 0 == code: assert expecting_pass, "OAM MTU is not modified successfully. Result: {}".format(res) else: assert not expecting_pass, "OAM MTU WAS modified unexpectedly. Result: {}".format(res) assert 0 == code_revert, "OAM MTU is not reverted successfully. Result: {}".format(res_revert) LOG.tc_step("Check openstack cli, application and pods after modify and revert {} oam mtu".format(origin_active)) check_containers(prev_bad_pods, check_app=is_stx_openstack_applied)
def test_restore(restore_setup): controller1 = 'controller-1' controller0 = 'controller-0' lab = restore_setup["lab"] is_aio_lab = lab.get('system_type', 'Standard') == 'CPE' is_sx = is_aio_lab and (len(lab['controller_nodes']) < 2) tis_backup_files = restore_setup['tis_backup_files'] backup_src = RestoreVars.get_restore_var('backup_src'.upper()) backup_src_path = RestoreVars.get_restore_var('backup_src_path'.upper()) controller_node = lab[controller0] con_ssh = ControllerClient.get_active_controller(name=lab['short_name'], fail_ok=True) sys_prompt = Prompt.TIS_NODE_PROMPT_BASE.format('.*' + lab['name'].split('_')[0]) controller_prompt = '{}|{}'.format(sys_prompt, Prompt.CONTROLLER_0) controller_node.telnet_conn.set_prompt(controller_prompt) if not con_ssh: LOG.info("Establish ssh connection with {}".format(controller0)) controller_node.ssh_conn = install_helper.ssh_to_controller( controller_node.host_ip, initial_prompt=controller_prompt) controller_node.ssh_conn.deploy_ssh_key() con_ssh = controller_node.ssh_conn ControllerClient.set_active_controller(con_ssh) LOG.info("Restore system from backup....") system_backup_file = [ file for file in tis_backup_files if "system.tgz" in file ].pop() images_backup_file = [ file for file in tis_backup_files if "images.tgz" in file ].pop() LOG.tc_step("Restoring {}".format(controller0)) LOG.info("System config restore from backup file {} ...".format( system_backup_file)) if backup_src.lower() == 'usb': system_backup_path = "{}/{}".format(BackupRestore.USB_BACKUP_PATH, system_backup_file) else: system_backup_path = "{}{}".format(HostLinuxUser.get_home(), system_backup_file) compute_configured = install_helper.restore_controller_system_config( system_backup=system_backup_path, is_aio=is_aio_lab)[2] # return LOG.info('re-connect to the active controller using ssh') con_ssh.close() controller_node.ssh_conn = install_helper.ssh_to_controller( controller_node.host_ip, initial_prompt=controller_prompt) LOG.info("Source Keystone user admin environment ...") LOG.info("set prompt to:{}, telnet_conn:{}".format( controller_prompt, controller_node.telnet_conn)) controller_node.telnet_conn.exec_cmd("cd; source /etc/platform/openrc") con_ssh = install_helper.ssh_to_controller(controller_node.host_ip) controller_node.ssh_conn = con_ssh ControllerClient.set_active_controller(con_ssh) make_sure_all_hosts_locked(con_ssh) if backup_src.lower() == 'local': images_backup_path = "{}{}".format(HostLinuxUser.get_home(), images_backup_file) common.scp_from_test_server_to_active_controller( "{}/{}".format(backup_src_path, images_backup_file), HostLinuxUser.get_home()) else: images_backup_path = "{}/{}".format(BackupRestore.USB_BACKUP_PATH, images_backup_file) LOG.info( "Images restore from backup file {} ...".format(images_backup_file)) new_prompt = r'{}.*~.*\$ |controller\-0.*~.*\$ '.format( lab['name'].split('_')[0]) LOG.info('set prompt to:{}'.format(new_prompt)) con_ssh.set_prompt(new_prompt) install_helper.restore_controller_system_images( images_backup=images_backup_path, tel_net_session=controller_node.telnet_conn) # this is a workaround for CGTS-8190 install_helper.update_auth_url(con_ssh) LOG.tc_step( "Verifying restoring controller-0 is complete and is in available state ..." ) LOG.debug('Wait for system ready in 60 seconds') time.sleep(60) timeout = HostTimeout.REBOOT + 60 availability = HostAvailState.AVAILABLE is_available = system_helper.wait_for_hosts_states( controller0, availability=HostAvailState.AVAILABLE, fail_ok=True, timeout=timeout) if not is_available: LOG.warn( 'After {} seconds, the first node:{} does NOT reach {}'.format( timeout, controller0, availability)) LOG.info('Check if drbd is still synchronizing data') con_ssh.exec_sudo_cmd('drbd-overview') is_degraded = system_helper.wait_for_hosts_states( controller0, availability=HostAvailState.DEGRADED, fail_ok=True, timeout=300) if is_degraded: LOG.warn('Node: {} is degraded: {}'.format( controller0, HostAvailState.DEGRADED)) con_ssh.exec_sudo_cmd('drbd-overview') else: LOG.fatal('Node:{} is NOT in Available nor Degraded status') # the customer doc does have wording regarding this situation, continue # assert False, 'Node:{} is NOT in Available nor Degraded status' # delete the system backup files from sysadmin home LOG.tc_step("Copying backup files to /opt/backups ... ") if backup_src.lower() == 'local': con_ssh.exec_cmd("rm -f {} {}".format(system_backup_path, images_backup_path)) cmd_rm_known_host = r'sed -i "s/^[^#]\(.*\)"/#\1/g /etc/ssh/ssh_known_hosts; \sync' con_ssh.exec_sudo_cmd(cmd_rm_known_host) # transfer all backup files to /opt/backups from test server with con_ssh.login_as_root(): con_ssh.scp_on_dest(source_user=TestFileServer.get_user(), source_ip=TestFileServer.get_server(), source_pswd=TestFileServer.get_password(), source_path=backup_src_path + "/*", dest_path=StxPath.BACKUPS + '/', timeout=1200) else: # copy all backupfiles from USB to /opt/backups cmd = " cp {}/* {}".format(BackupRestore.USB_BACKUP_PATH, StxPath.BACKUPS) con_ssh.exec_sudo_cmd(cmd, expect_timeout=600) LOG.tc_step("Checking if backup files are copied to /opt/backups ... ") assert int(con_ssh.exec_cmd("ls {} | wc -l".format(StxPath.BACKUPS))[1]) >= 2, \ "Missing backup files in {}".format(StxPath.BACKUPS) if is_aio_lab: LOG.tc_step("Restoring Cinder Volumes ...") restore_volumes() LOG.tc_step('Run restore-complete (CGTS-9756)') cmd = 'echo "{}" | sudo -S config_controller --restore-complete'.format( HostLinuxUser.get_password()) controller_node.telnet_conn.login() controller_node.telnet_conn.exec_cmd( cmd, extra_expects=[' will reboot on completion']) LOG.info('- wait untill reboot completes, ') time.sleep(120) LOG.info('- confirm the active controller is actually back online') controller_node.telnet_conn.login() LOG.tc_step( "reconnecting to the active controller after restore-complete") con_ssh = install_helper.ssh_to_controller(controller_node.host_ip) if not compute_configured: LOG.tc_step( 'Latest 18.07 EAR1 or Old-load on AIO/CPE lab: config its ' 'compute functionalities') # install_helper.run_cpe_compute_config_complete(controller_node, controller0) # LOG.info('closing current ssh connection') # con_ssh.close() LOG.tc_step('Run restore-complete (CGTS-9756)') controller_node.telnet_conn.login() cmd = 'echo "{}" | sudo -S config_controller --restore-complete'.\ format(HostLinuxUser.get_password()) controller_node.telnet_conn.exec_cmd(cmd, extra_expects=' will reboot ') controller_node.telnet_conn.close() LOG.info( 'Wait until "config_controller" reboot the active controller') time.sleep(180) controller_node.telnet_conn = install_helper.open_telnet_session( controller_node) controller_node.telnet_conn.login() time.sleep(120) con_ssh = install_helper.ssh_to_controller(controller_node.host_ip) controller_node.ssh_conn = con_ssh ControllerClient.set_active_controller(con_ssh) host_helper.wait_for_hosts_ready(controller0) LOG.tc_step('Install the standby controller: {}'.format(controller1)) if not is_sx: install_non_active_node(controller1, lab) elif len(lab['controller_nodes']) >= 2: LOG.tc_step('Install the standby controller: {}'.format(controller1)) install_non_active_node(controller1, lab) boot_interfaces = lab['boot_device_dict'] hostnames = system_helper.get_hosts() storage_hosts = [host for host in hostnames if 'storage' in host] compute_hosts = [ host for host in hostnames if 'storage' not in host and 'controller' not in host ] if len(storage_hosts) > 0: # con_ssh.exec_sudo_cmd('touch /etc/ceph/ceph.client.None.keyring') for storage_host in storage_hosts: LOG.tc_step("Restoring {}".format(storage_host)) install_helper.open_vlm_console_thread( storage_host, boot_interface=boot_interfaces, vlm_power_on=True) LOG.info( "Verifying {} is Locked, Diabled and Online ...".format( storage_host)) system_helper.wait_for_hosts_states( storage_host, administrative=HostAdminState.LOCKED, operational=HostOperState.DISABLED, availability=HostAvailState.ONLINE) LOG.info("Unlocking {} ...".format(storage_host)) rc, output = host_helper.unlock_host(storage_host, available_only=True) assert rc == 0, "Host {} failed to unlock: rc = {}, msg: {}".format( storage_host, rc, output) LOG.info("Veryifying the Ceph cluster is healthy ...") storage_helper.wait_for_ceph_health_ok(timeout=600) LOG.info("Importing images ...") image_backup_files = install_helper.get_backup_files( IMAGE_BACKUP_FILE_PATTERN, StxPath.BACKUPS, con_ssh) LOG.info("Image backup found: {}".format(image_backup_files)) imported = install_helper.import_image_from_backup( image_backup_files) LOG.info("Images successfully imported: {}".format(imported)) LOG.tc_step("Restoring Cinder Volumes ...") restore_volumes() LOG.tc_step('Run restore-complete (CGTS-9756), regular lab') controller_node.telnet_conn.login() cmd = 'echo "{}" | sudo -S config_controller --restore-complete'.format( HostLinuxUser.get_password()) controller_node.telnet_conn.exec_cmd( cmd, extra_expects='controller-0 login:'******'rebuild ssh connection') con_ssh = install_helper.ssh_to_controller(controller_node.host_ip) controller_node.ssh_conn = con_ssh LOG.tc_step("Restoring Compute Nodes ...") if len(compute_hosts) > 0: for compute_host in compute_hosts: LOG.tc_step("Restoring {}".format(compute_host)) install_helper.open_vlm_console_thread( compute_host, boot_interface=boot_interfaces, vlm_power_on=True) LOG.info( "Verifying {} is Locked, Diabled and Online ...".format( compute_host)) system_helper.wait_for_hosts_states( compute_host, administrative=HostAdminState.LOCKED, operational=HostOperState.DISABLED, availability=HostAvailState.ONLINE) LOG.info("Unlocking {} ...".format(compute_host)) rc, output = host_helper.unlock_host(compute_host, available_only=True) assert rc == 0, "Host {} failed to unlock: rc = {}, msg: {}".format( compute_host, rc, output) LOG.info("All nodes {} are restored ...".format(hostnames)) else: LOG.warn('Only 1 controller, but not AIO lab!!??') LOG.tc_step("Delete backup files from {} ....".format(StxPath.BACKUPS)) con_ssh.exec_sudo_cmd("rm -rf {}/*".format(StxPath.BACKUPS)) LOG.tc_step('Perform post-restore testing/checking') post_restore_test(con_ssh) LOG.tc_step("Waiting until all alarms are cleared ....") timeout = 300 healthy, alarms = system_helper.wait_for_all_alarms_gone(timeout=timeout, fail_ok=True) if not healthy: LOG.warn('Alarms exist: {}, after waiting {} seconds'.format( alarms, timeout)) rc, message = con_ssh.exec_sudo_cmd('drbd-overview') if rc != 0 or (r'[===>' not in message and r'] sync\'ed: ' not in message): LOG.warn('Failed to get drbd-overview information') LOG.info('Wait for the system to be ready in {} seconds'.format( HostTimeout.REBOOT)) system_helper.wait_for_all_alarms_gone(timeout=HostTimeout.REBOOT, fail_ok=False) LOG.tc_step("Verifying system health after restore ...") rc, failed = system_helper.get_system_health_query(con_ssh=con_ssh) assert rc == 0, "System health not OK: {}".format(failed) collect_logs()
def make_sure_all_hosts_locked(con_ssh, max_tries=5): """ Make sure all the hosts are locked before doing system restore. Args: con_ssh: - ssh connection to the target lab max_tries: - number of times to try before fail the entire test case when any hosts keep failing to lock. Return: None """ LOG.info('System restore procedure requires to lock all nodes except the ' 'active controller/controller-0') LOG.info('current host list before trying to lock them') cli.system('host-list') base_cmd = 'host-lock' locked_offline = { 'administrative': HostAdminState.LOCKED, 'availability': HostAvailState.OFFLINE } for tried in range(1, max_tries + 1): hosts = [ h for h in system_helper.get_hosts(administrative='unlocked', con_ssh=con_ssh) if h != 'controller-0' ] if not hosts: LOG.info( 'all hosts all locked except the controller-0 after tried:{}'. format(tried)) break cmd = base_cmd if tried > 1: cmd = base_cmd + ' -f' locking = [] already_locked = 0 for host in hosts: LOG.info('try:{} locking:{}'.format(tried, host)) admin_state = system_helper.get_host_values(host, 'administrative', con_ssh=con_ssh)[0] if admin_state != 'locked': code, output = cli.system(cmd + ' ' + host, ssh_client=con_ssh, fail_ok=True) if 0 != code: LOG.warn('Failed to lock host:{} using CLI:{}'.format( host, cmd)) else: locking.append(host) else: already_locked += 1 if locking: LOG.info( 'Wating for those accepted locking instructions to be locked: try:{}' .format(tried)) system_helper.wait_for_hosts_states(locking, con_ssh=con_ssh, timeout=600, **locked_offline) elif already_locked == len(hosts): LOG.info( 'all hosts all locked except the controller-0 after tried:{}'. format(tried)) break else: LOG.info('All hosts were rejecting to lock after tried:{}'.format( tried)) else: cli.system('host-list', ssh_client=con_ssh) LOG.info('Failed to lock or force-lock some of the hosts') assert False, 'Failed to lock or force-lock some of the hosts after tried:{} times'.\ format(max_tries) cli.system('host-list', ssh_client=con_ssh) code, output = cli.system('host-list', ssh_client=con_ssh, fail_ok=True) LOG.debug('code:{}, output:{}'.format(code, output))