def test_upload_charts_via_helm_upload(copy_test_apps): """ Test upload helm charts via helm-upload cmd directly. i.e., without using sysinv cmd. Args: copy_test_apps: Setups: - Copy test files from test server to tis system (module) Test Steps: - Upload helm charts from given controller via 'helm-upload <tar_file>' - Verify the charts appear at /www/pages/helm_charts/ on both controllers (if applicable) """ app_dir = copy_test_apps LOG.tc_step( "Upload helm charts via helm-upload cmd from active controller " "and check charts are in /www/pages/") file_path = container_helper.upload_helm_charts(tar_file=os.path.join( app_dir, HELM_TAR), delete_first=True)[1] if system_helper.get_standby_controller_name(): LOG.tc_step("Swact active controller and verify uploaded charts " "are synced over") host_helper.swact_host() con_ssh = ControllerClient.get_active_controller() charts_exist = con_ssh.file_exists(file_path) assert charts_exist, "{} does not exist after swact to {}".format( file_path, con_ssh.get_hostname()) LOG.info("{} successfully synced after swact".format(file_path))
def test_swact_controller_platform(wait_for_con_drbd_sync_complete): """ Verify swact active controller Test Steps: - Swact active controller - Verify standby controller and active controller are swapped - Verify nodes are ready in kubectl get nodes """ if system_helper.is_aio_simplex(): skip("Simplex system detected") if not wait_for_con_drbd_sync_complete: skip(SkipSysType.LESS_THAN_TWO_CONTROLLERS) LOG.tc_step('retrieve active and available controllers') pre_active_controller, pre_standby_controller = \ system_helper.get_active_standby_controllers() assert pre_standby_controller, "No standby controller available" LOG.tc_step("Swact active controller and ensure active controller " "is changed") host_helper.swact_host(hostname=pre_active_controller) LOG.tc_step("Check hosts are Ready in kubectl get nodes after swact") kube_helper.wait_for_nodes_ready(hosts=(pre_active_controller, pre_standby_controller), timeout=30)
def test_swact_host(hostname, timeout, fail_ok): LOG.tc_step("wait for previous swact complete") host_helper._wait_for_openstack_cli_enable() system_helper.wait_for_host_values('controller-0', timeout=60, fail_ok=False, task='') system_helper.wait_for_host_values('controller-1', timeout=60, fail_ok=False, task='') LOG.tc_step("swact host") if fail_ok: code, msg = host_helper.swact_host(hostname=hostname, swact_start_timeout=timeout, fail_ok=fail_ok) if timeout == 1: assert code == 3 host_helper.wait_for_swact_complete(hostname, fail_ok=False) else: assert code in [-1, 0, 1, 2] else: if timeout == 1: with raises(exceptions.HostPostCheckFailed): host_helper.swact_host(hostname=hostname, swact_start_timeout=1, fail_ok=False) host_helper.wait_for_swact_complete(hostname, fail_ok=False) else: host_helper.swact_host(hostname=hostname, swact_start_timeout=timeout, fail_ok=False)
def remove(): LOG.fixture_step("Removing custom firewall rules") user_file_dir = ProjVar.get_var('USER_FILE_DIR') empty_path = user_file_dir + "iptables-empty.rules" client = get_cli_client(central_region=True) client.exec_cmd('touch {}'.format(empty_path)) _modify_firewall_rules(empty_path) active, standby = system_helper.get_active_standby_controllers() con_ssh = ControllerClient.get_active_controller() LOG.fixture_step("Verify custom ports on {}".format(active)) for port in custom_ports: # Verifying ports that are in the iptables file are closed _verify_port_from_natbox(con_ssh, port, port_expected_open=False) if standby: LOG.fixture_step("Swact {}".format(active)) host_helper.swact_host(active) LOG.fixture_step("Verify custom ports on {}".format(standby)) for port in custom_ports: # Verifying ports that are in the iptables file are closed after swact _verify_port_from_natbox(con_ssh, port, port_expected_open=False)
def test_swact_controller_host(): """ SWACT Controller host - it must fail on simplex """ active_controller_host = system_helper.get_active_controller_name() LOG.info( "Active Controller Before SWACT: {}".format(active_controller_host)) standby_controller_host = system_helper.get_standby_controller_name() LOG.info( "Standby Controller Before SWACT: {}".format(standby_controller_host)) # On simplex swact must fail host_helper.swact_host(fail_ok=system_helper.is_aio_simplex()) # host_helper.wait_for_swact_complete(before_host=active_controller_host) active_controller_host = system_helper.get_active_controller_name() LOG.info( "Active Controller After SWACT: {}".format(active_controller_host)) standby_controller_host = system_helper.get_standby_controller_name() LOG.info( "Standby Controller After SWACT: {}".format(standby_controller_host)) # Re-SWACT only if duplex if not system_helper.is_aio_simplex(): host_helper.swact_host()
def test_heat_vm_scale_after_actions(vm_scaling_stack, actions): """ Test VM auto scaling with swact: Create heat stack for auto scaling using NestedAutoScale.yaml, swact and perform vm scale up and down. Test Steps: - Create a heat stack for auto scaling vm () - Verify heat stack is created successfully - Verify heat resources are created - live migrate the vm if not sx - cold migrate the vm if not sx - swact if not sx - reboot -f vm host - trigger auto scale by boosting cpu usage in the vm (using dd) - verify it scale up to the max number of vms (3) - trigger scale down by killing dd in the vm - verify the vm scale down to min number (1) - Delete Heat stack and verify resource deletion """ stack_name, vm_id = vm_scaling_stack vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id) if not system_helper.is_aio_simplex(): actions = actions.split('-') if "swact" in actions: LOG.tc_step("Swact before scale in/out") host_helper.swact_host() if "live_migrate" in actions: LOG.tc_step("live migrate vm before scale in/out") vm_helper.live_migrate_vm(vm_id) if "cold_migrate" in actions: LOG.tc_step("cold migrate vm before scale in/out") vm_helper.cold_migrate_vm(vm_id) if "host_reboot" in actions: if system_helper.is_aio_simplex(): host_helper.reboot_hosts('controller-0') vm_helper.wait_for_vm_status(vm_id, status=VMStatus.ACTIVE, timeout=600, check_interval=10, fail_ok=False) vm_helper.wait_for_vm_pingable_from_natbox( vm_id, timeout=VMTimeout.DHCP_RETRY) else: LOG.tc_step("evacuate vm before scale in/out") vm_host = vm_helper.get_vm_host(vm_id=vm_id) vm_helper.evacuate_vms(host=vm_host, vms_to_check=vm_id) LOG.tc_step( "Wait for {} vms to auto scale out to {} after running dd in vm(s)". format(stack_name, 3)) vm_helper.wait_for_auto_vm_scale_out(stack_name, expt_max=3) LOG.tc_step( "Wait for {} vms to auto scale in to {} after killing dd processes in vms" .format(stack_name, 1)) vm_helper.wait_for_auto_vm_scale_in(stack_name, expt_min=1)
def test_lock_with_vms_mig_fail(self, target_hosts_negative): """ Test lock host with vms on it - Negative test. i.e., lock should be rejected Args: target_hosts_negative: target host(s) to perform lock Prerequisites: hosts storage backing are pre-configured to storage backing under test. ie., only 1 host should support the storage backing under test. Test Setups: - Set instances quota to 10 if it was less than 8 - Determine storage backing(s) under test, i.e., storage backings supported by only 1 host on the system - Create flavors with storage extra specs set based on storage backings under test - Create vms_to_test that can be live migrated using created flavors - Determine target host(s) to perform lock based on which host(s) have the most vms_to_test Test Steps: - Lock target host - Verify lock rejected and vms status unchanged - Repeat above steps if more than one target host Test Teardown: - Delete created vms and volumes - Delete created flavors - Unlock locked target host(s) """ target_hosts, storages_to_test = target_hosts_negative LOG.info( "Negative test: host-lock attempt on host(s) with {} storage backing(s). \n" "Host(s) to attempt lock: {}".format(storages_to_test, target_hosts_negative)) vms_per_host = vm_helper.get_vms_per_host() for host in target_hosts: if system_helper.get_active_controller_name() == host: host_helper.swact_host(hostname=host) host_helper.wait_for_hypervisors_up(host) host_helper.wait_for_webservice_up(host) vms_on_host = vms_per_host[host] pre_vms_status = vm_helper.get_vms_info(vms=vms_on_host, fields='Status') LOG.tc_step("Lock target host {}...".format(host)) lock_code, lock_output = host_helper.lock_host(host=host, check_first=False, fail_ok=True, swact=True) # Add locked host to cleanup list if lock_code in [0, 3]: self.hosts_locked.append(host) post_vms_status = vm_helper.get_vms_info(vms=vms_on_host, fields='Status') LOG.tc_step("Verify lock rejected and vms status unchanged.") assert lock_code in [1, 2, 4, 5 ], "Unexpected result: {}".format(lock_output) assert pre_vms_status == post_vms_status, "VM(s) status has changed after host-lock {}".format( host)
def test_enable_tpm(swact_first): con_ssh = ControllerClient.get_active_controller() LOG.tc_step('Check if TPM is already configured') code, cert_id, cert_type = get_tpm_status(con_ssh) if code == 0: LOG.info('TPM already configured on the lab, cert_id:{}, cert_type:{}'. format(cert_id, cert_type)) LOG.tc_step('disable TPM first in order to test enabling TPM') code, output = remove_cert_from_tpm(con_ssh, fail_ok=False, check_first=False) assert 0 == code, 'failed to disable TPM' time.sleep(30) LOG.info('Waiting alarm: out-of-config cleaned up') system_helper.wait_for_alarm_gone(EventLogID.CONFIG_OUT_OF_DATE) else: LOG.info('TPM is NOT configured on the lab') LOG.info('-code:{}, cert_id:{}, cert_type:{}'.format( code, cert_id, cert_type)) if swact_first: LOG.tc_step('Swact the active controller as instructed') if len(system_helper.get_controllers()) < 2: LOG.info('Less than 2 controllers, skip swact') else: host_helper.swact_host(fail_ok=False) copy_config_from_local( con_ssh, local_conf_backup_dir, os.path.join(HostLinuxUser.get_home(), conf_backup_dir)) LOG.tc_step('Install HTTPS Certificate into TPM') code, output = store_cert_into_tpm( con_ssh, check_first=False, fail_ok=False, pem_password=HostLinuxUser.get_password()) assert 0 == code, 'Failed to instll certificate into TPM, cert-file' LOG.info('OK, certificate is installed into TPM') LOG.info('Wait the out-of-config alarm cleared') system_helper.wait_for_alarm_gone(EventLogID.CONFIG_OUT_OF_DATE) LOG.tc_step( 'Verify the configurations changes for impacted components, expecting all changes exit' ) verify_configuration_changes(expected=True, connection=con_ssh)
def upgrade_host_lock_unlock(host, con_ssh=None): """ swact, if required, lock and unlock before upgrade. Args: host (str): hostname or id in string format con_ssh (SSHClient): Returns: (return_code(int), msg(str)) (0, "Host is host is locked/unlocked) """ LOG.info("Checking if host {} is active ....".format(host)) active_controller = system_helper.get_active_controller_name() swact_back = False if active_controller == host: LOG.tc_step("Swact active controller and ensure active controller is changed") exit_code, output = host_helper.swact_host(hostname=active_controller) assert 0 == exit_code, "{} is not recognized as active controller".format(active_controller) active_controller = system_helper.get_active_controller_name() swact_back = True LOG.info("Host {}; doing lock/unlock to the host ....".format(host)) rc, output = host_helper.lock_host(host, con_ssh=con_ssh) if rc != 0 and rc != -1: err_msg = "Lock host {} rejected".format(host) LOG.warn(err_msg) return 1, err_msg rc, output = host_helper.unlock_host(host, available_only=True, con_ssh=con_ssh) if rc != 0: err_msg = "Unlock host {} failed: {}".format(host, output) return 1, err_msg if swact_back: time.sleep(60) if not system_helper.wait_for_host_values(host, timeout=360, fail_ok=True, operational=HostOperState.ENABLED, availability=HostAvailState.AVAILABLE): err_msg = " Swacting to standby is not possible because {} is not in available state " \ "within the specified timeout".format(host) assert False, err_msg LOG.tc_step("Swact active controller back and ensure active controller is changed") rc, output = host_helper.swact_host(hostname=active_controller) if rc != 0: err_msg = "Failed to swact back to host {}: {}".format(host, output) return 1, err_msg LOG.info("Swacted and {} has become active......".format(host)) return 0, "Host {} is locked and unlocked successfully".format(host)
def test_reboot_active_controller(no_simplex): active, standby = system_helper.get_active_standby_controllers() LOG.tc_step("'sudo reboot -f' from {}".format(active)) host_helper.reboot_hosts(active, wait_for_offline=True, wait_for_reboot_finish=True, force_reboot=True) system_helper.wait_for_hosts_states(standby, timeout=360, check_interval=30, availability=['available']) kube_helper.wait_for_pods_healthy(check_interval=30, all_namespaces=True) host_helper.swact_host(hostname=standby)
def _test_firewall_rules_custom(remove_custom_firewall): """ Verify specified ports from the custom firewall rules are open and non-specified ports are closed. Skip Condition: - N/A Test Setup: - SCP iptables.rules from test server to lab Test Steps: - Install custom firewall rules - Check ports that should be both open and closed based on the custom firewall rules - Swact and check ports that should be both open and closed based on the custom firewall rules - Remove custom firewall rules - Check ports that are in the custom firewall rules are no longer open - Swact and check ports that are in the custom firewall rules are no longer open """ # The following ports must be in the iptables.rules file or the test will fail custom_ports, firewall_rules_path = remove_custom_firewall LOG.tc_step("Installing custom firewall rules") _modify_firewall_rules(firewall_rules_path) active_controller, standby_controller = system_helper.get_active_standby_controllers( ) con_ssh = ControllerClient.get_active_controller() LOG.tc_step("Verify custom ports on {}".format(active_controller)) for port in custom_ports: # Verifying ports that are in the iptables file are open _verify_port_from_natbox(con_ssh, port, port_expected_open=True) # Verifying ports that are not in the iptables file are still closed _verify_port_from_natbox(con_ssh, port + 1, port_expected_open=False) if standby_controller: LOG.tc_step("Swact {}".format(active_controller)) host_helper.swact_host(active_controller) active_controller = system_helper.get_active_controller_name() con_ssh = ControllerClient.get_active_controller() LOG.tc_step("Verify custom ports on {}".format(active_controller)) for port in custom_ports: # Verifying ports that are in the iptables file are open after swact _verify_port_from_natbox(con_ssh, port, port_expected_open=True) # Verifying ports that are not in the iptables file are still closed after swact _verify_port_from_natbox(con_ssh, port + 1, port_expected_open=False)
def test_swact_controller_platform(wait_for_con_drbd_sync_complete, collect_kpi): """ Verify swact active controller Test Steps: - Swact active controller - Verify standby controller and active controller are swapped - Verify nodes are ready in kubectl get nodes """ if system_helper.is_aio_simplex(): skip("Simplex system detected") if not wait_for_con_drbd_sync_complete: skip(SkipSysType.LESS_THAN_TWO_CONTROLLERS) LOG.tc_step('retrieve active and available controllers') pre_active_controller, pre_standby_controller = system_helper.get_active_standby_controllers( ) assert pre_standby_controller, "No standby controller available" collect_kpi = None if container_helper.is_stx_openstack_deployed( ) else collect_kpi init_time = None if collect_kpi: init_time = common.get_date_in_format(date_format=KPI_DATE_FORMAT) LOG.tc_step( "Swact active controller and ensure active controller is changed") host_helper.swact_host(hostname=pre_active_controller) LOG.tc_step("Check hosts are Ready in kubectl get nodes after swact") kube_helper.wait_for_nodes_ready(hosts=(pre_active_controller, pre_standby_controller), timeout=30) if collect_kpi: kpi_name = SwactPlatform.NAME kpi_log_parser.record_kpi(local_kpi_file=collect_kpi, kpi_name=kpi_name, init_time=init_time, log_path=SwactPlatform.LOG_PATH, end_pattern=SwactPlatform.END, host=pre_standby_controller, start_host=pre_active_controller, start_pattern=SwactPlatform.START, start_path=SwactPlatform.START_PATH, uptime=1, fail_ok=False)
def test_modify_timezone_sys_event_timestamp(): """ Test alarm timestamps line up with a timezone change Prerequisites - N/A Test Setups - Get a random timezone for testing Test Steps - Get the UUID and timestamp from the most recent event - Change the timezone and wait until the change is complete - Wait for out_of_date alarms to clear - Compare the timestamp from the event using the UUID - Verify the timestamp changed with the timezone change Test Teardown - N/A """ LOG.tc_step("Gathering pre-modify timestamp for last event in system event-list") event = system_helper.get_events(fields=('UUID', 'Event Log ID', 'Entity Instance ID', 'State', 'Time Stamp'), limit=1, combine_entries=False)[0] event_uuid, event_log_id, entity_instance_id, event_state, pre_timestamp = event # Pick a random timezone to test that is not the current timezone timezone_to_test = __select_diff_timezone() LOG.tc_step("Modify timezone to {}".format(timezone_to_test)) system_helper.modify_timezone(timezone=timezone_to_test) LOG.tc_step("Waiting for timezone for previous event to change in system event-list") timeout = time.time() + 60 post_timestamp = None while time.time() < timeout: post_timestamp = system_helper.get_events(fields='Time Stamp', event_id=event_log_id, uuid=event_uuid, entity_id=entity_instance_id, state=event_state)[0][0] if pre_timestamp != post_timestamp: break time.sleep(5) else: assert pre_timestamp != post_timestamp, "Timestamp did not change with timezone change." if not system_helper.is_aio_simplex(): LOG.tc_step("Swact and verify timezone persists") host_helper.swact_host() post_swact_timezone = system_helper.get_timezone() assert post_swact_timezone == timezone_to_test post_swact_timestamp = system_helper.get_events(fields='Time Stamp', event_id=event_log_id, uuid=event_uuid, entity_id=entity_instance_id, state=event_state)[0][0] assert post_swact_timestamp == post_timestamp
def test_admin_password(scenario, less_than_two_cons, _revert_admin_pw): """ Test the admin password change Test Steps: - lock standby controller change password and unlock - change passowrd and swact - check alarams """ if 'swact' in scenario and less_than_two_cons: skip(SkipSysType.LESS_THAN_TWO_CONTROLLERS) host = system_helper.get_standby_controller_name() assert host, "No standby controller on system" if scenario == "lock_standby_change_pswd": # lock the standby LOG.tc_step("Attempting to lock {}".format(host)) res, out = host_helper.lock_host(host=host) LOG.tc_step("Result of the lock was: {}".format(res)) # change password prev_pswd = Tenant.get('admin')['password'] post_pswd = '!{}9'.format(prev_pswd) LOG.tc_step('Changing admin password to {}'.format(post_pswd)) code, output = keystone_helper.set_user('admin', password=post_pswd, auth_info=Tenant.get( 'admin_platform')) # assert "Warning: 'admin' password changed. Please wait 5 minutes before Locking/Unlocking # the controllers" in output LOG.tc_step("Sleep for 180 seconds after admin password change") time.sleep(180) # CGTS-6928 LOG.tc_step("Check admin password is updated in keyring") assert post_pswd == security_helper.get_admin_password_in_keyring() if scenario == "change_pswd_swact": LOG.tc_step("Swact active controller") host_helper.swact_host() else: LOG.tc_step("Unlock host {}".format(host)) res = host_helper.unlock_host(host) LOG.info("Unlock hosts result: {}".format(res)) LOG.tc_step("Check admin password is updated in keyring") assert post_pswd == security_helper.get_admin_password_in_keyring()
def test_swact_failed_controller_negative(fail_controller): """ TC610_3 Verify that swacting to a failed controller is rejected Test Setup: - Reset the standby controller Test Steps: - Attempt to swact from the active controller - Verify that the swact was rejected and the active controller is the same Teardown: - Wait until the controller is online again Returns: """ if not fail_controller: skip("Couldn't put controller into failed state.") active = system_helper.get_active_controller_name() LOG.tc_step("Attempting to swact to failed controller.") code, out = host_helper.swact_host(fail_ok=True) LOG.tc_step("Verifying that the swact didn't occur.") assert 1 == code, "FAIL: The swact wasn't rejected" curr_active = system_helper.get_active_controller_name() assert curr_active == active, "FAIL: The active controller was changed. " \ "Previous: {} Current: {}".format(active, curr_active)
def test_reapply_stx_openstack_no_change(stx_openstack_applied_required, check_nodes, controller): """ Args: stx_openstack_applied_required: Pre-requisite: - stx-openstack application in applied state Test Steps: - Re-apply stx-openstack application - Check openstack pods healthy """ # if controller == 'controller-1': # skip("CGTS-10708") if system_helper.is_aio_simplex() and controller != 'controller-0': skip('Simplex system only has controller-0') active, standby = system_helper.get_active_standby_controllers() if active != controller: if not standby: skip('{} is not ready to take over'.format(controller)) LOG.tc_step("Swact active controller to test reapply from {}".format(controller)) host_helper.swact_host() time.sleep(60) LOG.info("helm list before reapply after swact") from utils.clients.ssh import ControllerClient con_ssh = ControllerClient.get_active_controller() end_time = time.time() + 180 while time.time() < end_time: code = con_ssh.exec_cmd('helm list', expect_timeout=60)[0] if code == 0: break time.sleep(30) LOG.tc_step("Re-apply stx-openstack application") container_helper.apply_app(app_name='stx-openstack') LOG.tc_step("Check openstack pods in good state on all controllers after stx-openstack " "re-applied") for host in get_valid_controllers(): check_openstack_pods_healthy(host=host, timeout=120)
def test_host_operations_with_custom_kubectl_app(deploy_delete_kubectl_app): """ Test create, delete custom app via kubectl run cmd Args: deploy_delete_kubectl_app: fixture Setups: - Create kubectl app via kubectl run Test Steps: - If duplex: swact and verify pod still Running - Lock/unlock controller and verify pod still Running Teardown: - Delete kubectl deployment and service - Verify pod is removed """ app_name, pod_name = deploy_delete_kubectl_app active, standby = system_helper.get_active_standby_controllers() if standby: LOG.tc_step("Swact active controller and verify {} test app is " "running ".format(pod_name)) host_helper.swact_host() kube_helper.wait_for_pods_status(pod_names=pod_name, namespace='default', fail_ok=False) LOG.tc_step("Lock/unlock {} and verify {} test app is " "running.".format(active, pod_name)) HostsToRecover.add(active) host_helper.lock_host(active, swact=False) # wait for services to stabilize before unlocking time.sleep(20) host_helper.unlock_host(active) pod_name = kube_helper.get_pods(field='NAME', namespace='default', name=app_name, strict=False)[0] kube_helper.wait_for_pods_status(pod_names=pod_name, namespace=None, fail_ok=False)
def sys_controlled_swact(number_of_times=1): """ This is to identify the storage nodes and turn them off and on via vlm :return: """ for i in range(0, number_of_times): active, standby = system_helper.get_active_standby_controllers() LOG.tc_step("Doing iteration of {} of total iteration {}".format( i, number_of_times)) LOG.tc_step("'sudo reboot -f' from {}".format(standby)) host_helper.swact_host(hostname=active) LOG.tc_step("Check vms status after controller swact") vms = get_all_vms() vm_helper.wait_for_vms_values(vms, fail_ok=False, timeout=600) for vm in vms: vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm)
def ensure_host_provisioned(host, con_ssh=None): """ check if host is provisioned. Args: host (str): hostname or id in string format con_ssh (SSHClient): Returns: (return_code(int), msg(str)) # 1, 2, 3, 4, 5 only returns when fail_ok=True (0, "Host is host is provisioned) """ LOG.info("Checking if host {} is already provisioned ....".format(host)) if is_host_provisioned(host, con_ssh=None): return 0, "Host {} is provisioned" active_controller = system_helper.get_active_controller_name() conter_swact_back = False if active_controller == host: LOG.tc_step("Swact active controller and ensure active controller is changed") exit_code, output = swact_host(hostname=active_controller) assert 0 == exit_code, "{} is not recognized as active controller".format(active_controller) active_controller = system_helper.get_active_controller_name() conter_swact_back = True LOG.info("Host {} not provisioned ; doing lock/unlock to provision the host ....".format(host)) rc, output = lock_host(host, con_ssh=con_ssh) if rc != 0 and rc != -1: err_msg = "Lock host {} rejected".format(host) raise exceptions.HostError(err_msg) rc, output = unlock_host(host, available_only=True, con_ssh=con_ssh) if rc != 0: err_msg = "Unlock host {} failed: {}".format(host, output) raise exceptions.HostError(err_msg) if conter_swact_back: LOG.tc_step("Swact active controller back and ensure active controller is changed") exit_code, output = swact_host(hostname=active_controller) assert 0 == exit_code, "{} is not recognized as active controller".format(active_controller) LOG.info("Checking if host {} is provisioned after lock/unlock ....".format(host)) if not is_host_provisioned(host, con_ssh=None): raise exceptions.HostError("Failed to provision host {}") # Delay for the alarm to clear . Could be improved. time.sleep(120) return 0, "Host {} is provisioned after lock/unlock".format(host)
def _test_firewall_rules_default(): """ Verify default ports are open. Test Steps: - Confirm iptables service is running on active controller - Check if lab is http(s), add corresponding port to check - Confirm the default ports are open - Swact and repeat the above steps """ # Cannot test connecting to the ports as they are in use. default_ports = [ 123, 161, 199, 5000, 6080, 6385, 8000, 8003, 8004, 8041, 8774, 8776, 8778, 9292, 9696, 15491 ] from consts.proj_vars import ProjVar region = ProjVar.get_var('REGION') if region != 'RegionOne' and region in MULTI_REGION_MAP: default_ports.remove(5000) default_ports.remove(9292) default_ports.append(8443) if CliAuth.get_var( 'HTTPS') else default_ports.append(8080) active_controller = system_helper.get_active_controller_name() con_ssh = ControllerClient.get_active_controller() _verify_iptables_status(con_ssh, active_controller) _check_ports_with_netstat(con_ssh, active_controller, default_ports) active_controller, new_active = system_helper.get_active_standby_controllers( ) if new_active: LOG.tc_step( "Swact {} and verify firewall rules".format(active_controller)) host_helper.swact_host(active_controller) con_ssh = ControllerClient.get_active_controller() _verify_iptables_status(con_ssh, new_active) _check_ports_with_netstat(con_ssh, new_active, default_ports)
def test_dc_swact_host(swact_precheck, check_central_alarms): """ Test host swact on central region Args: swact_precheck(fixture): check subclouds managed and online Setup: - Ensure primary subcloud is managed Test Steps: - Unmanage primary subcloud - Swact the host - Verify subclouds are managed Teardown: - Manage unmanaged subclouds """ primary_subcloud, managed_subcloud = swact_precheck ssh_central = ControllerClient.get_active_controller(name="RegionOne") LOG.tc_step("Unmanage {}".format(primary_subcloud)) dc_helper.unmanage_subcloud(subcloud=primary_subcloud, check_first=True) LOG.tc_step("Swact host on central region") central_auth = Tenant.get('admin_platform', dc_region='RegionOne') host_helper.swact_host(auth_info=central_auth) LOG.tc_step("Check subclouds after host swact on central region") for managed_subcloud in managed_subcloud: dc_helper.wait_for_subcloud_status(subcloud=managed_subcloud, avail=SubcloudStatus.AVAIL_ONLINE, mgmt=SubcloudStatus.MGMT_MANAGED, sync=SubcloudStatus.SYNCED, con_ssh=ssh_central) LOG.tc_step("Manage {}".format(primary_subcloud)) dc_helper.manage_subcloud(subcloud=primary_subcloud, check_first=True) dc_helper.wait_for_subcloud_status(subcloud=primary_subcloud, avail=SubcloudStatus.AVAIL_ONLINE, mgmt=SubcloudStatus.MGMT_MANAGED, sync=SubcloudStatus.SYNCED, con_ssh=ssh_central)
def test_disable_tpm(swact_first): ssh_client = ControllerClient.get_active_controller() LOG.tc_step('Check if TPM is already configured') code, cert_id, cert_type = get_tpm_status(ssh_client) if code == 0: LOG.info('TPM is configured on the lab') if swact_first: LOG.tc_step('Swact the active controller as instructed') if len(system_helper.get_controllers()) < 2: LOG.info('Less than 2 controllers, skip swact') else: host_helper.swact_host(fail_ok=False) copy_config_from_local( ssh_client, local_conf_backup_dir, os.path.join(HostLinuxUser.get_home(), conf_backup_dir)) LOG.tc_step('Disabling TPM') code, output = remove_cert_from_tpm(ssh_client, fail_ok=False, check_first=False) assert 0 == code, 'failed to disable TPM' LOG.info('Wait the out-of-config alarm cleared') system_helper.wait_for_alarm_gone(EventLogID.CONFIG_OUT_OF_DATE) LOG.tc_step( 'Verify the configurations changes for impacted components, DO NOT expect any of the changes' ) verify_configuration_changes(expected=False, connection=ssh_client) else: LOG.info('TPM is NOT configured on the lab, skip the test') skip('TPM is NOT configured on the lab, skip the test')
def test_swact_compute_negative(): """ TC610_5 Verify that trying to swact a compute node is rejected Test Steps: - Attempt to swact from a compute """ computes = system_helper.get_hosts(personality='compute') if not computes: skip("No compute host in system") for compute in computes: LOG.tc_step("Attempting to swact from {}".format(compute)) code, out = host_helper.swact_host(compute, fail_ok=True) LOG.tc_step("Verifying that the swact didn't occur.") assert 1 == code, "FAIL: Swacting {} wasn't rejected".format(compute)
def test_swact_standby_controller_negative(): """ TC610_2 Verify that trying to swact a standby controller is rejected Test Steps: - Get the standby controller - Attempt to swact the controller - Verify that the swact doesn't happen """ standby = system_helper.get_standby_controller_name() active = system_helper.get_active_controller_name() LOG.tc_step( "Attempting to swact from standby controller {}".format(standby)) code, out = host_helper.swact_host(standby, fail_ok=True) LOG.tc_step("Verifying that the swact didn't occur.") assert 0 != code, "FAIL: The swact wasn't rejected" curr_active = system_helper.get_active_controller_name() assert curr_active == active, "FAIL: The active controller was changed. " \ "Previous: {} Current: {}".format(active, curr_active)
def test_swact_no_standby_negative(lock_controller): """ TC610_4 Verify swact without standby controller is rejected Test Setup: - Lock the standby controller Test Steps: - Attempt to swact when no standby controller available Teardown: - Unlock the controller """ LOG.tc_step("Attempting to swact when no standby controller available") active = system_helper.get_active_controller_name() code, out = host_helper.swact_host(hostname=active, fail_ok=True) LOG.tc_step("Verifying that the swact didn't occur.") assert 1 == code, "FAIL: The swact wasn't rejected" curr_active = system_helper.get_active_controller_name() assert curr_active == active, "FAIL: The active controller was changed. " \ "Previous: {} Current: {}".format(active, curr_active)
def test_dc_modify_timezone(prev_check): """ Test timezone modify on system controller and subcloud. Ensure timezone change is not propagated. Setups: - Ensure both central and subcloud regions are configured with UTC - Get the timestamps for host created_at before timezone modify Test Steps - Change the timezone in central region and wait until the change is applied - Change the timezone to a different zone in subcloud and wait until the change is applied - Verify host created_at timestamp updated according to the local timezone for the region - Swact on subcloud and ensure timezone and host created_at timestamp persists locally - Swact central controller and ensure timezone and host created_at timestamp persists in central and subcloud Teardown - Change timezone to UTC in both central and subcloud regions - Ensure host created_at timestamp is reverted to original """ prev_central_time, prev_sub_time, central_zone, sub_zone, central_auth, subcloud_auth, \ subcloud = prev_check LOG.tc_step("Modify timezone to {} in central region".format(central_zone)) system_helper.modify_timezone(timezone=central_zone, auth_info=central_auth) LOG.tc_step( "Waiting for timestamp for host created_at to update in central region" ) post_central_time = wait_for_timestamp_update( prev_timestamp=prev_central_time, auth_info=central_auth) assert post_central_time != prev_central_time, \ "host created_at timestamp did not update after timezone changed " \ "to {} in central region".format(central_zone) LOG.tc_step("Modify timezone to {} in {}".format(sub_zone, subcloud)) system_helper.modify_timezone(timezone=sub_zone, auth_info=subcloud_auth) LOG.tc_step( "Waiting for timestamp for same host created_at to update in {}". format(subcloud)) post_sub_time = wait_for_timestamp_update(prev_timestamp=prev_sub_time, auth_info=subcloud_auth) assert post_sub_time != prev_sub_time, \ "host created_at timestamp did not update after timezone changed to {} " \ "in {}".format(sub_zone, subcloud) assert post_sub_time != post_central_time, \ "Host created_at timestamp is the same on central and {} when configured with different " \ "timezones".format(subcloud) LOG.tc_step( "Ensure host created_at timestamp does not change after subcloud sync audit" ) dc_helper.wait_for_sync_audit(subclouds=subcloud, fail_ok=True, timeout=660) post_sync_sub_time = system_helper.get_host_values( host='controller-0', fields='created_at', auth_info=subcloud_auth)[0] assert post_sub_time == post_sync_sub_time, \ "Host created_at timestamp changed after sync audit on {}".format(subcloud) if not system_helper.is_aio_simplex(): LOG.tc_step( "Swact in {} region and verify timezone persists locally".format( subcloud)) host_helper.swact_host(auth_info=subcloud_auth) post_swact_sub_zone = system_helper.get_timezone( auth_info=subcloud_auth) assert post_swact_sub_zone == sub_zone post_swact_sub_time = system_helper.get_host_values( host='controller-0', fields='created_at', auth_info=subcloud_auth)[0] assert post_swact_sub_time == post_sub_time if system_helper.get_standby_controller_name(auth_info=central_auth): LOG.tc_step( "Swact in central region, and ensure timezone persists locally in central" " and subcloud") host_helper.swact_host(auth_info=central_auth) # Verify central timezone persists post_swact_central_zone = system_helper.get_timezone( auth_info=central_auth) assert post_swact_central_zone == central_zone post_swact_central_time = system_helper.get_host_values( host='controller-0', fields='created_at', auth_info=central_auth)[0] assert post_swact_central_time == post_central_time # Verify subcloud timezone persists post_central_swact_sub_zone = system_helper.get_timezone( auth_info=subcloud_auth) assert post_central_swact_sub_zone == sub_zone post_central_swact_sub_time = system_helper.get_host_values( host='controller-0', fields='created_at', auth_info=subcloud_auth)[0] assert post_central_swact_sub_time == post_sub_time
def test_ntfs(stx_openstack_required, host_type="controller"): """ This test will test NTFS mount and NTFS formatted device creation on a TiS system. Arguments: - host_type (string) - host type to be tested, e.g. controller, compute, storage Returns: - Nothing Test Steps: 1. Check if desired host has USB inserted. If not, skip 2. Wipe USB 3. Change label of device 4. Create partitions on NTFS device 5. Format partitions 4. Copy large image to NTFS mount point 5. Test mount and big file creation on NTFS mounted device """ # Could pass these in through parametrize instead mount_type = "ntfs" mount_point = "/media/ntfs/" guest_os = 'win_2012' boot_source = "image" host, usb_device = locate_usb(host_type, min_size=13) if not host: skip("No USB hardware found on {} host type".format(host_type)) hosts_with_image_backing = host_helper.get_hosts_in_storage_backing(storage_backing='image') if len(hosts_with_image_backing) == 0: skip("No hosts with image backing present") # if the host with the USB is not the active controler, swact controllers con_ssh = ControllerClient.get_active_controller() active_controller = system_helper.get_active_controller_name(con_ssh) if host != active_controller: host_helper.swact_host() with host_helper.ssh_to_host(host) as host_ssh: wipe_usb(host_ssh, usb_device) umount_usb(host_ssh, mount_point=mount_point) create_usb_label(host_ssh, usb_device, label="msdos") create_usb_partition(host_ssh, usb_device, startpt="0", endpt="2048") format_usb(host_ssh, usb_device, partition="1") create_usb_partition(host_ssh, usb_device, startpt="2049", endpt="100%") format_usb(host_ssh, usb_device, partition="2") mount_usb(host_ssh, usb_device, partition="2", mount_type=mount_type, mount_point=mount_point) LOG.tc_step("Copy the windows guest image to the mount point") src_img = glance_helper.scp_guest_image(img_os=guest_os, dest_dir=mount_point, con_ssh=con_ssh) LOG.tc_step("Create flavor for windows guest image") flv_id = nova_helper.create_flavor(name=guest_os, vcpus=4, ram=8192, storage_backing="local_image", guest_os=guest_os)[1] nova_helper.set_flavor(flv_id, **{FlavorSpec.CPU_POLICY: "dedicated"}) ResourceCleanup.add("flavor", flv_id) LOG.tc_step("Import image into glance") glance_helper.create_image(name=guest_os, source_image_file=src_img, disk_format="qcow2", container_format="bare", con_ssh=con_ssh, cleanup="function") LOG.tc_step("Boot VM") vm_id = vm_helper.boot_vm(name=guest_os, flavor=flv_id, guest_os=guest_os, source=boot_source, cleanup="function")[1] LOG.tc_step("Ping vm and ssh to it") vm_helper.wait_for_vm_pingable_from_natbox(vm_id) with vm_helper.ssh_to_vm_from_natbox(vm_id) as vm_ssh: output = vm_ssh.exec_cmd('pwd', fail_ok=False)[1] LOG.info(output)
def test_swact_100_times(): """ Skip Condition: - Less than two controllers on system Test Steps: - Boot a vm and ensure it's pingable - Start writing from pre-existed vm before swacting - Repeat following steps 100 times: - ensure system has standby controller - system host-swact - ensure all services are active in sudo sm-dump on new active controller - ensure pre-existed vm is still pingable from NatBox - ensure writing did not stop on pre-existed vm - ensure new vm can be launched in 2 minutes - ensure newly booted vm is pingable from NatBox - delete newly booted vm Teardown: - delete vms, volumes """ if len(system_helper.get_controllers()) < 2: skip("Less than two controllers on system") if not system_helper.get_standby_controller_name(): assert False, "No standby controller on system" LOG.tc_step("Boot a vm and ensure it's pingable") vm_base = vm_helper.boot_vm(name='pre_swact', cleanup='function')[1] LOG.tc_step("Start writing from pre-existed vm before swacting") end_event = Events("End write in base vm") base_vm_thread = vm_helper.write_in_vm(vm_base, end_event=end_event, expect_timeout=40, thread_timeout=60*100) try: for i in range(100): iter_str = "Swact iter{}/100 - ".format(i+1) LOG.tc_step("{}Ensure system has standby controller".format(iter_str)) standby = system_helper.get_standby_controller_name() assert standby LOG.tc_step("{}Swact active controller and ensure active controller is changed".format(iter_str)) host_helper.swact_host() LOG.tc_step("{}Check all services are up on active controller via sudo sm-dump".format(iter_str)) host_helper.wait_for_sm_dump_desired_states(controller=standby, fail_ok=False) LOG.tc_step("{}Ensure pre-existed vm still pingable post swact".format(iter_str)) vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_base, timeout=45) time.sleep(5) LOG.tc_step("{}Ensure writing from pre-existed vm resumes after swact".format(iter_str)) assert base_vm_thread.res is True, "Writing in pre-existed vm stopped after {}".format(iter_str.lower()) LOG.tc_step("{}Attempt to boot new vm after 2 minutes of post swact and ensure it's pingable". format(iter_str)) time.sleep(60) for j in range(3): code, vm_new, msg = vm_helper.boot_vm(name='post_swact', fail_ok=True, cleanup='function') if code == 0: break LOG.warning("VM failed to boot - attempt{}".format(j+1)) vm_helper.delete_vms(vms=vm_new) assert j < 2, "No vm can be booted 2+ minutes after swact" LOG.tc_step("{}VM{} failed to boot, wait for 30 seconds and retry".format(j+1, iter_str)) time.sleep(30) vm_helper.wait_for_vm_pingable_from_natbox(vm_new) LOG.tc_step("{}Delete the vm created".format(iter_str)) vm_helper.delete_vms(vms=vm_new) except: raise finally: LOG.tc_step("End the base_vm_thread") end_event.set() base_vm_thread.wait_for_thread_end(timeout=20) post_standby = system_helper.get_standby_controller_name() assert post_standby, "System does not have standby controller after last swact"
def test_dc_modify_https(revert_https): """ Test enable/disable https Test Steps: - Ensure central region and subcloud admin endpoint are https - Ensure central region https to be different than subcloud - Wait for subcloud sync audit and ensure subcloud https is not changed - Verify cli's in subcloud and central region - Modify https on central and subcloud - Verify cli's in subcloud and central region - swact central and subcloud - Ensure central region and subcloud admin endpoint are https Teardown: - Revert https config on central and subcloud """ origin_https_sub, origin_https_central, central_auth, sub_auth, use_dnsname = revert_https subcloud = ProjVar.get_var('PRIMARY_SUBCLOUD') LOG.tc_step( "Before testing, Ensure central region and subcloud admin internal endpoint are https") assert keystone_helper.is_https_enabled(interface='admin', auth_info=central_auth), \ "Central region admin internal endpoint is not https" assert keystone_helper.is_https_enabled(interface='admin', auth_info=sub_auth), \ "Subcloud admin internal endpoint is not https" new_https_sub = not origin_https_sub new_https_central = not origin_https_central LOG.tc_step("Ensure central region https to be different than {}".format(subcloud)) security_helper.modify_https(enable_https=new_https_sub, auth_info=central_auth) LOG.tc_step('Check public endpoints accessibility for central region') security_helper.check_services_access(region='RegionOne', auth_info=central_auth, use_dnsname=use_dnsname) LOG.tc_step('Check platform horizon accessibility') security_helper.check_platform_horizon_access(use_dnsname=use_dnsname) LOG.tc_step("Wait for subcloud sync audit with best effort and ensure {} https is not " "changed".format(subcloud)) dc_helper.wait_for_sync_audit(subclouds=subcloud, fail_ok=True, timeout=660) assert origin_https_sub == keystone_helper.is_https_enabled(auth_info=sub_auth), \ "HTTPS config changed in subcloud" LOG.tc_step("Verify cli's in {} and central region".format(subcloud)) verify_cli(sub_auth, central_auth) if new_https_central != new_https_sub: LOG.tc_step("Set central region https to {}".format(new_https_central)) security_helper.modify_https(enable_https=new_https_central, auth_info=central_auth) LOG.tc_step("Ensure central region and subcloud admin internal endpoint are still https") assert keystone_helper.is_https_enabled(interface='admin', auth_info=central_auth), \ "Central region admin internal endpoint is not https" assert keystone_helper.is_https_enabled(interface='admin', auth_info=sub_auth), \ "Subcloud admin internal endpoint is not https" LOG.tc_step('Check public endpoints accessibility for central region') security_helper.check_services_access(region='RegionOne', auth_info=central_auth, use_dnsname=use_dnsname) LOG.tc_step('Check platform horizon accessibility') security_helper.check_platform_horizon_access(use_dnsname=use_dnsname) LOG.tc_step("Set {} https to {}".format(subcloud, new_https_sub)) security_helper.modify_https(enable_https=new_https_sub, auth_info=sub_auth) LOG.tc_step('Check public endpoints accessibility for {} region'.format(subcloud)) security_helper.check_services_access(region=subcloud, auth_info=sub_auth, use_dnsname=use_dnsname) LOG.tc_step("Ensure central region and subcloud admin internal endpoint are still https") assert keystone_helper.is_https_enabled(interface='admin', auth_info=central_auth), \ "Central region admin internal endpoint is not https" assert keystone_helper.is_https_enabled(interface='admin', auth_info=sub_auth), \ "Subcloud admin internal endpoint is not https" LOG.tc_step("Verify cli's in {} and central region after https modify on " "subcloud".format(subcloud)) verify_cli(sub_auth, central_auth) LOG.tc_step("Swact on central region") host_helper.swact_host(auth_info=central_auth) LOG.tc_step( "Verify cli's in {} and central region after central region swact" .format(subcloud)) verify_cli(sub_auth, central_auth) if not system_helper.is_aio_simplex(auth_info=sub_auth): LOG.tc_step("Swact on subcloud {}".format(subcloud)) host_helper.swact_host(auth_info=sub_auth) LOG.tc_step("Verify cli's in {} and central region after subcloud swact".format(subcloud)) verify_cli(sub_auth, central_auth) LOG.tc_step("Ensure after swact, central region and subcloud admin internal endpoint are https") assert keystone_helper.is_https_enabled(interface='admin', auth_info=central_auth), \ "Central region admin internal endpoint is not https" assert keystone_helper.is_https_enabled(interface='admin', auth_info=sub_auth), \ "Subcloud admin internal endpoint is not https"
def test_swact_controllers(stx_openstack_required, wait_for_con_drbd_sync_complete): """ Verify swact active controller Test Steps: - Boot a vm on system and check ping works - Swact active controller - Verify standby controller and active controller are swapped - Verify vm is still pingable """ if not wait_for_con_drbd_sync_complete: skip(SkipSysType.LESS_THAN_TWO_CONTROLLERS) LOG.tc_step('retrieve active and available controllers') pre_active_controller, pre_standby_controller = system_helper.get_active_standby_controllers( ) assert pre_standby_controller, "No standby controller available" pre_res_sys, pre_msg_sys = system_helper.wait_for_services_enable( timeout=20, fail_ok=True) up_hypervisors = host_helper.get_up_hypervisors() pre_res_neutron, pre_msg_neutron = network_helper.wait_for_agents_healthy( up_hypervisors, timeout=20, fail_ok=True) LOG.tc_step("Boot a vm from image and ping it") vm_id_img = vm_helper.boot_vm(name='swact_img', source='image', cleanup='function')[1] vm_helper.wait_for_vm_pingable_from_natbox(vm_id_img) LOG.tc_step("Boot a vm from volume and ping it") vm_id_vol = vm_helper.boot_vm(name='swact', cleanup='function')[1] vm_helper.wait_for_vm_pingable_from_natbox(vm_id_vol) LOG.tc_step( "Swact active controller and ensure active controller is changed") host_helper.swact_host(hostname=pre_active_controller) LOG.tc_step("Verify standby controller and active controller are swapped") post_active_controller = system_helper.get_active_controller_name() post_standby_controller = system_helper.get_standby_controller_name() assert pre_standby_controller == post_active_controller, \ "Prev standby: {}; Post active: {}".format( pre_standby_controller, post_active_controller) assert pre_active_controller == post_standby_controller, \ "Prev active: {}; Post standby: {}".format( pre_active_controller, post_standby_controller) LOG.tc_step("Check boot-from-image vm still pingable after swact") vm_helper.wait_for_vm_pingable_from_natbox(vm_id_img, timeout=30) LOG.tc_step("Check boot-from-volume vm still pingable after swact") vm_helper.wait_for_vm_pingable_from_natbox(vm_id_vol, timeout=30) LOG.tc_step( "Check system services and neutron agents after swact from {}".format( pre_active_controller)) post_res_sys, post_msg_sys = system_helper.wait_for_services_enable( fail_ok=True) post_res_neutron, post_msg_neutron = network_helper.wait_for_agents_healthy( hosts=up_hypervisors, fail_ok=True) assert post_res_sys, \ "\nPost-evac system services stats: {}\nPre-evac system services stats: {}". \ format(post_msg_sys, pre_msg_sys) assert post_res_neutron, \ "\nPost evac neutron agents stats: {}\nPre-evac neutron agents stats: {}". \ format(pre_msg_neutron, post_msg_neutron) LOG.tc_step("Check hosts are Ready in kubectl get nodes after swact") kube_helper.wait_for_nodes_ready(hosts=(pre_active_controller, pre_standby_controller), timeout=30)