def cleanup_app(): if container_helper.get_apps(application=HELM_APP_NAME): LOG.fixture_step("Remove {} app if applied".format(HELM_APP_NAME)) container_helper.remove_app(app_name=HELM_APP_NAME) LOG.fixture_step("Delete {} app".format(HELM_APP_NAME)) container_helper.delete_app(app_name=HELM_APP_NAME)
def reset(): app_name = 'stx-openstack' post_status = container_helper.get_apps(application=app_name, field='status')[0] if not post_status.endswith('ed'): LOG.fixture_step("Wait for application apply finish") container_helper.wait_for_apps_status(apps=app_name, status=AppStatus.APPLIED, timeout=1800, check_interval=15, fail_ok=False) user_overrides = container_helper.get_helm_override_values(chart='nova', namespace='openstack', fields='user_overrides')[0] if not user_overrides or user_overrides == 'None': LOG.info("No change in nova user_overrides. Do nothing.") return LOG.fixture_step("Update nova helm-override to reset values") container_helper.update_helm_override(chart='nova', namespace='openstack', reset_vals=True) user_overrides = container_helper.get_helm_override_values(chart='nova', namespace='openstack', fields='user_overrides')[0] assert not user_overrides, "nova helm user_overrides still exist after reset-values" LOG.fixture_step("Re-apply stx-openstack application and ensure it is applied") container_helper.apply_app(app_name='stx-openstack', check_first=False, applied_timeout=1800) check_cmd = 'grep foo {}'.format(conf_path) LOG.fixture_step("Ensure user_override is removed from {} in nova-compute " "containers".format(conf_path)) for host in valid_hosts: with host_helper.ssh_to_host(host) as host_ssh: LOG.info("Wait for nova-compute pods running on {}".format(host)) kube_helper.wait_for_openstack_pods_status(application='nova', component='compute', con_ssh=host_ssh, status=PodStatus.RUNNING) LOG.info("Check new release generated for nova compute pods on {}".format(host)) nova_compute_pods = kube_helper.get_openstack_pods(field='NAME', application='nova', component='compute', con_ssh=host_ssh)[0] nova_compute_pods = sorted(nova_compute_pods) if NEW_NOVA_COMPUTE_PODS: assert NEW_NOVA_COMPUTE_PODS != nova_compute_pods, \ "No new release generated after reset values" LOG.info("Check custom conf is removed from {} in nova compute " "container on {}".format(conf_path, host)) for nova_compute_pod in nova_compute_pods: code, output = kube_helper.exec_cmd_in_container(cmd=check_cmd, pod=nova_compute_pod, fail_ok=True, con_ssh=host_ssh, namespace='openstack', container_name='nova-compute') assert code == 1, "{} on {} still contains user override info after " \ "reset nova helm-override values and reapply stx-openstack " \ "app: {}".format(conf_path, host, output)
def test_system_health_pre_session(): LOG.tc_step("Check {} status".format(PLATFORM_APP)) assert container_helper.get_apps(application=PLATFORM_APP)[0] == AppStatus.APPLIED, \ "{} is not {}".format(PLATFORM_APP, AppStatus.APPLIED) LOG.tc_step("Check system alarms") check_helper.check_alarms(before_alarms=[]) LOG.info("No new alarms found.")
def test_openstack_pods_healthy(): """ Check the health of the OpenStack pods: healthy, running or in completed state """ application_status = container_helper.get_apps( application="stx-openstack")[0] assert application_status == "applied", "System status is not in state applied" command_health = kube_helper.wait_for_pods_healthy( namespace="stx-openstack") assert command_health is None, "Check PODs health has failed"
def __verify_alarms(request, scope): before_alarms = __get_alarms(scope=scope) prev_bad_pods = kube_helper.get_unhealthy_pods(all_namespaces=True) prev_applied_apps = container_helper.get_apps(field='application', status=AppStatus.APPLIED) def verify_(): LOG.fixture_step( "({}) Verify system alarms, applications and pods status after " "test {} ended...".format(scope, scope)) alarm_res, new_alarms = check_helper.check_alarms( before_alarms=before_alarms, fail_ok=True) post_apps_status = container_helper.get_apps( field=('application', 'status'), application=prev_applied_apps) LOG.info('prev bad pods: {}'.format(prev_bad_pods)) exclude = False prev_name = None if prev_bad_pods: exclude = True prev_name = prev_bad_pods kube_helper.wait_for_pods_healthy(timeout=300, all_namespaces=True, name=prev_name, exclude=exclude, strict=True) # check no new bad application LOG.info("Check applications status...") new_bad_apps = [ item for item in post_apps_status if item[1] != AppStatus.APPLIED ] if new_bad_apps: applying_apps = [ item[0] for item in post_apps_status if item[1] == AppStatus.APPLYING ] if applying_apps: LOG.info('Wait for {} to apply'.format(applying_apps)) container_helper.wait_for_apps_status(apps=applying_apps, status=AppStatus.APPLIED, check_interval=20, timeout=600) new_bad_apps = [ item for item in new_bad_apps if item[0] not in applying_apps ] assert not new_bad_apps, "Applications no longer applied after test " \ "{}: {}".format(scope, new_bad_apps) # check no new alarms assert alarm_res, "New alarm(s) appeared within test {}: {}".format( scope, new_alarms) request.addfinalizer(verify_) return
def wait_for_recover(): post_status = container_helper.get_apps(application=app_name)[0] if not post_status == AppStatus.APPLIED: LOG.info("Dump info for unhealthy pods") kube_helper.dump_pods_info() if not post_status.endswith('ed'): LOG.fixture_step("Wait for application apply finish") container_helper.wait_for_apps_status(apps=app_name, status=AppStatus.APPLIED, timeout=3600, check_interval=15, fail_ok=False)
def test_openstack_services_healthy(): """ Pre-requisite: - stx-openstack application exists Test steps: - Check stx-openstack application in applied state via system application-list - Check all openstack pods in running or completed state via kubectl get """ LOG.tc_step("Check stx-openstack application is applied") status = container_helper.get_apps(application='stx-openstack')[0] if not status: skip('Openstack application is not uploaded.') assert status == AppStatus.APPLIED, "stx-openstack is in {} status instead of " \ "applied".format(status) LOG.tc_step("Check openstack pods are in running or completed status via kubectl get on all " "controllers") controllers = get_valid_controllers() for host in controllers: check_openstack_pods_healthy(host=host, timeout=60)
def test_launch_app_via_sysinv(copy_test_apps): """ Test upload, apply, remove, delete custom app via system cmd Args: copy_test_apps (str): module fixture cleanup_app: fixture Setups: - Copy test files from test server to tis system (module) - Remove and delete test app if exists Test Steps: - system application-upload test app tar file and wait for it to be uploaded - system application-apply test app and wait for it to be applied - wget <oam_ip>:<app_targetPort> from remote host - Verify app contains expected content - system application-remove test app and wait for it to be uninstalled - system application-delete test app from system """ app_dir = copy_test_apps app_name = HELM_APP_NAME central_ssh = ControllerClient.get_active_controller(name='RegionOne') central_auth = Tenant.get('admin_platform', dc_region='SystemController') platform_app = container_helper.get_apps(auth_info=central_auth, application='platform-integ-apps') LOG.info('Test platform-integ-apps is applied') assert len(platform_app) != 0 and platform_app[0] == 'applied' subclouds = dc_helper.get_subclouds() LOG.tc_step("Upload and apply {} on system controller".format(app_name)) container_helper.upload_app(app_name=app_name, app_version=HELM_APP_VERSION, tar_file=os.path.join(app_dir, HELM_TAR), auth_info=central_auth) container_helper.apply_app(app_name=app_name, auth_info=central_auth) LOG.tc_step( "Check docker image stored in System controller registry.local") code, output = cli.system(cmd="registry-image-list | fgrep hellokitty", ssh_client=central_ssh, fail_ok=True) assert code == 0 # LOG.info("code %s, output %s", code, output) for subcloud in subclouds: subcloud_auth = Tenant.get('admin_platform', dc_region=subcloud) LOG.tc_step("Upload/apply custom app on subcloud: {}".format(subcloud)) platform_app = container_helper.get_apps( auth_info=subcloud_auth, application='platform-integ-apps') LOG.info('Test platform-integ-apps is applied, on subcloud {}'.format( subcloud)) assert len(platform_app) != 0 and platform_app[0] == 'applied' LOG.tc_step("Upload and apply {} on subcloud: {}".format( app_name, subcloud)) container_helper.upload_app(app_name=app_name, app_version=HELM_APP_VERSION, tar_file=os.path.join(app_dir, HELM_TAR), auth_info=subcloud_auth) container_helper.apply_app(app_name=app_name, auth_info=subcloud_auth) LOG.tc_step("Check docker image stored on {} registry.central".format( subcloud)) code, output = cli.system(cmd="registry-image-list | fgrep hellokitty", ssh_client=central_ssh, auth_info=subcloud_auth, fail_ok=True) assert code == 0