def cleanup_app():
if container_helper.get_apps(application=HELM_APP_NAME):
LOG.fixture_step("Remove {} app if applied".format(HELM_APP_NAME))
container_helper.remove_app(app_name=HELM_APP_NAME)
LOG.fixture_step("Delete {} app".format(HELM_APP_NAME))
container_helper.delete_app(app_name=HELM_APP_NAME)
def reset():
app_name = 'stx-openstack'
post_status = container_helper.get_apps(application=app_name, field='status')[0]
if not post_status.endswith('ed'):
LOG.fixture_step("Wait for application apply finish")
container_helper.wait_for_apps_status(apps=app_name, status=AppStatus.APPLIED,
timeout=1800,
check_interval=15, fail_ok=False)
user_overrides = container_helper.get_helm_override_values(chart='nova',
namespace='openstack',
fields='user_overrides')[0]
if not user_overrides or user_overrides == 'None':
LOG.info("No change in nova user_overrides. Do nothing.")
return
LOG.fixture_step("Update nova helm-override to reset values")
container_helper.update_helm_override(chart='nova', namespace='openstack', reset_vals=True)
user_overrides = container_helper.get_helm_override_values(chart='nova',
namespace='openstack',
fields='user_overrides')[0]
assert not user_overrides, "nova helm user_overrides still exist after reset-values"
LOG.fixture_step("Re-apply stx-openstack application and ensure it is applied")
container_helper.apply_app(app_name='stx-openstack', check_first=False,
applied_timeout=1800)
check_cmd = 'grep foo {}'.format(conf_path)
LOG.fixture_step("Ensure user_override is removed from {} in nova-compute "
"containers".format(conf_path))
for host in valid_hosts:
with host_helper.ssh_to_host(host) as host_ssh:
LOG.info("Wait for nova-compute pods running on {}".format(host))
kube_helper.wait_for_openstack_pods_status(application='nova',
component='compute',
con_ssh=host_ssh,
status=PodStatus.RUNNING)
LOG.info("Check new release generated for nova compute pods on {}".format(host))
nova_compute_pods = kube_helper.get_openstack_pods(field='NAME', application='nova',
component='compute',
con_ssh=host_ssh)[0]
nova_compute_pods = sorted(nova_compute_pods)
if NEW_NOVA_COMPUTE_PODS:
assert NEW_NOVA_COMPUTE_PODS != nova_compute_pods, \
"No new release generated after reset values"
LOG.info("Check custom conf is removed from {} in nova compute "
"container on {}".format(conf_path, host))
for nova_compute_pod in nova_compute_pods:
code, output = kube_helper.exec_cmd_in_container(cmd=check_cmd,
pod=nova_compute_pod,
fail_ok=True,
con_ssh=host_ssh,
namespace='openstack',
container_name='nova-compute')
assert code == 1, "{} on {} still contains user override info after " \
"reset nova helm-override values and reapply stx-openstack " \
"app: {}".format(conf_path, host, output)
def test_system_health_pre_session():
LOG.tc_step("Check {} status".format(PLATFORM_APP))
assert container_helper.get_apps(application=PLATFORM_APP)[0] == AppStatus.APPLIED, \
"{} is not {}".format(PLATFORM_APP, AppStatus.APPLIED)
LOG.tc_step("Check system alarms")
check_helper.check_alarms(before_alarms=[])
LOG.info("No new alarms found.")
def test_openstack_pods_healthy():
"""
Check the health of the OpenStack pods: healthy, running or in completed state
"""
application_status = container_helper.get_apps(
application="stx-openstack")[0]
assert application_status == "applied", "System status is not in state applied"
command_health = kube_helper.wait_for_pods_healthy(
namespace="stx-openstack")
assert command_health is None, "Check PODs health has failed"
def __verify_alarms(request, scope):
before_alarms = __get_alarms(scope=scope)
prev_bad_pods = kube_helper.get_unhealthy_pods(all_namespaces=True)
prev_applied_apps = container_helper.get_apps(field='application',
status=AppStatus.APPLIED)
def verify_():
LOG.fixture_step(
"({}) Verify system alarms, applications and pods status after "
"test {} ended...".format(scope, scope))
alarm_res, new_alarms = check_helper.check_alarms(
before_alarms=before_alarms, fail_ok=True)
post_apps_status = container_helper.get_apps(
field=('application', 'status'), application=prev_applied_apps)
LOG.info('prev bad pods: {}'.format(prev_bad_pods))
exclude = False
prev_name = None
if prev_bad_pods:
exclude = True
prev_name = prev_bad_pods
kube_helper.wait_for_pods_healthy(timeout=300,
all_namespaces=True,
name=prev_name,
exclude=exclude,
strict=True)
# check no new bad application
LOG.info("Check applications status...")
new_bad_apps = [
item for item in post_apps_status if item[1] != AppStatus.APPLIED
]
if new_bad_apps:
applying_apps = [
item[0] for item in post_apps_status
if item[1] == AppStatus.APPLYING
]
if applying_apps:
LOG.info('Wait for {} to apply'.format(applying_apps))
container_helper.wait_for_apps_status(apps=applying_apps,
status=AppStatus.APPLIED,
check_interval=20,
timeout=600)
new_bad_apps = [
item for item in new_bad_apps if item[0] not in applying_apps
]
assert not new_bad_apps, "Applications no longer applied after test " \
"{}: {}".format(scope, new_bad_apps)
# check no new alarms
assert alarm_res, "New alarm(s) appeared within test {}: {}".format(
scope, new_alarms)
request.addfinalizer(verify_)
return
def wait_for_recover():
post_status = container_helper.get_apps(application=app_name)[0]
if not post_status == AppStatus.APPLIED:
LOG.info("Dump info for unhealthy pods")
kube_helper.dump_pods_info()
if not post_status.endswith('ed'):
LOG.fixture_step("Wait for application apply finish")
container_helper.wait_for_apps_status(apps=app_name,
status=AppStatus.APPLIED,
timeout=3600,
check_interval=15,
fail_ok=False)
def test_openstack_services_healthy():
"""
Pre-requisite:
- stx-openstack application exists
Test steps:
- Check stx-openstack application in applied state via system application-list
- Check all openstack pods in running or completed state via kubectl get
"""
LOG.tc_step("Check stx-openstack application is applied")
status = container_helper.get_apps(application='stx-openstack')[0]
if not status:
skip('Openstack application is not uploaded.')
assert status == AppStatus.APPLIED, "stx-openstack is in {} status instead of " \
"applied".format(status)
LOG.tc_step("Check openstack pods are in running or completed status via kubectl get on all "
"controllers")
controllers = get_valid_controllers()
for host in controllers:
check_openstack_pods_healthy(host=host, timeout=60)
def test_launch_app_via_sysinv(copy_test_apps):
"""
Test upload, apply, remove, delete custom app via system cmd
Args:
copy_test_apps (str): module fixture
cleanup_app: fixture
Setups:
- Copy test files from test server to tis system (module)
- Remove and delete test app if exists
Test Steps:
- system application-upload test app tar file and wait for it to be
uploaded
- system application-apply test app and wait for it to be applied
- wget <oam_ip>:<app_targetPort> from remote host
- Verify app contains expected content
- system application-remove test app and wait for it to be uninstalled
- system application-delete test app from system
"""
app_dir = copy_test_apps
app_name = HELM_APP_NAME
central_ssh = ControllerClient.get_active_controller(name='RegionOne')
central_auth = Tenant.get('admin_platform', dc_region='SystemController')
platform_app = container_helper.get_apps(auth_info=central_auth,
application='platform-integ-apps')
LOG.info('Test platform-integ-apps is applied')
assert len(platform_app) != 0 and platform_app[0] == 'applied'
subclouds = dc_helper.get_subclouds()
LOG.tc_step("Upload and apply {} on system controller".format(app_name))
container_helper.upload_app(app_name=app_name,
app_version=HELM_APP_VERSION,
tar_file=os.path.join(app_dir, HELM_TAR),
auth_info=central_auth)
container_helper.apply_app(app_name=app_name, auth_info=central_auth)
LOG.tc_step(
"Check docker image stored in System controller registry.local")
code, output = cli.system(cmd="registry-image-list | fgrep hellokitty",
ssh_client=central_ssh,
fail_ok=True)
assert code == 0
# LOG.info("code %s, output %s", code, output)
for subcloud in subclouds:
subcloud_auth = Tenant.get('admin_platform', dc_region=subcloud)
LOG.tc_step("Upload/apply custom app on subcloud: {}".format(subcloud))
platform_app = container_helper.get_apps(
auth_info=subcloud_auth, application='platform-integ-apps')
LOG.info('Test platform-integ-apps is applied, on subcloud {}'.format(
subcloud))
assert len(platform_app) != 0 and platform_app[0] == 'applied'
LOG.tc_step("Upload and apply {} on subcloud: {}".format(
app_name, subcloud))
container_helper.upload_app(app_name=app_name,
app_version=HELM_APP_VERSION,
tar_file=os.path.join(app_dir, HELM_TAR),
auth_info=subcloud_auth)
container_helper.apply_app(app_name=app_name, auth_info=subcloud_auth)
LOG.tc_step("Check docker image stored on {} registry.central".format(
subcloud))
code, output = cli.system(cmd="registry-image-list | fgrep hellokitty",
ssh_client=central_ssh,
auth_info=subcloud_auth,
fail_ok=True)
assert code == 0