def test_dc_dns_modify(ensure_synced, scenario):
"""
Update DNS servers on central region and check it is propagated to subclouds
Args:
ensure_synced: test fixture
scenario: DNS change scenario
Setups:
- Ensure primary subcloud is managed and DNS config is valid and synced
Test Steps:
- Un-manage primary subcloud
- Configure DNS servers on central region to new value based on given scenario
- Wait for new DNS config to sync over to managed online subclouds
- Ensure DNS config is not updated on unmanaged primary subcloud
- Re-manage primary subcloud and ensure DNS config syncs over
- Verify nslookup works in Central Region and primary subcloud
Teardown:
- Reset DNS servers to original value (module)
"""
primary_subcloud, managed_subclouds, prev_dns_servers = ensure_synced
new_dns_servers = compose_new_dns_servers(
scenario=scenario, prev_dns_servers=prev_dns_servers)
LOG.tc_step("Unmanage {}".format(primary_subcloud))
dc_helper.unmanage_subcloud(subcloud=primary_subcloud, check_first=True)
LOG.tc_step(
"Reconfigure DNS servers on central region from {} to {}".format(
prev_dns_servers, new_dns_servers))
system_helper.set_dns_servers(new_dns_servers,
auth_info=Tenant.get('admin_platform',
dc_region='RegionOne'))
LOG.tc_step(
"Wait for new DNS config to sync over to managed online subclouds")
for managed_sub in managed_subclouds:
dc_helper.wait_for_subcloud_dns_config(subcloud=managed_sub,
expected_dns=new_dns_servers)
LOG.tc_step(
"Ensure DNS config is not updated on unmanaged subcloud: {}".format(
primary_subcloud))
code = dc_helper.wait_for_subcloud_dns_config(subcloud=primary_subcloud,
expected_dns=new_dns_servers,
timeout=60,
fail_ok=True)[0]
assert 1 == code, "Actual return code: {}".format(code)
LOG.tc_step('Re-manage {} and ensure DNS config syncs over'.format(
primary_subcloud))
dc_helper.manage_subcloud(subcloud=primary_subcloud, check_first=False)
dc_helper.wait_for_subcloud_dns_config(subcloud=primary_subcloud,
expected_dns=new_dns_servers)
LOG.tc_step('Verify nslookup works in Central Region and {}'.format(
primary_subcloud))
verify_dns_on_central_and_subcloud(primary_subcloud)
def test_dc_keypair(keypair_precheck):
"""
Create keypair on central region and check it is propagated to subclouds
Args:
keypair_precheck: test fixture for setup/teardown
Setups:
- Ensure primary subcloud is managed and keypair info is synced
Test Steps:
- Un-manage primary subcloud
- Add a new keypair on central region
- Wait for new keypair to sync over to managed online subclouds
- Ensure central keypair is not updated on unmanaged primary subcloud
- Re-manage primary subcloud and ensure new keypair syncs over
Teardown:
- Delete new created keypair
"""
primary_subcloud, managed_subclouds, central_keypair = keypair_precheck
central_auth = Tenant.get('admin', dc_region='RegionOne')
LOG.tc_step("Unmanage {}".format(primary_subcloud))
dc_helper.unmanage_subcloud(subcloud=primary_subcloud, check_first=False)
LOG.tc_step('Add new keypair to central region')
nova_helper.create_keypair(NEW_KEYPAIR, auth_info=central_auth)
LOG.tc_step("Wait for new keypair to sync over to managed subclouds: {}".format(managed_subclouds))
expt_keypair = central_keypair + [NEW_KEYPAIR]
dc_helper.wait_for_sync_audit(subclouds=managed_subclouds, filters_regex='keypair')
for managed_sub in managed_subclouds:
dc_helper.wait_for_subcloud_keypair(subcloud=managed_sub, expected_keypair=expt_keypair,
timeout=30, check_interval=10)
LOG.tc_step("Ensure new keypair is not synced to unmanaged subcloud: {}".format(primary_subcloud))
code_keypair = dc_helper.wait_for_subcloud_keypair(subcloud=primary_subcloud,
expected_keypair=expt_keypair,
timeout=15, check_interval=5, fail_ok=True)[0]
assert code_keypair == 1, "keypair is updated unexpectedly on unmanaged subcloud {}".format(primary_subcloud)
LOG.tc_step('Re-manage {} and ensure keypair syncs over'.format(primary_subcloud))
dc_helper.manage_subcloud(subcloud=primary_subcloud, check_first=False)
dc_helper.wait_for_subcloud_keypair(subcloud=primary_subcloud, expected_keypair=expt_keypair)
def test_dc_swact_host(swact_precheck, check_central_alarms):
"""
Test host swact on central region
Args:
swact_precheck(fixture): check subclouds managed and online
Setup:
- Ensure primary subcloud is managed
Test Steps:
- Unmanage primary subcloud
- Swact the host
- Verify subclouds are managed
Teardown:
- Manage unmanaged subclouds
"""
primary_subcloud, managed_subcloud = swact_precheck
ssh_central = ControllerClient.get_active_controller(name="RegionOne")
LOG.tc_step("Unmanage {}".format(primary_subcloud))
dc_helper.unmanage_subcloud(subcloud=primary_subcloud, check_first=True)
LOG.tc_step("Swact host on central region")
central_auth = Tenant.get('admin_platform', dc_region='RegionOne')
host_helper.swact_host(auth_info=central_auth)
LOG.tc_step("Check subclouds after host swact on central region")
for managed_subcloud in managed_subcloud:
dc_helper.wait_for_subcloud_status(subcloud=managed_subcloud,
avail=SubcloudStatus.AVAIL_ONLINE,
mgmt=SubcloudStatus.MGMT_MANAGED,
sync=SubcloudStatus.SYNCED,
con_ssh=ssh_central)
LOG.tc_step("Manage {}".format(primary_subcloud))
dc_helper.manage_subcloud(subcloud=primary_subcloud, check_first=True)
dc_helper.wait_for_subcloud_status(subcloud=primary_subcloud,
avail=SubcloudStatus.AVAIL_ONLINE,
mgmt=SubcloudStatus.MGMT_MANAGED,
sync=SubcloudStatus.SYNCED,
con_ssh=ssh_central)
def test_dc_ntp_modify(ntp_precheck):
"""
Update NTP servers on central region and check it is propagated to subclouds
Args:
ntp_precheck (fixture for test setup and teardown)
Setups:
- Ensure primary subcloud is manged and NTP config is in sync with central region
- Un-manage rest of the subclouds except one
Test Steps:
- Un-manage primary subcloud
- Configure NTP servers on above unmanaged subcloud to remove the first NTP server
- Configure NTP servers on central region to add an invalid server 8.8.8.8
- Lock/unlock controllers on central region to apply the config
- Wait for new NTP config to sync over to the only managed osubcloud and config out-of-date alarms appear
- Lock/unlock controllers on managed subcloud to apply config
- Ensure central NTP config does not sync to unmanaged primary subcloud
- Re-manage primary subcloud and ensure NTP config syncs over
- Lock/unlock controllers in primary subcloud to apply new NTP configuration
- Verify fm alarm 100.114 appears for invalid/unreachable NTP server on central region and managed
Teardown:
- Reset NTP servers to original value
- Lock/unlock controllers on all managed subclouds to clear the config out of date alarm
- Re-manage subclouds that were umanaged in setup
- Verify no config out-of-date alarms on the re-managed subclouds
"""
primary_subcloud, managed_subcloud, prev_central_ntp = ntp_precheck
new_central_ntp = ['8.8.8.8'] + prev_central_ntp[:-1]
local_subcloud_ntp = prev_central_ntp[1:]
central_auth = Tenant.get('admin_platform', dc_region='RegionOne')
primary_sub_auth = Tenant.get('admin_platform', dc_region=primary_subcloud)
auth_list = [central_auth, primary_sub_auth]
if managed_subcloud:
managed_sub_auth = Tenant.get('admin_platform', dc_region=managed_subcloud)
auth_list.append(managed_sub_auth)
LOG.tc_step("Unmanage {}".format(primary_subcloud))
dc_helper.unmanage_subcloud(subcloud=primary_subcloud, check_first=True)
LOG.tc_step("While {} is unmanaged, modify its NTP servers locally from {} to {}".
format(primary_subcloud, prev_central_ntp, local_subcloud_ntp))
system_helper.modify_ntp(ntp_servers=local_subcloud_ntp, auth_info=primary_sub_auth)
LOG.tc_step("Reconfigure NTP servers on central region from {} to {}".format(prev_central_ntp, new_central_ntp))
system_helper.modify_ntp(ntp_servers=new_central_ntp, auth_info=central_auth)
if managed_subcloud:
LOG.tc_step("Wait for new NTP config to sync over to managed subcloud: {}".format(managed_subcloud))
dc_helper.wait_for_subcloud_ntp_config(subcloud=managed_subcloud, expected_ntp=new_central_ntp)
LOG.tc_step("Ensure NTP config is not updated on unmanaged subcloud: {}".format(primary_subcloud))
code = dc_helper.wait_for_subcloud_ntp_config(subcloud=primary_subcloud, expected_ntp=new_central_ntp,
timeout=60, fail_ok=True, clear_alarm=False)[0]
assert 1 == code, "Actual return code: {}".format(code)
assert local_subcloud_ntp == system_helper.get_ntp_servers(auth_info=primary_sub_auth)
LOG.tc_step('Re-manage {} and ensure NTP config syncs over'.format(primary_subcloud))
dc_helper.manage_subcloud(subcloud=primary_subcloud, check_first=False)
dc_helper.wait_for_subcloud_ntp_config(subcloud=primary_subcloud, expected_ntp=new_central_ntp)
LOG.tc_step('Verify NTP alarm appeared for invalid server 8.8.8.8 on central and managed subclouds')
for auth_info in auth_list:
system_helper.wait_for_alarm(alarm_id=EventLogID.NTP_ALARM, auth_info=auth_info, timeout=660)
def ntp_precheck(request, check_alarms):
LOG.info("Gather NTP config and subcloud management info")
central_auth = Tenant.get('admin_platform', dc_region='RegionOne')
central_ntp = system_helper.get_ntp_servers(auth_info=central_auth)
primary_subcloud = ProjVar.get_var('PRIMARY_SUBCLOUD')
subcloud_auth = Tenant.get('admin_platform', dc_region=primary_subcloud)
subcloud_ntp = system_helper.get_ntp_servers(auth_info=subcloud_auth)
if not central_ntp == subcloud_ntp:
dc_helper.wait_for_subcloud_ntp_config(subcloud=primary_subcloud)
managed_subclouds = dc_helper.get_subclouds(mgmt='managed', avail='online')
ssh_map = ControllerClient.get_active_controllers_map()
managed_subclouds = [subcloud for subcloud in managed_subclouds if subcloud in ssh_map]
if primary_subcloud in managed_subclouds:
managed_subclouds.remove(primary_subcloud)
managed_subcloud = None
if managed_subclouds:
managed_subcloud = managed_subclouds.pop()
LOG.fixture_step("Leave only one subcloud besides primary subcloud to be managed: {}".format(managed_subcloud))
subclouds_to_revert = []
if managed_subclouds:
LOG.info("Unmange: {}".format(managed_subclouds))
for subcloud in managed_subclouds:
if not system_helper.get_alarms(alarm_id=EventLogID.CONFIG_OUT_OF_DATE,
auth_info=Tenant.get('admin_platform', subcloud)):
subclouds_to_revert.append(subcloud)
dc_helper.unmanage_subcloud(subcloud)
def revert():
reverted = False
try:
LOG.fixture_step("Manage primary subcloud {} if unmanaged".format(primary_subcloud))
dc_helper.manage_subcloud(primary_subcloud)
LOG.fixture_step("Revert NTP config if changed")
res = system_helper.modify_ntp(ntp_servers=central_ntp, auth_info=central_auth, check_first=True,
clear_alarm=False)[0]
if res != -1:
LOG.fixture_step("Lock unlock config out-of-date hosts in central region")
system_helper.wait_and_clear_config_out_of_date_alarms(auth_info=central_auth,
wait_with_best_effort=True)
LOG.fixture_step("Lock unlock config out-of-date hosts in {}".format(primary_subcloud))
dc_helper.wait_for_subcloud_ntp_config(subcloud=primary_subcloud, expected_ntp=central_ntp,
clear_alarm=True)
if managed_subcloud:
LOG.fixture_step("Lock unlock config out-of-date hosts in {}".format(managed_subcloud))
dc_helper.wait_for_subcloud_ntp_config(subcloud=managed_subcloud, expected_ntp=central_ntp,
clear_alarm=True)
if subclouds_to_revert:
LOG.fixture_step("Manage unmanaged subclouds and check they are unaffected")
for subcloud in subclouds_to_revert:
dc_helper.manage_subcloud(subcloud)
assert not system_helper.get_alarms(alarm_id=EventLogID.CONFIG_OUT_OF_DATE,
auth_info=Tenant.get('admin_platform', dc_region=subcloud))
reverted = True
finally:
if not reverted:
for subcloud in subclouds_to_revert:
dc_helper.manage_subcloud(subcloud)
request.addfinalizer(revert)
return primary_subcloud, managed_subcloud, central_ntp
def test_dc_dns_override_local_change(ensure_synced):
"""
Verify DNS modification on subcloud will be overridden by central region config
Args:
ensure_synced: test fixture
Setups:
- Ensure primary subcloud is managed and DNS config is valid and synced
Test Steps:
- Un-manage primary subcloud
- Configure DNS servers on primary subcloud to a unreachable ip address (8.4.4.4)
- Wait for sync log for any managed subcloud with best effort
- Ensure DNS config is not updated on unmanaged primary subcloud
- Verify nslookup passes on central region and fails on primary subcloud
- Re-manage primary subcloud and ensure DNS config syncs over
- Verify nslookup in Central Region and primary subcloud are working as expected
Teardown:
- Manage primary subcloud if not managed (module)
- Reset DNS servers to original value on central region (module)
"""
primary_subcloud, managed_subclouds, sc_dns = ensure_synced
new_dns_servers = compose_new_dns_servers(scenario='unreachable_server',
prev_dns_servers=sc_dns)
LOG.tc_step("Unmanage {}".format(primary_subcloud))
dc_helper.unmanage_subcloud(subcloud=primary_subcloud, check_first=True)
LOG.tc_step("Reconfigure DNS on {} from {} to {}".format(
primary_subcloud, sc_dns, new_dns_servers))
system_helper.set_dns_servers(new_dns_servers,
auth_info=Tenant.get(
'admin_platform',
dc_region=primary_subcloud))
managed_cloud = managed_subclouds[0] if managed_subclouds else ''
LOG.tc_step(
"Wait for sync update log for managed subcloud {} with best effort".
format(managed_cloud))
dc_helper.wait_for_sync_audit(subclouds=managed_cloud,
fail_ok=True,
timeout=660)
LOG.tc_step(
"Ensure DNS config is not updated on unmanaged subcloud: {}".format(
primary_subcloud))
code = dc_helper.wait_for_subcloud_dns_config(subcloud=primary_subcloud,
expected_dns=sc_dns,
fail_ok=True,
timeout=60)[0]
assert 1 == code, "Actual return code: {}".format(code)
LOG.tc_step("Verify nslookup fails on {}".format(primary_subcloud))
central_res, local_res = verify_dns_on_central_and_subcloud(
primary_subcloud, fail_ok=True, sc_dns=sc_dns)
assert 0 == central_res, "nslookup failed on central region"
assert 1 == local_res, "nslookup succeeded on {} with unreachable DNS servers configured".\
format(primary_subcloud)
central_auth = Tenant.get('admin_platform', dc_region='RegionOne')
if system_helper.get_standby_controller_name(auth_info=central_auth):
LOG.tc_step("Swact in central region")
host_helper.swact_host(auth_info=central_auth)
LOG.tc_step(
'Re-manage {} and ensure local DNS config is overridden by central config'
.format(primary_subcloud))
dc_helper.manage_subcloud(subcloud=primary_subcloud, check_first=False)
dc_helper.wait_for_subcloud_dns_config(subcloud=primary_subcloud,
expected_dns=sc_dns)
LOG.tc_step('Verify nslookup works in Central Region and {}'.format(
primary_subcloud))
verify_dns_on_central_and_subcloud(primary_subcloud, sc_dns=sc_dns)
def test_dc_snmp(snmp_precheck):
"""
Update DNS servers on central region and check it is propagated to subclouds
Args:
snmp_precheck: test fixture for setup/teardown
Setups:
- Ensure primary subcloud is managed and SNMP config is synced
Test Steps:
- Un-manage primary subcloud
- Add a SNMP community string and a trapdest on unmanaged subcloud locally
- Add a different SNMP community string and trapdest on central region
- Wait for new SNMP configs to sync over to managed online subclouds
- Ensure central SNMP configs are not updated on unmanaged primary subcloud
- Re-manage primary subcloud and ensure DNS config syncs over
- Verify nslookup works in Central Region and primary subcloud
Teardown:
- Delete DNS servers to original value (module)
"""
primary_subcloud, managed_subclouds, central_comms, central_trapdests = snmp_precheck
central_auth = Tenant.get('admin_platform', dc_region='RegionOne')
sub_auth = Tenant.get('admin_platform', dc_region=primary_subcloud)
LOG.tc_step("Unmanage {}".format(primary_subcloud))
dc_helper.unmanage_subcloud(subcloud=primary_subcloud, check_first=False)
LOG.tc_step(
'Add SNMP community string and trapdest to unmanaged subcloud - {}'.
format(primary_subcloud, SNMP_COMM))
system_helper.create_snmp_comm('cgcsauto_comm_local', auth_info=sub_auth)
system_helper.create_snmp_trapdest(comm_string="cgcsauto_trapdest_local",
ip_addr='8.8.8.8',
auth_info=sub_auth)
LOG.tc_step('Add SNMP community string and trapdest to central region')
system_helper.create_snmp_comm(SNMP_COMM, auth_info=central_auth)
system_helper.create_snmp_trapdest(comm_string=SNMP_TRAPDEST[0],
ip_addr=SNMP_TRAPDEST[1],
auth_info=central_auth)
LOG.tc_step(
"Wait for new SNMP config to sync over to managed subclouds: {}".
format(managed_subclouds))
expt_comms = central_comms + [SNMP_COMM]
expt_trapdests = central_trapdests + [SNMP_TRAPDEST[1]]
dc_helper.wait_for_sync_audit(subclouds=managed_subclouds)
for managed_sub in managed_subclouds:
dc_helper.wait_for_subcloud_snmp_comms(subcloud=managed_sub,
expected_comms=expt_comms,
timeout=30,
check_interval=10)
dc_helper.wait_for_subcloud_snmp_trapdests(
subcloud=managed_sub,
expected_trapdests=expt_trapdests,
timeout=30,
check_interval=10)
LOG.tc_step(
"Ensure central SNMP config is not synced to unmanaged subcloud: {}".
format(primary_subcloud))
code_comm = dc_helper.wait_for_subcloud_snmp_comms(
subcloud=primary_subcloud,
expected_comms=expt_comms,
timeout=15,
check_interval=5,
fail_ok=True)[0]
code_trapdest = dc_helper.wait_for_subcloud_snmp_trapdests(
subcloud=primary_subcloud,
expected_trapdests=expt_trapdests,
timeout=15,
check_interval=5,
fail_ok=True)[0]
assert code_comm == 1, "SNMP comm is updated unexpectedly on unmanaged subcloud {}".format(
primary_subcloud)
assert code_trapdest == 1, "SNMP trapdest is updated unexpectedly on unmanaged subcloud {}".format(
primary_subcloud)
LOG.tc_step('Re-manage {} and ensure DNS config syncs over'.format(
primary_subcloud))
dc_helper.manage_subcloud(subcloud=primary_subcloud, check_first=False)
dc_helper.wait_for_subcloud_snmp_comms(subcloud=primary_subcloud,
expected_comms=expt_comms)
dc_helper.wait_for_subcloud_snmp_trapdests(
subcloud=primary_subcloud,
expected_trapdests=expt_trapdests,
timeout=30,
check_interval=10)