def backup_sysconfig_images(backup_info):
"""
Backup system images on storage lab
Args:
backup_info - settings for doing system backup
Returns:
None
"""
backup_dest = backup_info['backup_dest']
backup_dest_path = backup_info['backup_dest_full_path']
dest_server = backup_info['dest_server']
copy_to_usb = backup_info['copy_to_usb']
install_helper.backup_system(backup_dest=backup_dest,
backup_dest_path=backup_dest_path,
dest_server=dest_server,
copy_to_usb=copy_to_usb)
# storage lab start backup image files separately if it's a storage lab
# if number of storage nodes is greater than 0
if len(system_helper.get_storage_nodes()) > 0:
LOG.tc_step("Storage lab detected. copying images to backup.")
image_ids = glance_helper.get_images()
for img_id in image_ids:
prop_key = 'store'
image_properties = glance_helper.get_image_properties(
img_id, prop_key, rtn_dict=True)
LOG.debug('image store backends:{}'.format(image_properties))
if image_properties and image_properties.get(prop_key,
None) == 'rbd':
LOG.info('rbd based image, exporting it: {}, store:{}'.format(
img_id, image_properties))
install_helper.export_image(
img_id,
backup_dest=backup_info['backup_dest'],
backup_dest_path=backup_info['backup_dest_full_path'],
dest_server=backup_info['dest_server'],
copy_to_usb=backup_info['copy_to_usb'])
else:
LOG.warn(
'No property found!!! for image {}, properties:{}'.format(
img_id, image_properties))
prop_key = 'direct_url'
direct_url = glance_helper.get_image_properties(
img_id, prop_key)[0]
LOG.info(
'found direct_url, still consider it as rbd based image, exporting it: {}, stores:{}'
.format(img_id, image_properties))
if direct_url and direct_url.startswith('rbd://'):
install_helper.export_image(
img_id,
backup_dest=backup_dest,
backup_dest_path=backup_dest_path,
dest_server=dest_server,
copy_to_usb=copy_to_usb)
else:
LOG.warn(
'non-rbd based image, skip it: {}, store:{}'.format(
img_id, image_properties))
def test_robustness_service_function_chaining(protocol, nsh_aware, same_host,
add_protocol, symmetric,
check_system,
add_admin_role_module):
"""
Test Service Function Chaining
Test Steps:
- Check if the system is compatible
- Boot the source VM, dest VM & SFC VM in same host or diff host based on <same_host: True or False>
- Install necessary software and package inside guest for packet forwarding test
- Create port pair using nsh_ware <True:False>
- Create port pair group
- Create SFC flow classifier using protocol <tcp:icmp:udp>
- Create port Chain
- Check packet forwarding from source to dest vm via SFC vm
- Migrate VM by force_lock compute host
- Check packet forwarding from source to dest vm via SFC vm
- Create new flow classifier with new protocol (add_protocol)
- Update port chain with new flow classifier
- Check packet forwarding from source to dest vm via SFC vm with new classifier
- Evacuate VM by rebooting compute host
- Verify VM evacuated
- Check packet forwarding from source to dest vm via SFC vm with new classifier
Test Teardown:
- Delete port chain, port pair group, port pair, flow classifier, vms, volumes created
"""
nsh_aware = True if nsh_aware == 'nsh_aware' else False
same_host = True if same_host == 'same_host' else False
symmetric = True if symmetric == 'symmetric' else False
LOG.tc_step("Check if the system is compatible to run this test")
computes = check_system
LOG.tc_step("Boot the VM in same host: {}".format(same_host))
hosts_to_boot = [computes[0]] * 3 if same_host else computes[0:3]
LOG.info("Boot the VM in following compute host 1:{}, 2:{}, 3:{}".format(
hosts_to_boot[0], hosts_to_boot[1], hosts_to_boot[2]))
LOG.tc_step("Boot the source and dest VM")
vm_ids = []
vm_ids, source_vm_id, dest_vm_id, internal_net_id, mgmt_net_id, mgmt_nic = _setup_vm(
vm_ids, hosts_to_boot)
vm_helper.ping_vms_from_vm(to_vms=source_vm_id,
from_vm=dest_vm_id,
net_types=['mgmt'],
retry=10)
LOG.tc_step("Boot the SFC VM")
sfc_vm_ids = []
sfc_vm_ids, sfc_vm_under_test, ingress_port_id, egress_port_id = _setup_sfc_vm(
sfc_vm_ids, hosts_to_boot, mgmt_nic, internal_net_id)
vm_helper.ping_vms_from_vm(to_vms=source_vm_id,
from_vm=sfc_vm_under_test,
net_types=['mgmt'],
retry=10)
# if protocol != 'icmp':
LOG.tc_step("Install software package nc in vm {} {}".format(
source_vm_id, dest_vm_id))
_install_sw_packages_in_vm(source_vm_id)
_install_sw_packages_in_vm(dest_vm_id)
LOG.tc_step("copy vxlan tool in sfc vm {}".format(sfc_vm_under_test))
vm_helper.scp_to_vm_from_natbox(vm_id=sfc_vm_under_test,
source_file='/home/cgcs/sfc/vxlan_tool.py',
dest_file='/root/vxlan_tool.py')
LOG.tc_step("Create port pair")
port_pair_ids = []
port_pair_id = _setup_port_pair(nsh_aware, ingress_port_id, egress_port_id)
port_pair_ids.append(port_pair_id)
LOG.tc_step("Create port pair group")
port_pair_group_ids = []
port_pair_group_id = _setup_port_pair_groups(port_pair_id)
port_pair_group_ids.append(port_pair_group_id)
name = 'sfc_flow_classifier'
LOG.tc_step("Create flow classifier:{}".format(name))
flow_classifier, dest_vm_internal_net_ip = _setup_flow_classifier(
name, source_vm_id, dest_vm_id, protocol)
LOG.tc_step("Create port chain")
port_chain_id = _setup_port_chain(port_pair_group_id, flow_classifier,
symmetric)
LOG.tc_step(
"Execute vxlan.py tool and verify {} packet received VM1 to VM2".
format(protocol))
_check_packets_forwarded_in_sfc_vm(source_vm_id,
dest_vm_id,
sfc_vm_ids,
dest_vm_internal_net_ip,
protocol,
nsh_aware,
symmetric,
load_balancing=False)
LOG.tc_step("Force lock {}".format(hosts_to_boot))
if not same_host:
for host_to_boot in hosts_to_boot:
HostsToRecover.add(host_to_boot)
lock_code, lock_output = host_helper.lock_host(host_to_boot,
force=True,
check_first=True)
assert lock_code == 0, "Failed to force lock {}. Details: {}".format(
host_to_boot, lock_output)
else:
HostsToRecover.add(hosts_to_boot[0])
lock_code, lock_output = host_helper.lock_host(hosts_to_boot[0],
force=True,
check_first=True)
assert lock_code == 0, "Failed to force lock {}. Details: {}".format(
hosts_to_boot[0], lock_output)
# Expect VMs to migrate off force-locked host (non-gracefully)
LOG.tc_step(
"Wait for 'Active' status of VMs after host force lock completes")
vm_helper.wait_for_vms_values(vm_ids, fail_ok=False)
LOG.tc_step(
"Execute vxlan.py tool and verify {} packet received VM1 to VM2".
format(protocol))
_check_packets_forwarded_in_sfc_vm(source_vm_id,
dest_vm_id,
sfc_vm_ids,
dest_vm_internal_net_ip,
protocol,
nsh_aware,
symmetric,
load_balancing=False)
LOG.tc_step(
"Create new flow classifier with protocol {}".format(add_protocol))
flow_classifier_name = 'new_sfc_flow_classifier'
new_flow_classifier, dest_vm_internal_net_ip = _setup_flow_classifier(
flow_classifier_name, source_vm_id, dest_vm_id, add_protocol)
LOG.tc_step("Update port chain with new flow classifier:".format(
new_flow_classifier))
network_helper.set_sfc_port_chain(port_chain_id,
port_pair_groups=port_pair_group_id,
flow_classifiers=new_flow_classifier)
LOG.tc_step(
"Execute vxlan.py tool and verify {} packet received VM1 to VM2".
format(add_protocol))
_check_packets_forwarded_in_sfc_vm(source_vm_id,
dest_vm_id,
sfc_vm_ids,
dest_vm_internal_net_ip,
add_protocol,
nsh_aware,
symmetric,
load_balancing=False)
LOG.info("Get the host to reboot where the VMs launched")
hosts_to_reboot = vm_helper.get_vms_hosts(vm_ids=vm_ids)
LOG.tc_step(
"Reboot VMs host {} and ensure vms are evacuated to other host".format(
hosts_to_reboot))
vm_helper.evacuate_vms(host=hosts_to_reboot,
vms_to_check=vm_ids,
ping_vms=True)
LOG.tc_step(
"Execute vxlan.py tool and verify {} packet received VM1 to VM2".
format(add_protocol))
_check_packets_forwarded_in_sfc_vm(source_vm_id,
dest_vm_id,
sfc_vm_ids,
dest_vm_internal_net_ip,
add_protocol,
nsh_aware,
symmetric,
load_balancing=False)
def _check_packets_forwarded_in_sfc_vm(source_vm_id,
dest_vm_id,
sfc_vm_ids,
dest_vm_internal_net_ip,
protocol,
nsh_aware,
symmetric,
load_balancing=False):
end_event = Events("Hello or ping sent to vm")
start_event = Events("VM {} started listening".format(dest_vm_id))
received_event = Events("Greeting received on vm {}".format(dest_vm_id))
vms_events = {}
for sfc_vm in sfc_vm_ids:
start_event_sfc = Events("SFC vm {} started listening".format(sfc_vm))
received_event_sfc = Events(
"Packets received on SFC vm {}".format(sfc_vm))
vms_events[sfc_vm] = (start_event_sfc, received_event_sfc)
greeting = "hello"
port = 20010
vm_thread = None
if protocol != 'icmp':
func_args = (start_event, end_event, received_event, dest_vm_id,
dest_vm_internal_net_ip, greeting, port, protocol,
load_balancing)
vm_thread = MThread(_ssh_to_dest_vm_and_wait_for_greetings, *func_args)
sfc_vm_threads = []
for sfc_vm in sfc_vm_ids:
start_event_sfc, received_event_sfc = vms_events[sfc_vm]
func_args = (start_event_sfc, end_event, received_event_sfc, sfc_vm,
protocol, nsh_aware, symmetric)
sfc_vm_thread = MThread(_ssh_to_sfc_vm_and_wait_for_packets,
*func_args)
sfc_vm_threads.append(sfc_vm_thread)
LOG.tc_step(
"Starting VM ssh session threads to ping (icmp) or send hello (tcp, udp)"
)
if protocol != 'icmp':
vm_thread.start_thread()
for sfc_vm_thread in sfc_vm_threads:
LOG.tc_step("Starting each SFC VM threads")
sfc_vm_thread.start_thread()
try:
if protocol != 'icmp':
start_event.wait_for_event(timeout=180, fail_ok=False)
for sfc_vm in sfc_vm_ids:
start_event_sfc, received_event_sfc = vms_events[sfc_vm]
start_event_sfc.wait_for_event(timeout=120, fail_ok=False)
if protocol == 'icmp':
LOG.tc_step(
"Ping from from vm {} to vm {}, and check it's received".
format(source_vm_id, dest_vm_id))
_ping_from_source_to_dest_vm(source_vm_id, end_event,
dest_vm_internal_net_ip)
else:
if load_balancing:
LOG.tc_step(
"Send Hello msg from vm using tcp_client.py {} to vm {}, and check it's received"
.format(source_vm_id, dest_vm_id))
_send_hello_message_from_vm_using_tcp_client(
source_vm_id, end_event, dest_vm_internal_net_ip)
else:
LOG.tc_step(
"Send Hello msg from vm {} to vm {}, and check it's received"
.format(source_vm_id, dest_vm_id))
_send_hello_message_from_vm(source_vm_id, greeting, end_event,
dest_vm_internal_net_ip, port,
protocol)
if protocol != 'icmp':
assert received_event.wait_for_event(
timeout=30), "Received Event {} is not set".format(
received_event)
for sfc_vm in sfc_vm_ids:
start_event_sfc, received_event_sfc = vms_events[sfc_vm]
assert received_event_sfc.wait_for_event(
timeout=10), "Received Event is not set in SFC function"
finally:
end_event.set()
if protocol != 'icmp':
vm_thread.wait_for_thread_end(timeout=40, fail_ok=False)
for sfc_vm_thread in sfc_vm_threads:
sfc_vm_thread.wait_for_thread_end(timeout=40, fail_ok=False)
def test_dc_fault_scenario(subcloud_to_test):
"""
Test Fault Scenario on Distributed Cloud
Args:
subcloud_to_test (str): module fixture
Setup:
- Make sure there is consistency between alarm summary on
Central Cloud and on subclouds
Test Steps:
- Make subcloud offline (e. g. delete route)
Step1:
- Ensure suncloud shows offline
Step2:
- Raise alarm on subcloud
- Ensure relative alarm raised on subcloud,
- Ensure system alarm-summary on subcloud has changed
- Ensure dcmanager alarm summary on system controller has no change
Step3:
- Resume connectivity to subcloud (e. g. add route back)
- Ensure suncloud shows online and in-sync
- Ensure system alarm-summary on subcloud matches dcmanager alarm summary on system
controller
Step4:
- Clean alarm on subcloud
- Ensure relative alarm cleared on subcloud
- Ensure system alarm-summary on subcloud matches dcmanager alarm summary on system
controller
"""
ssh_central = ControllerClient.get_active_controller(name="RegionOne")
ssh_subcloud = ControllerClient.get_active_controller(
name=subcloud_to_test)
subcloud_table = {}
try:
code, output = cli.dcmanager(
"subcloud show {}".format(subcloud_to_test),
ssh_client=ssh_central)
gateway = table_parser.get_value_two_col_table(
table_parser.table(output), "management_gateway_ip")
code, hosts_raw = cli.system("host-list", ssh_client=ssh_subcloud)
hosts_id = table_parser.get_values(table_parser.table(hosts_raw), 'id')
for host_id in hosts_id:
code, route_raw = cli.system("host-route-list {}".format(host_id),
ssh_client=ssh_subcloud)
route_table = table_parser.filter_table(
table_parser.table(route_raw), **{'gateway': gateway})
subcloud_table[host_id] = route_table
LOG.tc_step(
"Delete route for subcloud: {} and wait for it to go offline.".
format(subcloud_to_test))
ssh_subcloud = ControllerClient.get_active_controller(
name=subcloud_to_test)
for host_id in subcloud_table:
command = "host-route-delete {}".format(
table_parser.get_values(subcloud_table[host_id], "uuid")[0])
cli.system(command, ssh_client=ssh_subcloud)
dc_helper.wait_for_subcloud_status(subcloud_to_test,
avail=SubcloudStatus.AVAIL_OFFLINE,
timeout=DCTimeout.SYNC,
con_ssh=ssh_central)
LOG.tc_step("Raise alarm on subcloud: {}".format(subcloud_to_test))
ssh_subcloud = ControllerClient.get_active_controller(
name=subcloud_to_test)
code_sub_before, output_sub_before = cli.fm("alarm-summary",
ssh_client=ssh_subcloud)
code_central_before, output_central_before = cli.dcmanager(
'alarm summary')
ssh_subcloud.exec_cmd(
"fmClientCli -c \"### ###300.005###clear###system.vm###host="
"testhost-0### ###critical### ###processing-error###cpu-cycles-limit-exceeded"
"### ###True###True###'\"",
fail_ok=False)
LOG.info("Ensure relative alarm was raised at subcloud: {}".format(
subcloud_to_test))
system_helper.wait_for_alarm(
alarm_id=EventLogID.PROVIDER_NETWORK_FAILURE, con_ssh=ssh_subcloud)
code_sub_after, output_sub_after = cli.fm("alarm-summary",
ssh_client=ssh_subcloud)
code_central_after, output_central_after = cli.dcmanager(
'alarm summary')
LOG.info(
"Ensure fm alarm summary on subcloud: {} has changed but dcmanager alarm"
"summary has not changed".format(subcloud_to_test))
assert output_central_before == output_central_after and output_sub_before != \
output_sub_after
add_routes_to_subcloud(subcloud_to_test, subcloud_table)
dc_helper.wait_for_subcloud_status(subcloud_to_test,
avail=SubcloudStatus.AVAIL_ONLINE,
sync=SubcloudStatus.SYNCED,
timeout=DCTimeout.SYNC,
con_ssh=ssh_central)
alarm_summary_add_and_del(subcloud_to_test)
finally:
cli.dcmanager("subcloud show {}".format(subcloud_to_test),
ssh_client=ssh_central,
fail_ok=True)
add_routes_to_subcloud(subcloud_to_test, subcloud_table, fail_ok=True)
LOG.info("Clear alarm on subcloud: {}".format(subcloud_to_test))
ssh_subcloud.exec_cmd('fmClientCli -D host=testhost-0')
check_alarm_summary_match_subcloud(subcloud=subcloud_to_test)
def test_horizon_update_compute_defaults(defaults_pg):
"""
Tests the Update Compute Default Quotas functionality:
Setups:
- Login as Admin
- Go to Admin > System > Defaults
Teardown:
- Back to Defaults page
- Logout
Test Steps:
- Go to Compute Quotas Tab
- Updates default Compute Quotas by adding a random number between 1 and 10
- Verifies that the updated values are present in the
Quota Compute Defaults table
- Updates default Compute Quotas back to original status
- Go to Volume Quotas Tab
- Updates default Volume Quotas by adding a random number between 1 and 10
- Verifies that the updated values are present in the
Quota Volume Defaults table
- Updates default Volume Quotas back to original status
"""
add_up = random.randint(1, 10)
defaults_pg.go_to_compute_quotas_tab()
default_compute_quota_values = defaults_pg.compute_quota_values
LOG.tc_step('Updates default Compute Quotas by add {}.'.format(add_up))
defaults_pg.update_compute_defaults(add_up)
assert defaults_pg.find_message_and_dismiss(messages.SUCCESS)
assert not defaults_pg.find_message_and_dismiss(messages.ERROR)
LOG.tc_step(
'Verifies that the updated values are present in the Quota Default Computes table'
)
assert len(default_compute_quota_values) > 0
for quota_name in default_compute_quota_values:
assert defaults_pg.is_compute_quota_a_match(
quota_name, default_compute_quota_values[quota_name] + add_up)
LOG.tc_step('Updates default Compute Quotas back to original status')
time.sleep(1)
defaults_pg.update_compute_defaults(-add_up)
defaults_pg.go_to_volume_quotas_tab()
default_volume_quota_values = defaults_pg.volume_quota_values
LOG.tc_step('Updates default Volume Quotas by add {}.'.format(add_up))
defaults_pg.update_volume_defaults(add_up)
assert defaults_pg.find_message_and_dismiss(messages.SUCCESS)
assert not defaults_pg.find_message_and_dismiss(messages.ERROR)
LOG.tc_step(
'Verifies that the updated values are present in the Quota Default Volumes table'
)
assert len(default_volume_quota_values) > 0
for quota_name in default_volume_quota_values:
assert defaults_pg.is_volume_quota_a_match(
quota_name, default_volume_quota_values[quota_name] + add_up)
LOG.tc_step('Updates default Volume Quotas back to original status')
time.sleep(1)
defaults_pg.update_volume_defaults(-add_up)
horizon.test_result = True
def test_instantiate_a_vm_with_a_large_volume_and_cold_migrate(
vms_, pre_alarm_):
"""
Test instantiate a vm with a large volume ( 20 GB and 40 GB) and cold migrate:
Args:
vms_ (dict): vms created by vms_ fixture
pre_alarm_ (list): alarm lists obtained by pre_alarm_ fixture
Test Setups:
- get tenant1 and management networks which are already created for lab setup
- get or create a "small" flavor
- get the guest image id
- create two large volumes (20 GB and 40 GB) in cinder
- boot two vms ( test_inst1, test_inst2) using volumes 20 GB and 40 GB respectively
Test Steps:
- Verify VM status is ACTIVE
- Validate that VMs boot, and that no timeouts or error status occur.
- Verify the VM can be pinged from NATBOX
- Verify login to VM and rootfs (dev/vda) filesystem is rw mode
- Attempt to cold migrate of VMs
- Validate that the VMs migrated and no errors or alarms are present
- Log into both VMs and validate that file systems are read-write
- Terminate VMs
Skip conditions:
- less than two hosts with the same storage backing
- less than two computes
- no storage node
"""
LOG.tc_step("Instantiate a vm with large volume.....")
vms = vms_
for vm in vms:
vm_id = vm['id']
LOG.tc_step(
"Checking VM status; VM Instance id is: {}......".format(vm_id))
vm_state = vm_helper.get_vm_status(vm_id)
assert vm_state == VMStatus.ACTIVE, 'VM {} state is {}; Not in ACTIVATE state as expected'\
.format(vm_id, vm_state)
LOG.tc_step("Verify VM can be pinged from NAT box...")
rc, boot_time = check_vm_boot_time(vm_id)
assert rc, "VM is not pingable after {} seconds ".format(boot_time)
LOG.tc_step("Verify Login to VM and check filesystem is rw mode....")
assert is_vm_filesystem_rw(vm_id), 'rootfs filesystem is not RW as expected for VM {}'\
.format(vm['display_name'])
LOG.tc_step(
"Attempting cold migration; vm id = {}; vm_name = {} ....".format(
vm_id, vm['display_name']))
code, msg = vm_helper.cold_migrate_vm(vm_id=vm_id, fail_ok=True)
LOG.tc_step("Verify cold migration succeeded...")
assert code == 0, "Expected return code 0. Actual return code: {}; details: {}".format(
code, msg)
LOG.tc_step(
"Verifying filesystem is rw mode after cold migration....")
assert is_vm_filesystem_rw(vm_id), 'After cold migration rootfs filesystem is not RW as expected for ' \
'VM {}'.format(vm['display_name'])
def test_admin_password(scenario, less_than_two_cons, _revert_admin_pw):
"""
Test the admin password change
Test Steps:
- lock standby controller change password and unlock
- change password and swact
- check alarms
"""
if 'swact' in scenario and less_than_two_cons:
skip(SkipSysType.LESS_THAN_TWO_CONTROLLERS)
host = system_helper.get_standby_controller_name()
assert host, "No standby controller on system"
if scenario == "lock_standby_change_pswd":
# lock the standby
LOG.tc_step("Attempting to lock {}".format(host))
res, out = host_helper.lock_host(host=host)
LOG.tc_step("Result of the lock was: {}".format(res))
# change password
prev_pswd = Tenant.get('admin')['password']
post_pswd = '!{}9'.format(prev_pswd)
LOG.tc_step('Changing admin password to {}'.format(post_pswd))
keystone_helper.set_user('admin', password=post_pswd, auth_info=Tenant.get(
'admin_platform'))
# assert "Warning: 'admin' password changed. Please wait 5 minutes before Locking/Unlocking
# the controllers" in output
LOG.tc_step("Sleep for 180 seconds after admin password change")
time.sleep(180)
LOG.tc_step("Check admin password is updated in keyring")
assert post_pswd == security_helper.get_admin_password_in_keyring()
if scenario == "change_pswd_swact":
LOG.tc_step("Swact active controller")
host_helper.swact_host()
else:
LOG.tc_step("Unlock host {}".format(host))
res = host_helper.unlock_host(host)
LOG.info("Unlock hosts result: {}".format(res))
LOG.tc_step("Check admin password is updated in keyring")
assert post_pswd == security_helper.get_admin_password_in_keyring()
def test_ping_between_two_vms(guest_os, vm1_vifs, vm2_vifs, check_avs_pattern):
"""
Ping between two vms with given vif models
Test Steps:
- Create a favor with dedicated cpu policy and proper root disk size
- Create a volume from guest image under test with proper size
- Boot two vms with given vif models from above volume and flavor
- Ping VMs from NatBox and between two vms
Test Teardown:
- Delete vms, volumes, flavor, glance image created
"""
reuse = False if 'e1000' in vm1_vifs or 'e1000' in vm2_vifs else True
cleanup = 'function' if not reuse or 'ubuntu' in guest_os else None
image_id = glance_helper.get_guest_image(guest_os,
cleanup=cleanup,
use_existing=reuse)
LOG.tc_step("Create a favor dedicated cpu policy")
flavor_id = nova_helper.create_flavor(name='dedicated',
guest_os=guest_os,
cleanup='function')[1]
nova_helper.set_flavor(flavor_id, **{FlavorSpec.CPU_POLICY: 'dedicated'})
mgmt_net_id = network_helper.get_mgmt_net_id()
tenant_net_id = network_helper.get_tenant_net_id()
internal_net_id = network_helper.get_internal_net_id()
net_ids = (mgmt_net_id, tenant_net_id, internal_net_id)
vms = []
for vifs_for_vm in (vm1_vifs, vm2_vifs):
# compose vm nics
nics = _compose_nics(vifs_for_vm,
net_ids=net_ids,
image_id=image_id,
guest_os=guest_os)
net_types = ['mgmt', 'data', 'internal'][:len(nics)]
LOG.tc_step("Create a volume from {} image".format(guest_os))
vol_id = cinder_helper.create_volume(name='vol-{}'.format(guest_os),
source_id=image_id,
guest_image=guest_os,
cleanup='function')[1]
LOG.tc_step(
"Boot a {} vm with {} vifs from above flavor and volume".format(
guest_os, vifs_for_vm))
vm_id = vm_helper.boot_vm('{}_vifs'.format(guest_os),
flavor=flavor_id,
cleanup='function',
source='volume',
source_id=vol_id,
nics=nics,
guest_os=guest_os)[1]
LOG.tc_step("Ping VM {} from NatBox(external network)".format(vm_id))
vm_helper.wait_for_vm_pingable_from_natbox(vm_id, fail_ok=False)
vms.append(vm_id)
LOG.tc_step(
"Ping between two vms over management, data, and internal networks")
vm_helper.ping_vms_from_vm(to_vms=vms[0],
from_vm=vms[1],
net_types=net_types)
vm_helper.ping_vms_from_vm(to_vms=vms[1],
from_vm=vms[0],
net_types=net_types)
def _test_setting_evacuate_priority(self, operation, priority, expt_error):
LOG.tc_step('Launch VM for test')
if not hasattr(TestPrioritizedVMEvacuation, 'vm_id'):
TestPrioritizedVMEvacuation.vm_id = vm_helper.boot_vm(
avail_zone='cgcsauto', cleanup='class')[1]
vm_id = TestPrioritizedVMEvacuation.vm_id
LOG.info(
'OK, VM launched (or already existing) for test, vm-id:{}'.format(
vm_id))
if priority == 'random':
priority = 'ab' + ''.join(
random.choice(string.ascii_letters + string.digits)
for _ in range(3))
expecting_fail = True if expt_error else False
if operation == 'set':
code, output = set_evacuate_priority(vm_id,
priority,
fail_ok=expecting_fail)
else:
priorities_set = get_vm_priority_metadata(vm_id)
expecting_fail = True if not priorities_set else False
LOG.info(
'Attempt to delete evacuation-priority, expecting {}'.format(
'PASS' if expecting_fail else 'FAIL'))
code, output = delete_evacuate_priority(vm_id,
fail_ok=expecting_fail)
if not expecting_fail:
assert 0 == code, \
'Fail to set Evacuation-priority:{} to VM:{}\ncode={}\noutput={}'.format(
priority, vm_id, code, output)
LOG.info(
'OK, {} Evacuation-Priority was accepted, set to "{}" on VM:{}'
.format(operation, priority, vm_id))
else:
assert 1 == code, \
'Fail to set Evacuation-priority:{} to VM:{}\ncode={}\noutput={}, ' \
'expecting failing, but not'.format(priority, vm_id, code, output)
LOG.info('OK, attempt to change Evacuation-Priority to:"{}" on '
'VM:{} failed as expected'.format(priority, vm_id))
priorities_set = get_vm_priority_metadata(vm_id)
actual_priority = None
if priorities_set and VMMetaData.EVACUATION_PRIORITY in priorities_set:
try:
actual_priority = int(
priorities_set[VMMetaData.EVACUATION_PRIORITY])
except ValueError:
pass
if operation == 'set':
if not expecting_fail:
assert actual_priority == priority, \
'Failed to set Evacuation-Priority, expecting:{}, actual:{}'.format(
priority, actual_priority)
else:
assert actual_priority is None or actual_priority != priority, \
'Failed, expecting Evacuation-Priority not set, but not. ' \
'expecting:{}, actual:{}'.format(priority, actual_priority)
else:
assert actual_priority is None, \
'Failed, expecting Evacuation-Priority been deleted, but not. ' \
'actual:{}'.format(actual_priority)
def launch_heat_stack():
"""
Launch the upgrade heat stacks on TiS server.
It will get the heat template tp TiS
check if the heat stack is already there if not it will create the heat stacks
Args:
Returns (code): 0/1 pass/fail
"""
# TODO: Update this to use a different stack for labs with only 1 or 2 hypervisors.
# check if the heat stack is already launched
stack_id = heat_helper.get_stacks(name=HeatTemplate.SYSTEM_TEST_HEAT_NAME)
if stack_id:
LOG.tc_step("Stack is already there")
return
# make sure heat templates are there in Tis
_get_large_heat()
file_dir = StxPath.CUSTOM_HEAT_TEMPLATES
file_name = HeatTemplate.SYSTEM_TEST_HEAT + "/" + HeatTemplate.SYSTEM_TEST_HEAT_NAME
heat_template_file = file_dir + file_name + "/"
pre_req_stack_name = "pre_req"
stack_id_pre_req = heat_helper.get_stacks(name=pre_req_stack_name,
auth_info=Tenant.get('admin'))
if not stack_id_pre_req:
LOG.tc_step(
"Creating pre-request heat stack to create images and flavors")
default_guest_img = GuestImages.IMAGE_FILES[
GuestImages.DEFAULT['guest']][2]
image_file_path = "file://{}/{}".format(
GuestImages.DEFAULT['image_dir'], default_guest_img)
pre_req_template_path = heat_template_file + "pre_req.yaml"
LOG.info("Creating heat stack for pre-req, images and flavors")
heat_helper.create_stack(stack_name=pre_req_stack_name,
template=pre_req_template_path,
parameters={'LOCATION': image_file_path},
auth_info=Tenant.get('admin'),
cleanup=None)
keypair_stack_name = 'Tenant1_Keypair'
stack_id_key_pair = heat_helper.get_stacks(name=keypair_stack_name)
if not stack_id_key_pair:
LOG.tc_step("Creating Tenant key via heat stack")
keypair_template = 'Tenant1_Keypair.yaml'
keypair_template = '{}/{}'.format(heat_template_file, keypair_template)
heat_helper.create_stack(stack_name=keypair_stack_name,
template=keypair_template,
cleanup=None)
# Now create the large-stack
LOG.tc_step(
"Creating heat stack to launch networks, ports, volumes, and vms")
large_heat_template = heat_template_file + "/templates/rnc/" + "rnc_heat.yaml"
env_file = heat_template_file + "/templates/rnc/" + "rnc_heat.env"
heat_helper.create_stack(stack_name=HeatTemplate.SYSTEM_TEST_HEAT_NAME,
template=large_heat_template,
environments=env_file,
timeout=1800,
cleanup=None)
def test_ceph_mon_process_kill(monitor):
"""
us69932_tc2_ceph_mon_process_kill from us69932_ceph_monitoring.odt
Verify that ceph mon processes recover when they are killed.
Args:
- Nothing
Setup:
- Requires system with storage nodes
Test Steps:
1. Run CEPH pre-check fixture to check:
- system has storage nodes
- health of the ceph cluster is okay
- that we have OSDs provisioned
2. Pick one ceph monitor and remove it from the quorum
3. Kill the monitor process
4. Check that the appropriate alarms are raised
5. Restore the monitor to the quorum
6. Check that the alarms clear
7. Ensure the ceph monitor is restarted under a different pid
Potential flaws:
1. We're not checking if unexpected alarms are raised (TODO)
Teardown:
- None
"""
LOG.tc_step('Get process ID of ceph monitor')
mon_pid = storage_helper.get_mon_pid(monitor)
with host_helper.ssh_to_host(monitor) as host_ssh:
with host_ssh.login_as_root() as root_ssh:
LOG.tc_step('Remove the monitor')
cmd = 'ceph mon remove {}'.format(monitor)
root_ssh.exec_cmd(cmd)
LOG.tc_step('Stop the ceph monitor')
cmd = 'service ceph stop mon.{}'.format(monitor)
root_ssh.exec_cmd(cmd)
LOG.tc_step('Check that ceph monitor failure alarm is raised')
system_helper.wait_for_alarm(alarm_id=EventLogID.STORAGE_DEGRADE,
timeout=300)
with host_helper.ssh_to_host(monitor) as host_ssh:
with host_ssh.login_as_root() as root_ssh:
LOG.tc_step('Get cluster fsid')
cmd = 'ceph fsid'
fsid = host_ssh.exec_cmd(cmd)[0]
ceph_conf = '/etc/ceph/ceph.conf'
LOG.tc_step('Remove old ceph monitor directory')
cmd = 'rm -rf /var/lib/ceph/mon/ceph-{}'.format(monitor)
root_ssh.exec_cmd(cmd)
LOG.tc_step('Re-add the monitor')
cmd = 'ceph-mon -i {} -c {} --mkfs --fsid {}'.format(
monitor, ceph_conf, fsid)
root_ssh.exec_cmd(cmd)
LOG.tc_step('Check the ceph storage alarm condition clears')
system_helper.wait_for_alarm_gone(alarm_id=EventLogID.STORAGE_DEGRADE,
timeout=360)
LOG.tc_step('Check the ceph-mon process is restarted with a different pid')
mon_pid2 = None
for i in range(0, PROC_RESTART_TIME):
mon_pid2 = storage_helper.get_mon_pid(monitor, fail_ok=True)
if mon_pid2 and mon_pid2 != mon_pid:
break
time.sleep(5)
LOG.info('Old pid is {} and new pid is {}'.format(mon_pid, mon_pid2))
msg = 'Process did not restart in time'
assert mon_pid2 and mon_pid2 != mon_pid, msg
def pre_system_backup():
"""
Actions before system backup, including:
- check the USB device is ready if it is the destination
- create folder for the backup files on destination server
- collect logs on the current system
Args:
Returns:
"""
lab = InstallVars.get_install_var('LAB')
LOG.info("Preparing lab for system backup....")
backup_dest = BackupVars.get_backup_var("BACKUP_DEST")
NATBoxClient.set_natbox_client()
_backup_info = {
'backup_dest': backup_dest,
'usb_parts_info': None,
'backup_dest_full_path': None,
'dest_server': None
}
if backup_dest == 'usb':
_backup_info['dest'] = 'usb'
active_controller_name = system_helper.get_active_controller_name()
if active_controller_name != 'controller-0':
msg = "controller-0 is not the active controller"
LOG.info(msg + ", try to swact the host")
host_helper.swact_host(active_controller_name)
active_controller_name = system_helper.get_active_controller_name()
assert active_controller_name == 'controller-0', msg
LOG.fixture_step(
"Checking if a USB flash drive is plugged in controller-0 node... "
)
usb_device = install_helper.get_usb_device_name()
assert usb_device, "No USB found in controller-0"
parts_info = install_helper.get_usb_device_partition_info(
usb_device=usb_device)
part1 = "{}1".format(usb_device)
part2 = "{}2".format(usb_device)
if len(parts_info) < 3:
skip(
"USB {} is not partitioned; Create two partitions using fdisk; partition 1 = {}1, "
"size = 2G, bootable; partition 2 = {}2, size equal to the avaialble space."
.format(usb_device, usb_device, usb_device))
devices = parts_info.keys()
LOG.info("Size of {} = {}".format(
part1, install_helper.get_usb_partition_size(part1)))
if not (part1 in devices
and install_helper.get_usb_partition_size(part1) >= 2):
skip("Insufficient size in {}; at least 2G is required. {}".format(
part1, parts_info))
if not (part2 in devices
and install_helper.get_usb_partition_size(part2) >= 10):
skip("Insufficient size in {}; at least 2G is required. {}".format(
part1, parts_info))
if not install_helper.mount_usb(part2):
skip("Fail to mount USB for backups")
LOG.tc_step("Erasing existing files from USB ... ")
assert install_helper.delete_backup_files_from_usb(
part2), "Fail to erase existing file from USB"
_backup_info['usb_parts_info'] = parts_info
_backup_info['backup_dest_full_path'] = BackupRestore.USB_BACKUP_PATH
elif backup_dest == 'local':
_backup_info['dest'] = 'local'
# save backup files in Test Server which local
backup_dest_path = BackupVars.get_backup_var('BACKUP_DEST_PATH')
backup_dest_full_path = '{}/{}'.format(backup_dest_path,
lab['short_name'])
# ssh to test server
test_server_attr = dict()
test_server_attr['name'] = TestFileServer.get_hostname().split('.')[0]
test_server_attr['server_ip'] = TestFileServer.get_server()
test_server_attr['prompt'] = r'\[{}@{} {}\]\$ '\
.format(TestFileServer.get_user(), test_server_attr['name'], TestFileServer.get_user())
test_server_conn = install_helper.establish_ssh_connection(
test_server_attr['name'],
user=TestFileServer.get_user(),
password=TestFileServer.get_password(),
initial_prompt=test_server_attr['prompt'])
test_server_conn.set_prompt(test_server_attr['prompt'])
test_server_conn.deploy_ssh_key(install_helper.get_ssh_public_key())
test_server_attr['ssh_conn'] = test_server_conn
test_server_obj = Server(**test_server_attr)
_backup_info['dest_server'] = test_server_obj
# test if backup path for the lab exist in Test server
if test_server_conn.exec_cmd(
"test -e {}".format(backup_dest_full_path))[0]:
test_server_conn.exec_cmd(
"mkdir -p {}".format(backup_dest_full_path))
# delete any existing files
test_server_conn.exec_cmd("rm -rf {}/*".format(backup_dest_full_path))
_backup_info['usb_parts_info'] = None
_backup_info['backup_dest_full_path'] = backup_dest_full_path
collect_logs('before_br')
_backup_info['is_storage_lab'] = (len(system_helper.get_storage_nodes()) >
0)
return _backup_info
def backup_load_iso_image(backup_info):
"""
Save a copy of the bootimage.iso for later restore.
Args:
backup_info
Returns:
True - the ISO is successfully copied to backup server
- False otherwise
"""
backup_dest = backup_info['backup_dest']
backup_dest_path = backup_info['backup_dest_full_path']
load_path = ProjVar.get_var('BUILD_PATH')
build_info = system_helper.get_build_info()
build_id = build_info['BUILD_ID']
assert re.match(TIS_BLD_DIR_REGEX, build_id), "Invalid Build Id pattern"
build_server = build_info['BUILD_SERVER']
if not build_server:
build_server = BuildServerPath.DEFAULT_BUILD_SERVER # default
with host_helper.ssh_to_build_server(
bld_srv=build_server) as build_server_conn:
cmd = "test -e " + load_path
assert build_server_conn.exec_cmd(cmd, rm_date=False)[0] == 0, 'The system build {} not found in {}:{}'.\
format(build_id, build_server, load_path)
iso_file_path = os.path.join(load_path, "export",
install_helper.UPGRADE_LOAD_ISO_FILE)
if not build_server_conn.exec_cmd("test -e " + iso_file_path):
LOG.warn("No ISO found on path:{}".format(iso_file_path))
return True
pre_opts = 'sshpass -p "{0}"'.format(HostLinuxUser.get_password())
# build_server_conn.rsync("-L " + iso_file_path, lab['controller-0 ip'],
build_server_conn.rsync("-L " + iso_file_path,
html_helper.get_ip_addr(),
os.path.join(HostLinuxUser.get_home(),
"bootimage.iso"),
pre_opts=pre_opts,
timeout=360)
if backup_dest == 'usb':
usb_part1 = None
usb_partition_info = install_helper.get_usb_device_partition_info()
for k, v in usb_partition_info.items():
if k[-1:] == "1":
usb_part1 = k
if not usb_part1:
LOG.info(
"No partition exist for burning load iso image in usb: {}".
format(usb_partition_info))
return False
# Check if the ISO is uploaded to controller-0
con_ssh = ControllerClient.get_active_controller()
cmd = "test -e " + os.path.join(HostLinuxUser.get_home(),
"bootimage.iso")
assert con_ssh.exec_cmd(
cmd)[0] == 0, 'The bootimage.iso file not found in {}'.format(
HostLinuxUser.get_home())
LOG.tc_step(
"Burning backup load ISO to /dev/{} ...".format(usb_part1))
# Write the ISO to USB
cmd = "echo {} | sudo -S dd if={} of=/dev/{} bs=1M oflag=direct; sync"\
.format(HostLinuxUser.get_password(), os.path.join(HostLinuxUser.get_home(), "bootimage.iso"), usb_part1)
rc, output = con_ssh.exec_cmd(cmd, expect_timeout=900)
if rc == 0:
LOG.info(
" The backup build iso file copied to USB for restore. {}".
format(output))
return True
else:
LOG.error(
"Failed to copy backup build iso file to USB {}: {}".format(
usb_part1, output))
return False
else:
LOG.tc_step(
"Copying load image ISO to local test server: {} ...".format(
backup_dest_path))
common.scp_from_active_controller_to_test_server(
os.path.join(HostLinuxUser.get_home(), "bootimage.iso"),
backup_dest_path)
LOG.info(" The backup build iso file copied to local test server: {}".
format(backup_dest_path))
return True
def test_backup(pre_system_backup):
"""
Test create backup on the system and it's avaliable and in-use volumes.
copy backup files to USB flash drive
Args:
Setup:
- create system backup use config_controller (create system,image tgz)
- backup image separately if its storage lab that use CEPH
- back up all available and in-use volumes from the lab
Test Steps:
- check system and img tgz are created for system backup
- check all images are back up in storage
- check all volumes tgz are created for backup
Teardown:
- Delete vm if booted
- Delete created flavor (module)
"""
backup_info = pre_system_backup
LOG.info('Before backup, perform configuration changes and launch VMs')
con_ssh = ControllerClient.get_active_controller()
backup_info['con_ssh'] = con_ssh
is_ceph = backup_info.get('is_storage_lab', False)
LOG.debug('This is a {} lab'.format(
'Storage/Ceph' if is_ceph else 'Non-Storage/Ceph'))
if is_ceph:
con_ssh.exec_sudo_cmd('touch /etc/ceph/ceph.client.None.keyring')
pre_backup_test(backup_info, con_ssh)
lab = InstallVars.get_install_var('LAB')
LOG.tc_step(
"System backup: lab={}; backup dest = {} backup destination path = {} ..."
.format(lab['name'], backup_info['backup_dest'],
backup_info['backup_dest_full_path']))
copy_to_usb = None
usb_part2 = None
backup_dest = backup_info['backup_dest']
if backup_dest == 'usb':
usb_partition_info = backup_info['usb_parts_info']
for k, v in usb_partition_info.items():
if k[-1:] == "1":
pass
# usb_part1 = k
elif k[-1:] == '2':
usb_part2 = k
copy_to_usb = usb_part2
backup_info['copy_to_usb'] = copy_to_usb
backup_info['backup_file_prefix'] = get_backup_file_name_prefix(
backup_info)
backup_info['cinder_backup'] = BackupVars.get_backup_var('cinder_backup')
reinstall_storage = BackupVars.get_backup_var('reinstall_storage')
if reinstall_storage:
if is_ceph:
backup_cinder_volumes(backup_info)
backup_sysconfig_images(backup_info)
else:
# if is_ceph:
# backup_cinder_volumes(backup_info)
backup_sysconfig_images(backup_info)
collect_logs('after_backup')
if system_helper.is_avs(con_ssh=con_ssh):
# Copying system backup ISO file for future restore
assert backup_load_iso_image(backup_info)
def test_system_upgrade_controllers(upgrade_setup,
check_system_health_query_upgrade):
lab = upgrade_setup['lab']
current_version = upgrade_setup['current_version']
upgrade_version = upgrade_setup['upgrade_version']
# run system upgrade-start
# must be run in controller-0
active_controller = system_helper.get_active_controller_name()
LOG.tc_step("Checking if active controller is controller-0......")
assert "controller-0" in active_controller, "The active controller is not " \
"controller-0. Make controller-0 " \
"active before starting upgrade"
force = False
LOG.tc_step("Checking system health for upgrade .....")
if check_system_health_query_upgrade[0] == 0:
LOG.info("System health OK for upgrade......")
elif check_system_health_query_upgrade[0] == 2:
LOG.info(
"System health indicate minor alarms; using --force option to start upgrade......"
)
force = True
else:
assert False, "System health query upgrade failed: {}".format(
check_system_health_query_upgrade[1])
LOG.info("Starting upgrade from release {} to target release {}".format(
current_version, upgrade_version))
upgrade_helper.system_upgrade_start(force=force)
LOG.tc_step("upgrade started successfully......")
# upgrade standby controller
LOG.tc_step("Upgrading controller-1")
upgrade_helper.upgrade_host("controller-1", lock=True)
LOG.tc_step("Host controller-1 is upgraded successfully......")
# unlock upgraded controller-1
LOG.tc_step("Unlocking controller-1 after upgrade......")
host_helper.unlock_host("controller-1",
available_only=True,
check_hypervisor_up=False)
LOG.tc_step("Host controller-1 unlocked after upgrade......")
time.sleep(60)
# Before Swacting ensure the controller-1 is in available state
if not system_helper.wait_for_host_values(
"controller-1",
timeout=360,
fail_ok=True,
operational=HostOperState.ENABLED,
availability=HostAvailState.AVAILABLE):
err_msg = " Swacting to controller-1 is not possible because controller-1 is not in available state " \
"within the specified timeout"
assert False, err_msg
# Swact to standby controller-1
LOG.tc_step("Swacting to controller-1 .....")
rc, output = host_helper.swact_host(hostname="controller-0")
assert rc == 0, "Failed to swact: {}".format(output)
LOG.info("Swacted and controller-1 has become active......")
time.sleep(60)
# upgrade controller-0
LOG.tc_step("Upgrading controller-0......")
controller0 = lab['controller-0']
LOG.info("Ensure controller-0 is provisioned before upgrade.....")
upgrade_helper.ensure_host_provisioned(controller0.name)
LOG.info("Host {} is provisioned for upgrade.....".format(
controller0.name))
# open vlm console for controller-0 for boot through mgmt interface
LOG.info("Opening a vlm console for controller-0 .....")
install_helper.open_vlm_console_thread("controller-0")
LOG.info("Starting {} upgrade.....".format(controller0.name))
upgrade_helper.upgrade_host(controller0.name, lock=True)
LOG.info("controller-0 is upgraded successfully.....")
# unlock upgraded controller-0
LOG.tc_step("Unlocking controller-0 after upgrade......")
host_helper.unlock_host(controller0.name, available_only=True)
LOG.info("Host {} unlocked after upgrade......".format(controller0.name))
def test_system_upgrade(upgrade_setup, check_system_health_query_upgrade):
"""
This test verifies the upgrade system using orchestration or manual (one node at a time) procedures. The system
hosts are upgraded in the order: controller-1, controller-0, storages, computes. Upgrading through orchestration or
manual is selected through the argument option --orchestration. The standby controller-1 is always upgraded first
using the manual upgrade regardless of the orchestration option. The remaining nodes are upgraded either one at a
time or through orchestration depending on the option selected. The default is to use upgrade orchestration. The
--orchestration is specified in form: [<host personality>[:<number of hosts>]], where:
<personality> is either compute or storage
<number of hosts> is the number of hosts that are upgraded manually before using orchestration.
e.g:
--orchestration compute:1 - do manual upgrade for controller-0, controller-1, all storages if exist
and one compute, the remaining computes are upgraded through orchestration.
--orchestration default - do manual upgrade for controller-1 and use orchestration for the rest of the nodes.
--orchestration controller - do manual upgrade for controller-1 and controller-0, the rest nodes are upgraded
through orchestration.
--orchestration storage:2 - use orchestration after 2 storages are upgraded manually.
option specified during executing this test, the system is upgraded
Args:
upgrade_setup:
check_system_health_query_upgrade:
Returns:
"""
lab = upgrade_setup['lab']
current_version = upgrade_setup['current_version']
upgrade_version = upgrade_setup['upgrade_version']
bld_server = upgrade_setup['build_server']
collect_kpi = upgrade_setup['col_kpi']
# orchestration = 'upgrade'
man_upgrade_nodes = upgrade_setup['man_upgrade_nodes']
orchestration_nodes = upgrade_setup['orchestration_nodes']
system_upgrade_health = list(check_system_health_query_upgrade)
missing_manifests = False
force = False
controller0 = lab['controller-0']
if not upgrade_helper.is_host_provisioned(controller0.name):
upgrade_helper.ensure_host_provisioned(controller0.name)
# update health query
# system_upgrade_health = list(upgrade_helper.get_system_health_query_upgrade())
system_upgrade_health = list(upgrade_helper.get_system_health_query_upgrade_2())
LOG.tc_step("Checking system health for upgrade .....")
if system_upgrade_health[0] == 0:
LOG.info("System health OK for upgrade......")
elif system_upgrade_health[0] == 2:
if system_upgrade_health[2] and "lock_unlock" in system_upgrade_health[2].keys():
controller_nodes = system_upgrade_health[2]["lock_unlock"][0]
LOG.info("Locking/Unlocking required for {} ......".format(controller_nodes))
if 'controller-1' in controller_nodes:
rc, output = upgrade_helper.upgrade_host_lock_unlock('controller-1')
assert rc == 0, "Failed to lock/unlock host {}: {}".format('controller-1', output)
if 'controller-0' in controller_nodes:
rc, output = upgrade_helper.upgrade_host_lock_unlock('controller-0')
assert rc == 0, "Failed to lock/unlock host {}: {}".format('controller-0', output)
time.sleep(60)
# system_upgrade_health[2]["swact"] = False
if system_upgrade_health[2]["swact"][0]:
LOG.info("Swact Required: {}".format(system_upgrade_health[2]["swact"][1]))
host_helper.swact_host('controller-0')
time.sleep(60)
host_helper.swact_host('controller-1')
time.sleep(60)
if system_upgrade_health[2]["force_upgrade"][0]:
LOG.info("{}; using --force option to start upgrade......"
.format(system_upgrade_health[2]["force_upgrade"][1]))
force = True
else:
assert False, "System health query upgrade failed: {}".format(system_upgrade_health[1])
#
#
# LOG.tc_step("Checking system health for upgrade .....")
# if system_upgrade_health[0] == 0:
# LOG.info("System health OK for upgrade......")
# if system_upgrade_health[0] == 1:
# assert False, "System health query upgrade failed: {}".format(system_upgrade_health[1])
#
# if system_upgrade_health[0] == 4 or system_upgrade_health[0] == 2:
# LOG.info("System health indicate missing manifests; lock/unlock controller-0 to resolve......")
# missing_manifests = True
#
# if system_upgrade_health[0] == 3 or system_upgrade_health[0] == 2:
#
# LOG.info("System health indicate minor alarms; using --force option to start upgrade......")
# force = True
#
# if missing_manifests:
# LOG.info("Locking/Unlocking to resolve missing manifests in controller......")
#
# lock_unlock_hosts = []
# if any("controller-1" in k for k in check_system_health_query_upgrade[1].keys()):
# lock_unlock_hosts.append('controller-1')
# if any("controller-0" in k for k in check_system_health_query_upgrade[1].keys()):
# lock_unlock_hosts.append('controller-0')
#
# for host in lock_unlock_hosts:
# rc, output = upgrade_helper.upgrade_host_lock_unlock(host)
# assert rc == 0, "Failed to lock/unlock host {}: {}".format(host, output)
upgrade_init_time = str(datetime.now())
LOG.tc_step("Starting upgrade from release {} to target release {}".format(current_version, upgrade_version))
current_state = upgrade_helper.get_upgrade_state()
if "No upgrade in progress" in current_state:
upgrade_helper.system_upgrade_start(force=force)
LOG.info("upgrade started successfully......")
elif "started" in current_state:
LOG.info("upgrade already started ......")
else:
LOG.info("upgrade is already in state {} please continue manual upgrade ......".format(current_state))
assert False, "upgrade is already in state {} please continue manual upgrade ......".format(current_state)
time.sleep(60)
# upgrade standby controller
LOG.tc_step("Upgrading controller-1")
upgrade_helper.upgrade_controller('controller-1')
time.sleep(60)
# Swact to standby controller-1
LOG.tc_step("Swacting to controller-1 .....")
# Before Swacting ensure the controller-1 is in available state
if not system_helper.wait_for_host_values("controller-1", timeout=900, fail_ok=True,
operational=HostOperState.ENABLED,
availability=HostAvailState.AVAILABLE):
err_msg = " Swacting to controller-1 is not possible because controller-1 is not in available state " \
"within the specified timeout"
assert False, err_msg
if collect_kpi:
upgrade_helper.collected_upgrade_controller1_kpi(lab, collect_kpi, init_time=upgrade_init_time)
rc, output = host_helper.swact_host(hostname="controller-0")
assert rc == 0, "Failed to swact: {}".format(output)
LOG.info("Swacted and controller-1 has become active......")
time.sleep(120)
# active_controller = system_helper.get_active_controller_name()
if 'controller-1' in man_upgrade_nodes:
man_upgrade_nodes.remove('controller-1')
if len(man_upgrade_nodes) > 0:
upgrade_helper.manual_upgrade_hosts(manual_nodes=man_upgrade_nodes)
if len(orchestration_nodes) > 0:
upgrade_helper.orchestration_upgrade_hosts(upgraded_hosts=man_upgrade_nodes,
orchestration_nodes=orchestration_nodes)
if collect_kpi:
if len(orchestration_nodes) > 0:
upgrade_helper.collect_upgrade_orchestration_kpi(lab, collect_kpi)
else:
if upgrade_setup['cpe']:
upgrade_helper.collected_upgrade_controller0_kpi(lab, collect_kpi)
# Activate the upgrade
LOG.tc_step("Activating upgrade....")
upgrade_helper.activate_upgrade()
LOG.info("Upgrade activate complete.....")
# Make controller-0 the active controller
# Swact to standby controller-0
LOG.tc_step("Making controller-0 active.....")
rc, output = host_helper.swact_host(hostname="controller-1")
assert rc == 0, "Failed to swact: {}".format(output)
LOG.info("Swacted to controller-0 ......")
# Complete upgrade
LOG.tc_step("Completing upgrade from {} to {}".format(current_version, upgrade_version))
upgrade_helper.complete_upgrade()
LOG.info("Upgrade is complete......")
LOG.info("Lab: {} upgraded successfully".format(lab['name']))
# Delete the previous load
LOG.tc_step("Deleting {} load... ".format(current_version))
upgrade_helper.delete_imported_load()
LOG.tc_step("Delete previous load version {}".format(current_version))
LOG.tc_step("Downloading images to upgraded {} lab ".format(upgrade_version))
install_helper.download_image(lab, bld_server, BuildServerPath.GUEST_IMAGE_PATHS[upgrade_version])
load_path = UpgradeVars.get_upgrade_var('TIS_BUILD_DIR')
LOG.tc_step("Downloading heat templates to upgraded {} lab ".format(upgrade_version))
install_helper.download_heat_templates(lab, bld_server, load_path)
LOG.tc_step("Downloading lab config scripts to upgraded {} lab ".format(upgrade_version))
install_helper.download_lab_config_files(lab, bld_server, load_path)
def test_vm_with_large_volume_and_evacuation(vms_, pre_alarm_):
"""
Test instantiate a vm with a large volume ( 20 GB and 40 GB) and evacuate:
Args:
vms_ (dict): vms created by vms_ fixture
pre_alarm_ (list): alarm lists obtained by pre_alarm_ fixture
Test Setups:
- get tenant1 and management networks which are already created for lab setup
- get or create a "small" flavor
- get the guest image id
- create two large volumes (20 GB and 40 GB) in cinder
- boot two vms ( test_inst1, test_inst2) using volumes 20 GB and 40 GB respectively
Test Steps:
- Verify VM status is ACTIVE
- Validate that VMs boot, and that no timeouts or error status occur.
- Verify the VM can be pinged from NATBOX
- Verify login to VM and rootfs (dev/vda) filesystem is rw mode
- live migrate, if required, to bring both VMs to the same compute
- Validate migrated VM and no errors or alarms are present
- Reboot compute host to initiate evacuation
- Verify VMs are evacuated
- Check for any system alarms
- Verify login to VM and rootfs (dev/vda) filesystem is still rw mode after evacuation
- Terminate VMs
Skip conditions:
- less that two computes
- no storage node
"""
vm_ids = []
for vm in vms_:
vm_id = vm['id']
vm_ids.append(vm_id)
LOG.tc_step(
"Checking VM status; VM Instance id is: {}......".format(vm_id))
vm_state = vm_helper.get_vm_status(vm_id)
assert vm_state == VMStatus.ACTIVE, 'VM {} state is {}; Not in ACTIVATE state as expected'\
.format(vm_id, vm_state)
LOG.tc_step("Verify VM can be pinged from NAT box...")
rc, boot_time = check_vm_boot_time(vm_id)
assert rc, "VM is not pingable after {} seconds ".format(boot_time)
LOG.tc_step("Verify Login to VM and check filesystem is rw mode....")
assert is_vm_filesystem_rw(vm_id), 'rootfs filesystem is not RW as expected for VM {}'\
.format(vm['display_name'])
LOG.tc_step(
"Checking if live migration is required to put the vms to a single compute...."
)
host_0 = vm_helper.get_vm_host(vm_ids[0])
host_1 = vm_helper.get_vm_host(vm_ids[1])
if host_0 != host_1:
LOG.tc_step("Attempting to live migrate vm {} to host {} ....".format(
(vms_[1])['display_name'], host_0))
code, msg = vm_helper.live_migrate_vm(vm_ids[1],
destination_host=host_0)
LOG.tc_step("Verify live migration succeeded...")
assert code == 0, "Live migration of vm {} to host {} did not success".format(
(vms_[1])['display_name'], host_0)
LOG.tc_step("Verify both VMs are in same host....")
assert host_0 == vm_helper.get_vm_host(
vm_ids[1]), "VMs are not in the same compute host"
LOG.tc_step(
"Rebooting compute {} to initiate vm evacuation .....".format(host_0))
vm_helper.evacuate_vms(host=host_0, vms_to_check=vm_ids, ping_vms=True)
LOG.tc_step("Login to VM and to check filesystem is rw mode....")
assert is_vm_filesystem_rw((vms_[0])['id']), 'After evacuation the rootfs filesystem is not RW as expected ' \
'for VM {}'.format((vms_[0])['display_name'])
LOG.tc_step("Login to VM and to check filesystem is rw mode....")
assert is_vm_filesystem_rw((vms_[1])['id']), 'After evacuation the rootfs filesystem is not RW as expected ' \
'for VM {}'.format((vms_[1])['display_name'])
def test_dc_snmp(snmp_precheck):
"""
Update DNS servers on central region and check it is propagated to subclouds
Args:
snmp_precheck: test fixture for setup/teardown
Setups:
- Ensure primary subcloud is managed and SNMP config is synced
Test Steps:
- Un-manage primary subcloud
- Add a SNMP community string and a trapdest on unmanaged subcloud locally
- Add a different SNMP community string and trapdest on central region
- Wait for new SNMP configs to sync over to managed online subclouds
- Ensure central SNMP configs are not updated on unmanaged primary subcloud
- Re-manage primary subcloud and ensure DNS config syncs over
- Verify nslookup works in Central Region and primary subcloud
Teardown:
- Delete DNS servers to original value (module)
"""
primary_subcloud, managed_subclouds, central_comms, central_trapdests = snmp_precheck
central_auth = Tenant.get('admin_platform', dc_region='RegionOne')
sub_auth = Tenant.get('admin_platform', dc_region=primary_subcloud)
LOG.tc_step("Unmanage {}".format(primary_subcloud))
dc_helper.unmanage_subcloud(subcloud=primary_subcloud, check_first=False)
LOG.tc_step(
'Add SNMP community string and trapdest to unmanaged subcloud - {}'.
format(primary_subcloud, SNMP_COMM))
system_helper.create_snmp_comm('cgcsauto_comm_local', auth_info=sub_auth)
system_helper.create_snmp_trapdest(comm_string="cgcsauto_trapdest_local",
ip_addr='8.8.8.8',
auth_info=sub_auth)
LOG.tc_step('Add SNMP community string and trapdest to central region')
system_helper.create_snmp_comm(SNMP_COMM, auth_info=central_auth)
system_helper.create_snmp_trapdest(comm_string=SNMP_TRAPDEST[0],
ip_addr=SNMP_TRAPDEST[1],
auth_info=central_auth)
LOG.tc_step(
"Wait for new SNMP config to sync over to managed subclouds: {}".
format(managed_subclouds))
expt_comms = central_comms + [SNMP_COMM]
expt_trapdests = central_trapdests + [SNMP_TRAPDEST[1]]
dc_helper.wait_for_sync_audit(subclouds=managed_subclouds)
for managed_sub in managed_subclouds:
dc_helper.wait_for_subcloud_snmp_comms(subcloud=managed_sub,
expected_comms=expt_comms,
timeout=30,
check_interval=10)
dc_helper.wait_for_subcloud_snmp_trapdests(
subcloud=managed_sub,
expected_trapdests=expt_trapdests,
timeout=30,
check_interval=10)
LOG.tc_step(
"Ensure central SNMP config is not synced to unmanaged subcloud: {}".
format(primary_subcloud))
code_comm = dc_helper.wait_for_subcloud_snmp_comms(
subcloud=primary_subcloud,
expected_comms=expt_comms,
timeout=15,
check_interval=5,
fail_ok=True)[0]
code_trapdest = dc_helper.wait_for_subcloud_snmp_trapdests(
subcloud=primary_subcloud,
expected_trapdests=expt_trapdests,
timeout=15,
check_interval=5,
fail_ok=True)[0]
assert code_comm == 1, "SNMP comm is updated unexpectedly on unmanaged subcloud {}".format(
primary_subcloud)
assert code_trapdest == 1, "SNMP trapdest is updated unexpectedly on unmanaged subcloud {}".format(
primary_subcloud)
LOG.tc_step('Re-manage {} and ensure DNS config syncs over'.format(
primary_subcloud))
dc_helper.manage_subcloud(subcloud=primary_subcloud, check_first=False)
dc_helper.wait_for_subcloud_snmp_comms(subcloud=primary_subcloud,
expected_comms=expt_comms)
dc_helper.wait_for_subcloud_snmp_trapdests(
subcloud=primary_subcloud,
expected_trapdests=expt_trapdests,
timeout=30,
check_interval=10)
def test_instantiate_a_vm_with_multiple_volumes_and_migrate():
"""
Test a vm with a multiple volumes live, cold migration and evacuation:
Test Setups:
- get guest image_id
- get or create 'small' flavor_id
- get tenenat and managment network ids
Test Steps:
- create volume for boot and another extra size 8GB
- boot vms from the created volume
- Validate that VMs boot, and that no timeouts or error status occur.
- Verify VM status is ACTIVE
- Attach the second volume to VM
- Attempt live migrate VM
- Login to VM and verify the filesystem is rw mode on both volumes
- Attempt cold migrate VM
- Login to VM and verify the filesystem is rw mode on both volumes
- Reboot the compute host to initiate evacuation
- Login to VM and verify the filesystem is rw mode on both volumes
- Terminate VMs
Skip conditions:
- less than two computes
- less than one storage
"""
# skip("Currently not working. Centos image doesn't see both volumes")
LOG.tc_step("Creating a volume size=8GB.....")
vol_id_0 = cinder_helper.create_volume(size=8)[1]
ResourceCleanup.add('volume', vol_id_0, scope='function')
LOG.tc_step("Creating a second volume size=8GB.....")
vol_id_1 = cinder_helper.create_volume(size=8, bootable=False)[1]
LOG.tc_step("Volume id is: {}".format(vol_id_1))
ResourceCleanup.add('volume', vol_id_1, scope='function')
LOG.tc_step("Booting instance vm_0...")
vm_id = vm_helper.boot_vm(name='vm_0',
source='volume',
source_id=vol_id_0,
cleanup='function')[1]
time.sleep(5)
LOG.tc_step("Verify VM can be pinged from NAT box...")
rc, boot_time = check_vm_boot_time(vm_id)
assert rc, "VM is not pingable after {} seconds ".format(boot_time)
LOG.tc_step("Login to VM and to check filesystem is rw mode....")
assert is_vm_filesystem_rw(
vm_id), 'vol_0 rootfs filesystem is not RW as expected.'
LOG.tc_step("Attemping to attach a second volume to VM...")
vm_helper.attach_vol_to_vm(vm_id, vol_id_1)
LOG.tc_step(
"Login to VM and to check filesystem is rw mode for both volumes....")
assert is_vm_filesystem_rw(vm_id, rootfs=[
'vda', 'vdb'
]), 'volumes rootfs filesystem is not RW as expected.'
LOG.tc_step("Attemping live migrate VM...")
vm_helper.live_migrate_vm(vm_id=vm_id)
LOG.tc_step(
"Login to VM and to check filesystem is rw mode after live migration...."
)
assert is_vm_filesystem_rw(vm_id, rootfs=[
'vda', 'vdb'
]), 'After live migration rootfs filesystem is not RW'
LOG.tc_step("Attempting cold migrate VM...")
vm_helper.cold_migrate_vm(vm_id)
LOG.tc_step(
"Login to VM and to check filesystem is rw mode after live migration...."
)
assert is_vm_filesystem_rw(vm_id, rootfs=[
'vda', 'vdb'
]), 'After cold migration rootfs filesystem is not RW'
LOG.tc_step("Testing VM evacuation.....")
before_host_0 = vm_helper.get_vm_host(vm_id)
LOG.tc_step("Rebooting compute {} to initiate vm evacuation .....".format(
before_host_0))
vm_helper.evacuate_vms(host=before_host_0,
vms_to_check=vm_id,
ping_vms=True)
LOG.tc_step(
"Login to VM and to check filesystem is rw mode after live migration...."
)
assert is_vm_filesystem_rw(
vm_id, rootfs=['vda', 'vdb']), 'After evacuation filesystem is not RW'
def test_attach_cinder_volume_to_instance(vol_vif, check_avs_pattern):
"""
Validate that cinder volume can be attached to VM created using wrl5_avp and wrl5_virtio image
Args:
vol_vif (str)
Test Steps:
- Create cinder volume
- Boot VM use WRL image
- Attach cinder volume to WRL virtio/avp instance
- Check VM nics vifs are not changed
Teardown:
- Delete VM
- Delete cinder volume
"""
mgmt_net_id = network_helper.get_mgmt_net_id()
tenant_net_id = network_helper.get_tenant_net_id()
internal_net_id = network_helper.get_internal_net_id()
vif_model = 'avp' if system_helper.is_avs() else 'virtio'
nics = [
{
'net-id': mgmt_net_id
},
{
'net-id': tenant_net_id
},
{
'net-id': internal_net_id,
'vif-model': vif_model
},
]
LOG.tc_step("Boot up VM from default tis image")
vm_id = vm_helper.boot_vm(name='vm_attach_vol_{}'.format(vol_vif),
source='image',
nics=nics,
cleanup='function')[1]
prev_ports = network_helper.get_ports(server=vm_id)
LOG.tc_step(
"Create an image with vif model metadata set to {}".format(vol_vif))
img_id = glance_helper.create_image('vif_{}'.format(vol_vif),
cleanup='function',
**{ImageMetadata.VIF_MODEL:
vol_vif})[1]
LOG.tc_step("Boot a volume from above image")
volume_id = cinder_helper.create_volume('vif_{}'.format(vol_vif),
source_id=img_id,
cleanup='function')[1]
# boot a cinder volume and attached it to vm
LOG.tc_step("Attach cinder Volume to VM")
vm_helper.attach_vol_to_vm(vm_id, vol_id=volume_id)
LOG.tc_step("Check vm nics vif models are not changed")
post_ports = network_helper.get_ports(server=vm_id)
assert prev_ports == post_ports
def test_dvr_vms_network_connection(vms_num, srv_grp_policy, server_groups,
router_info):
"""
Test vms East West connection by pinging vms' data network from vm
Args:
vms_num (int): number of vms to boot
srv_grp_policy (str): affinity to boot vms on same host, anti-affinity to boot vms on
different hosts
server_groups: test fixture to return affinity and anti-affinity server groups
router_info (str): id of tenant router
Skip Conditions:
- Only one nova host on the system
Setups:
- Enable DVR (module)
Test Steps
- Update router to distributed if not already done
- Boot given number of vms with specific server group policy to schedule vms on
same or different host(s)
- Ping vms' over data and management networks from one vm to test NS and EW traffic
Teardown:
- Delete vms
- Revert router to
"""
# Increase instance quota count if needed
current_vms = len(vm_helper.get_vms(strict=False))
quota_needed = current_vms + vms_num
vm_helper.ensure_vms_quotas(quota_needed)
if srv_grp_policy == 'anti-affinity' and len(
host_helper.get_up_hypervisors()) == 1:
skip("Only one nova host on the system.")
LOG.tc_step("Update router to distributed if not already done")
router_id = router_info
is_dvr = network_helper.get_router_values(router_id,
fields='distributed',
auth_info=Tenant.get('admin'))[0]
if not is_dvr:
network_helper.set_router_mode(router_id, distributed=True)
LOG.tc_step("Boot {} vms with server group policy {}".format(
vms_num, srv_grp_policy))
affinity_grp, anti_affinity_grp = server_groups(soft=True)
srv_grp_id = affinity_grp if srv_grp_policy == 'affinity' else anti_affinity_grp
vms = []
tenant_net_id = network_helper.get_tenant_net_id()
mgmt_net_id = network_helper.get_mgmt_net_id()
internal_net_id = network_helper.get_internal_net_id()
internal_vif = {'net-id': internal_net_id}
if system_helper.is_avs():
internal_vif['vif-model'] = 'avp'
nics = [{'net-id': mgmt_net_id}, {'net-id': tenant_net_id}, internal_vif]
for i in range(vms_num):
vol = cinder_helper.create_volume()[1]
ResourceCleanup.add(resource_type='volume', resource_id=vol)
vm_id = vm_helper.boot_vm('dvr_ew_traffic',
source='volume',
source_id=vol,
nics=nics,
cleanup='function',
hint={'group': srv_grp_id})[1]
vms.append(vm_id)
LOG.tc_step("Wait for vm {} pingable from NatBox".format(vm_id))
vm_helper.wait_for_vm_pingable_from_natbox(vm_id, fail_ok=False)
from_vm = vms[0]
LOG.tc_step(
"Ping vms over management and data networks from vm {}, and verify "
"ping successful.".format(from_vm))
vm_helper.ping_vms_from_vm(to_vms=vms,
from_vm=from_vm,
fail_ok=False,
net_types=['data', 'mgmt', 'internal'])
def test_vif_model_from_image(img_vif, check_avs_pattern):
"""
Test vif model set in image metadata is reflected in vm nics when use normal vnic type.
Args:
img_vif (str):
check_avs_pattern:
Test Steps:
- Create a glance image with given img_vif in metadata
- Create a cinder volume from above image
- Create a vm with 3 vnics from above cinder volume:
- nic1 and nic2 with normal vnic type
- nic3 with avp (if AVS, otherwise normal)
- Verify nic1 and nic2 vif model is the same as img_vif
- Verify nic3 vif model is avp (if AVS, otherwise normal)
"""
LOG.tc_step(
"Create an image with vif model metadata set to {}".format(img_vif))
img_id = glance_helper.create_image('vif_{}'.format(img_vif),
cleanup='function',
**{ImageMetadata.VIF_MODEL:
img_vif})[1]
LOG.tc_step("Boot a volume from above image")
volume_id = cinder_helper.create_volume('vif_{}'.format(img_vif),
source_id=img_id,
cleanup='function')[1]
mgmt_net_id = network_helper.get_mgmt_net_id()
tenant_net_id = network_helper.get_tenant_net_id()
internal_net_id = network_helper.get_internal_net_id()
vif_model = 'avp' if system_helper.is_avs() else img_vif
nics = [{
'net-id': mgmt_net_id
}, {
'net-id': tenant_net_id
}, {
'net-id': internal_net_id,
'vif-model': vif_model
}]
LOG.tc_step(
"Boot a vm from above volume with following nics: {}".format(nics))
vm_id = vm_helper.boot_vm(name='vif_img_{}'.format(img_vif),
nics=nics,
source='volume',
source_id=volume_id,
cleanup='function')[1]
LOG.tc_step(
"Verify vnics info from virsh to ensure tenant net vif is as specified in image metadata"
)
internal_mac = network_helper.get_ports(server=vm_id,
network=internal_net_id,
field='MAC Address')[0]
vm_interfaces = vm_helper.get_vm_interfaces_via_virsh(vm_id)
for vm_if in vm_interfaces:
if_mac, if_model = vm_if
if if_mac == internal_mac:
assert if_model == vif_model
else:
assert if_model == img_vif
def test_dc_stress_alarm(subcloud_to_test):
"""
Test Stress Scenario on Distributed Cloud
Args:
subcloud_to_test (str): module fixture
Setup:
- Make sure there is consistency between alarm summary on
Central Cloud and on subclouds
Test Steps:
Step1:
- Trigger large amount of alarms, quickly on one subcloud
- ensure system alarm-summary on subcloud matches dcmanager alarm summary on system
controller
Step2:
- Trigger large amount of alarms quickly for a long time on all subclouds
- Each alarm summary updates once every 30 seconds until the event is over
- Ensure system alarm-summary on subcloud matches dcmanager alarm summary on system
controller
Step3:
- Clear all alarms
- Ensure system alarm-summary on subcloud matches dcmanager alarm summary on system
controller
"""
ssh_client = ControllerClient.get_active_controller(name=subcloud_to_test)
# Step 1
LOG.tc_step("Trigger large amount of alarms, quickly on one subcloud")
try:
for i in range(1, ALARMS_NO + 1):
ssh_client.exec_cmd(
"fmClientCli -c \"### ###300.005###clear###system.vm###host="
"testhost-{}### ###critical### ###processing-error###cpu-cycles-limit-exceeded"
"### ###True###True###'\"".format(i),
fail_ok=False)
finally:
for i in range(1, ALARMS_NO + 1):
ssh_client.exec_cmd('fmClientCli -D host=testhost-{}'.format(i))
check_alarm_summary_match_subcloud(subcloud_to_test)
# Step 2
ssh_client_list = {}
for subcloud in dc_helper.get_subclouds(mgmt='managed'):
ssh_client_list[subcloud] = ControllerClient.get_active_controller(
name=subcloud_to_test)
try:
LOG.tc_step(
"Trigger large amount of alarms quickly for a long time on all subclouds"
)
for subcloud in ssh_client_list:
subcloud_ssh = ssh_client_list[subcloud]
for i in range(1, ALARMS_NO + 1):
subcloud_ssh.exec_cmd(
"fmClientCli -c \"### ###300.005###clear###"
"system.vm###host=testhost-{}### ###critical### ###processing-error###"
"cpu-cycles-limit-exceeded### ###True###True###'\"".format(
i),
fail_ok=False)
for subcloud in ssh_client_list:
check_alarm_summary_match_subcloud(subcloud)
finally:
# Step 3
LOG.tc_step("Clear all alarms on all subclouds")
for subcloud in ssh_client_list:
subcloud_ssh = ssh_client_list[subcloud]
for i in range(1, ALARMS_NO + 1):
subcloud_ssh.exec_cmd(
'fmClientCli -D host=testhost-{}'.format(i))
for subcloud in ssh_client_list:
check_alarm_summary_match_subcloud(subcloud)
def test_system_upgrade(upgrade_setup, check_system_health_query_upgrade):
lab = upgrade_setup['lab']
current_version = upgrade_setup['current_version']
upgrade_version = upgrade_setup['upgrade_version']
bld_server = upgrade_setup['build_server']
collect_kpi = upgrade_setup['col_kpi']
missing_manifests = False
cinder_configuration = False
force = False
controller0 = lab['controller-0']
if not upgrade_helper.is_host_provisioned(controller0.name):
rc, output = upgrade_helper.upgrade_host_lock_unlock(controller0.name)
assert rc == 0, "Failed to lock/unlock host {}: {}".format(
controller0.name, output)
# update health query
# system_upgrade_health = list(upgrade_helper.get_system_health_query_upgrade())
system_upgrade_health = list(
upgrade_helper.get_system_health_query_upgrade_2())
LOG.tc_step("Checking system health for upgrade .....")
if system_upgrade_health[0] == 0:
LOG.info("System health OK for upgrade......")
elif system_upgrade_health[0] == 2:
if system_upgrade_health[2] and "lock_unlock" in system_upgrade_health[
2].keys():
controller_nodes = system_upgrade_health[2]["lock_unlock"][0]
LOG.info("Locking/Unlocking required for {} ......".format(
controller_nodes))
if 'controller-1' in controller_nodes:
rc, output = upgrade_helper.upgrade_host_lock_unlock(
'controller-1')
assert rc == 0, "Failed to lock/unlock host {}: {}".format(
'controller-1', output)
if 'controller-0' in controller_nodes:
rc, output = upgrade_helper.upgrade_host_lock_unlock(
'controller-0')
assert rc == 0, "Failed to lock/unlock host {}: {}".format(
'controller-0', output)
time.sleep(60)
# system_upgrade_health[2]["swact"][0] = False
if system_upgrade_health[2]["swact"][0]:
LOG.info("Swact Required: {}".format(
system_upgrade_health[2]["swact"][1]))
host_helper.swact_host('controller-0')
time.sleep(60)
host_helper.swact_host('controller-1')
time.sleep(60)
if system_upgrade_health[2]["force_upgrade"][0]:
LOG.info("{}; using --force option to start upgrade......".format(
system_upgrade_health[2]["force_upgrade"][1]))
force = True
else:
assert False, "System health query upgrade failed: {}".format(
system_upgrade_health[1])
# if system_upgrade_health[0] == 0:
# LOG.info("System health OK for upgrade......")
# if system_upgrade_health[0] == 1:
# assert False, "System health query upgrade failed: {}".format(system_upgrade_health[1])
#
# if system_upgrade_health[0] == 4 or system_upgrade_health[0] == 2:
# LOG.info("System health indicate missing manifests; lock/unlock controller-0 to resolve......")
# missing_manifests = True
# if any("Cinder configuration" in k for k in system_upgrade_health[1].keys()):
# cinder_configuration = True
#
# if system_upgrade_health[0] == 3 or system_upgrade_health[0] == 2:
#
# LOG.info("System health indicate minor alarms; using --force option to start upgrade......")
# force = True
#
# if missing_manifests:
# LOG.info("Locking/Unlocking to resolve missing manifests in controller......")
#
# lock_unlock_hosts = []
# if any("controller-1" in k for k in system_upgrade_health[1].keys()):
# lock_unlock_hosts.append('controller-1')
# if any("controller-0" in k for k in system_upgrade_health[1].keys()):
# lock_unlock_hosts.append('controller-0')
# cinder_configuration = False
#
# for host in lock_unlock_hosts:
# rc, output = upgrade_helper.upgrade_host_lock_unlock(host)
# assert rc == 0, "Failed to lock/unlock host {}: {}".format(host, output)
#
# if cinder_configuration:
# LOG.info("Invalid Cinder configuration: Swact to controller-1 and back to synchronize.......")
# host_helper.swact_host('controller-0')
# time.sleep(60)
# host_helper.swact_host('controller-1')
LOG.tc_step("Starting upgrade from release {} to target release {}".format(
current_version, upgrade_version))
upgrade_helper.system_upgrade_start(force=force)
upgrade_helper.wait_for_upgrade_states("started")
LOG.info("upgrade started successfully......")
if collect_kpi:
upgrade_helper.collect_upgrade_start_kpi(lab, collect_kpi)
# upgrade standby controller
LOG.tc_step("Upgrading controller-1")
upgrade_helper.upgrade_host("controller-1", lock=True)
LOG.info("Host controller-1 is upgraded successfully......")
# unlock upgraded controller-1
LOG.tc_step("Unlocking controller-1 after upgrade......")
host_helper.unlock_host("controller-1",
timeout=(HostTimeout.CONTROLLER_UNLOCK + 10),
available_only=True,
check_hypervisor_up=False)
LOG.info("Host controller-1 unlocked after upgrade......")
time.sleep(60)
# Before Swacting ensure the controller-1 is in available state
if not system_helper.wait_for_host_values(
"controller-1",
timeout=600,
fail_ok=True,
operational=HostOperState.ENABLED,
availability=HostAvailState.AVAILABLE):
err_msg = " Swacting to controller-1 is not possible because controller-1 is not in available state " \
"within the specified timeout"
assert False, err_msg
# Swact to standby contime.sleep(60) troller-1
LOG.tc_step("Swacting to controller-1 .....")
rc, output = host_helper.swact_host(hostname="controller-0")
assert rc == 0, "Failed to swact: {}".format(output)
LOG.info("Swacted and controller-1 has become active......")
time.sleep(60)
# upgrade controller-0
LOG.tc_step("Upgrading controller-0......")
controller0 = lab['controller-0']
# open vlm console for controller-0 for boot through mgmt interface
if 'vbox' not in lab['name']:
LOG.info("Opening a vlm console for controller-0 .....")
install_helper.open_vlm_console_thread("controller-0", upgrade=True)
LOG.info("Starting {} upgrade.....".format(controller0.name))
upgrade_helper.upgrade_host(controller0.name, lock=True)
LOG.info("controller-0 is upgraded successfully.....")
# unlock upgraded controller-0
LOG.tc_step("Unlocking controller-0 after upgrade......")
host_helper.unlock_host(controller0.name, available_only=True)
LOG.info("Host {} unlocked after upgrade......".format(controller0.name))
upgrade_hosts = install_helper.get_non_controller_system_hosts()
LOG.info(
"Starting upgrade of the other system hosts: {}".format(upgrade_hosts))
for host in upgrade_hosts:
LOG.tc_step("Starting {} upgrade.....".format(host))
if "storage" in host:
# wait for replication to be healthy
ceph_health_timeout = 300
if 'vbox' in lab['name']:
ceph_health_timeout = 3600
storage_helper.wait_for_ceph_health_ok(timeout=ceph_health_timeout)
upgrade_helper.upgrade_host(host, lock=True)
LOG.info("{} is upgraded successfully.....".format(host))
LOG.tc_step("Unlocking {} after upgrade......".format(host))
host_helper.unlock_host(host, available_only=True)
LOG.info("Host {} unlocked after upgrade......".format(host))
LOG.info("Host {} upgrade complete.....".format(host))
# Activate the upgrade
LOG.tc_step("Activating upgrade....")
upgrade_helper.activate_upgrade()
LOG.info("Upgrade activate complete.....")
# Make controller-0 the active controller
# Swact to standby controller-0
LOG.tc_step("Making controller-0 active.....")
rc, output = host_helper.swact_host(hostname="controller-1")
assert rc == 0, "Failed to swact: {}".format(output)
LOG.info("Swacted to controller-0 ......")
# Complete upgrade
LOG.tc_step("Completing upgrade from {} to {}".format(
current_version, upgrade_version))
upgrade_helper.complete_upgrade()
LOG.info("Upgrade is complete......")
LOG.info("Lab: {} upgraded successfully".format(lab['name']))
# Delete the previous load
LOG.tc_step("Deleting {} load... ".format(current_version))
upgrade_helper.delete_imported_load()
LOG.tc_step("Delete previous load version {}".format(current_version))
LOG.tc_step(
"Downloading images to upgraded {} lab ".format(upgrade_version))
install_helper.download_image(
lab, bld_server, BuildServerPath.GUEST_IMAGE_PATHS[upgrade_version])
load_path = upgrade_setup['load_path']
LOG.tc_step(
"Downloading heat temples to upgraded {} lab ".format(upgrade_version))
install_helper.download_heat_templates(lab, bld_server, load_path)
LOG.tc_step("Downloading lab config scripts to upgraded {} lab ".format(
upgrade_version))
install_helper.download_lab_config_files(lab, bld_server, load_path)
def test_multiple_chained_service_function(protocol, nsh_aware, same_host,
symmetric, check_system,
add_admin_role_module):
"""
Test Multiple Chained Service Function
Test Steps:
- Check if the system is compatible
- Boot the source VM, dest VM & SFC VM in same host or diff host based on <same_host: True or False>
- Install necessary software and package inside guest for packet forwarding test
- Create port pair using nsh_ware <True:False>
- Create port pair group
- Create SFC flow classifier using protocol <tcp:icmp:udp>
- Create port Chain
- Create SFC VM2. port pair, port pair group
- Update port chain with new port pair group
- Check packet forwarding from source to dest vm via SFC vm
- Migrate VM by force_lock compute host
- Check packet forwarding from source to dest vm via SFC vm
Test Teardown:
- Delete port chain, port pair group, port pair, flow classifier, vms, volumes created
"""
nsh_aware = True if nsh_aware == 'nsh_aware' else False
same_host = True if same_host == 'same_host' else False
symmetric = True if symmetric == 'symmetric' else False
LOG.tc_step("Check if the system is compatible to run this test")
computes = check_system
LOG.tc_step("Boot the VM in same host: {}".format(same_host))
hosts_to_boot = [computes[0]] * 3 if same_host else computes[0:3]
LOG.info("Boot the VM in following compute host 1:{}, 2:{}, 3:{}".format(
hosts_to_boot[0], hosts_to_boot[1], hosts_to_boot[2]))
LOG.tc_step("Boot the source and dest VM")
vm_ids = []
vm_ids, source_vm_id, dest_vm_id, internal_net_id, mgmt_net_id, mgmt_nic = _setup_vm(
vm_ids, hosts_to_boot)
vm_helper.ping_vms_from_vm(to_vms=source_vm_id,
from_vm=dest_vm_id,
net_types=['mgmt'],
retry=10)
LOG.tc_step("Boot the SFC VM")
sfc_vm_ids = []
sfc_vm_ids, sfc_vm_under_test, ingress_port_id, egress_port_id = _setup_sfc_vm(
sfc_vm_ids, hosts_to_boot, mgmt_nic, internal_net_id)
vm_helper.ping_vms_from_vm(to_vms=source_vm_id,
from_vm=sfc_vm_under_test,
net_types=['mgmt'],
retry=10)
# if protocol != 'icmp':
LOG.tc_step("Install software package nc in vm {} {}".format(
source_vm_id, dest_vm_id))
_install_sw_packages_in_vm(source_vm_id)
_install_sw_packages_in_vm(dest_vm_id)
LOG.tc_step("copy vxlan tool in sfc vm {}".format(sfc_vm_under_test))
vm_helper.scp_to_vm_from_natbox(vm_id=sfc_vm_under_test,
source_file='/home/cgcs/sfc/vxlan_tool.py',
dest_file='/root/vxlan_tool.py')
LOG.tc_step("Create Port Pair")
port_pair_ids = []
port_pair_id = _setup_port_pair(nsh_aware, ingress_port_id, egress_port_id)
port_pair_ids.append(port_pair_id)
LOG.tc_step("Create Port Pair group")
port_pair_group_ids = []
port_pair_group_id = _setup_port_pair_groups(port_pair_id)
port_pair_group_ids.append(port_pair_group_id)
LOG.tc_step("Create flow classifier")
flow_classifier_name = 'sfc_flow_classifier'
flow_classifier, dest_vm_internal_net_ip = _setup_flow_classifier(
flow_classifier_name, source_vm_id, dest_vm_id, protocol)
LOG.tc_step("Create Port Chain")
port_chain_id = _setup_port_chain(port_pair_group_id, flow_classifier,
symmetric)
LOG.tc_step(
"Execute vxlan.py tool and verify {} packet received VM1 to VM2".
format(protocol))
_check_packets_forwarded_in_sfc_vm(source_vm_id,
dest_vm_id,
sfc_vm_ids,
dest_vm_internal_net_ip,
protocol,
nsh_aware,
symmetric,
load_balancing=False)
LOG.tc_step("Create second SFC VM")
sfc_vm_ids, sfc_vm_under_test2, ingress_port_id2, egress_port_id2 = _setup_sfc_vm(
sfc_vm_ids, hosts_to_boot, mgmt_nic, internal_net_id)
vm_helper.ping_vms_from_vm(to_vms=source_vm_id,
from_vm=sfc_vm_under_test2,
net_types=['mgmt'],
retry=10)
LOG.tc_step("copy vxlan tool in sfc vm {}".format(sfc_vm_under_test2))
vm_helper.scp_to_vm_from_natbox(vm_id=sfc_vm_under_test2,
source_file='/home/cgcs/sfc/vxlan_tool.py',
dest_file='/root/vxlan_tool.py')
LOG.tc_step("Create Port Pair for SFC VM2:{}".format(sfc_vm_under_test2))
port_pair_id2 = _setup_port_pair(nsh_aware, ingress_port_id2,
egress_port_id2)
port_pair_ids.append(port_pair_id2)
LOG.tc_step(
"Create Port Pair group for SFC VM2:{}".format(sfc_vm_under_test2))
port_pair_group_id2 = _setup_port_pair_groups(port_pair_id2)
port_pair_group_ids.append(port_pair_group_id2)
LOG.tc_step("Update port chain")
network_helper.set_sfc_port_chain(port_chain_id,
port_pair_groups=port_pair_group_ids,
flow_classifiers=flow_classifier)
LOG.tc_step(
"Execute vxlan.py tool and verify {} packet received VM1 to VM2".
format(protocol))
_check_packets_forwarded_in_sfc_vm(source_vm_id,
dest_vm_id,
sfc_vm_ids,
dest_vm_internal_net_ip,
protocol,
nsh_aware,
symmetric,
load_balancing=False)
def test_modify_mtu_oam_interface(mtu_range):
"""
of the 2016-04-04 sysinv_test_plan.pdf
20) Change the MTU value of the OAM interface using CLI
Verify that MTU on oam interfaces on both standby and active controller can be modified by cli
Args:
mtu_range (str): A string that contain the mtu want to be tested
Setup:
- Nothing
Test Steps:
- lock standby controller
- modify the imtu value of the controller
- unlock the controller
- revert and oam mtu of the controller and check system is still healthy
- swact the controller
- lock the controller
- modify the imtu value of the controller
- unlock the controller
- check the controllers have expected mtu
- revert the oam mtu of the controller and check system is still healthy
Teardown:
- Nothing
"""
is_sx = system_helper.is_aio_simplex()
origin_active, origin_standby = system_helper.get_active_standby_controllers()
if not origin_standby and not is_sx:
skip("Standby controller unavailable. Cannot lock controller.")
mtu = __get_mtu_to_mod(providernet_name='-ext', mtu_range=mtu_range)
first_host = origin_active if is_sx else origin_standby
max_mtu, cur_mtu, nic_name = get_max_allowed_mtus(host=first_host, network_type='oam')
LOG.info('OK, the max MTU for {} is {}'.format(nic_name, max_mtu))
expecting_pass = not max_mtu or mtu <= max_mtu
if not expecting_pass:
LOG.warn('Expecting to fail in changing MTU: changing to:{}, max-mtu:{}'.format(mtu, max_mtu))
oam_attributes = host_helper.get_host_interfaces(host=first_host, field='attributes', name='oam', strict=False)
# sample attributes: [MTU=9216,AE_MODE=802.3ad]
pre_oam_mtu = int(oam_attributes[0].split(',')[0].split('=')[1])
is_stx_openstack_applied = container_helper.is_stx_openstack_deployed(applied_only=True)
if not is_sx:
HostsToRecover.add(origin_standby)
prev_bad_pods = kube_helper.get_unhealthy_pods(all_namespaces=True)
LOG.tc_step("Modify {} oam interface MTU from {} to {} on standby controller, and "
"ensure it's applied successfully after unlock".format(origin_standby, pre_oam_mtu, mtu))
if mtu == cur_mtu:
LOG.info('Setting to same MTU: from:{} to:{}'.format(mtu, cur_mtu))
code, res = host_helper.modify_mtu_on_interfaces(origin_standby, mtu_val=mtu, network_type='oam',
lock_unlock=True, fail_ok=True)
LOG.tc_step("Revert OAM MTU to original value: {}".format(pre_oam_mtu))
code_revert, res_revert = host_helper.modify_mtu_on_interfaces(origin_standby, mtu_val=pre_oam_mtu,
network_type='oam',
lock_unlock=True, fail_ok=True)
if 0 == code:
assert expecting_pass, "OAM MTU is not modified successfully. Result: {}".format(res)
else:
assert not expecting_pass, "OAM MTU WAS modified unexpectedly. Result: {}".format(res)
assert 0 == code_revert, "OAM MTU is not reverted successfully. Result: {}".format(res_revert)
LOG.tc_step("Check openstack cli, application and pods status after modify and revert {} oam mtu".
format(origin_standby))
check_containers(prev_bad_pods, check_app=is_stx_openstack_applied)
LOG.tc_step("Ensure standby controller is in available state and attempt to swact active controller to {}".
format(origin_standby))
system_helper.wait_for_hosts_states(origin_active, availability=['available'])
host_helper.swact_host(fail_ok=False)
host_helper.wait_for_webservice_up(origin_standby)
prev_bad_pods = kube_helper.get_unhealthy_pods(all_namespaces=True)
HostsToRecover.add(origin_active)
LOG.tc_step("Modify {} oam interface MTU to: {}, and "
"ensure it's applied successfully after unlock".format(origin_active, mtu))
code, res = host_helper.modify_mtu_on_interfaces(origin_active,
mtu_val=mtu, network_type='oam', lock_unlock=True,
fail_ok=True)
LOG.tc_step("Revert OAM MTU to original value: {}".format(pre_oam_mtu))
code_revert, res_revert = host_helper.modify_mtu_on_interfaces(origin_active, mtu_val=pre_oam_mtu,
network_type='oam',
lock_unlock=True, fail_ok=True)
if 0 == code:
assert expecting_pass, "OAM MTU is not modified successfully. Result: {}".format(res)
else:
assert not expecting_pass, "OAM MTU WAS modified unexpectedly. Result: {}".format(res)
assert 0 == code_revert, "OAM MTU is not reverted successfully. Result: {}".format(res_revert)
LOG.tc_step("Check openstack cli, application and pods after modify and revert {} oam mtu".format(origin_active))
check_containers(prev_bad_pods, check_app=is_stx_openstack_applied)
def test_load_balancing_chained_service_function(protocol, nsh_aware,
same_host, symmetric,
check_system,
add_admin_role_module):
"""
Test Load Balancing Chained Service Function
Test Steps:
- Check if the system is compatible
- Boot the source VM, dest VM
- Boot 3 SFC VM
- Install necessary software and package inside guest for packet forwarding test
- Create port pair using nsh_ware <True:False> for each SFC VM
- Create port pair group with 3 port pair
- Create SFC flow classifier using protocol <tcp:icmp:udp>
- Create port Chain with port pair group created
- Check packet forwarding from source to dest vm via SFC vm
Test Teardown:
- Delete port chain, port pair group, port pair, flow classifier, vms, volumes created
"""
nsh_aware = True if nsh_aware == 'nsh_aware' else False
same_host = True if same_host == 'same_host' else False
symmetric = True if symmetric == 'symmetric' else False
LOG.tc_step("Check if the system is compatible to run this test")
computes = check_system
LOG.tc_step("Boot the VM in same host: {}".format(same_host))
hosts_to_boot = [computes[0]] * 3 if same_host else computes[0:3]
LOG.info("Boot the VM in following compute host 1:{}, 2:{}, 3:{}".format(
hosts_to_boot[0], hosts_to_boot[1], hosts_to_boot[2]))
LOG.tc_step("Boot the source and dest VM")
vm_ids = []
vm_ids, source_vm_id, dest_vm_id, internal_net_id, mgmt_net_id, mgmt_nic = _setup_vm(
vm_ids, hosts_to_boot)
vm_helper.ping_vms_from_vm(to_vms=source_vm_id,
from_vm=dest_vm_id,
net_types=['mgmt'],
retry=10)
LOG.tc_step("Boot the 3 SFC VM")
sfc_vm_ids = []
sfc_vm_ids, sfc_vm_under_test, ingress_port_id1, egress_port_id1 = \
_setup_sfc_vm(sfc_vm_ids, hosts_to_boot, mgmt_nic, internal_net_id)
sfc_vm_ids, sfc_vm_under_test2, ingress_port_id2, egress_port_id2 = \
_setup_sfc_vm(sfc_vm_ids, hosts_to_boot, mgmt_nic, internal_net_id)
sfc_vm_ids, sfc_vm_under_test3, ingress_port_id3, egress_port_id3 = \
_setup_sfc_vm(sfc_vm_ids, hosts_to_boot, mgmt_nic, internal_net_id)
for sfc_vm_id in sfc_vm_ids:
vm_helper.ping_vms_from_vm(to_vms=source_vm_id,
from_vm=sfc_vm_id,
net_types=['mgmt'],
retry=10)
# if protocol != 'icmp':
LOG.tc_step("Install software package nc in vm {} {}".format(
source_vm_id, dest_vm_id))
vm_helper.scp_to_vm_from_natbox(vm_id=source_vm_id,
source_file='/home/cgcs/sfc/tcp_client.py',
dest_file='/root/tcp_client.py')
vm_helper.scp_to_vm_from_natbox(
vm_id=source_vm_id,
source_file='/home/cgcs/sfc/loop_tcp_client.sh',
dest_file='/root/loop_tcp_client.sh')
vm_helper.scp_to_vm_from_natbox(
vm_id=source_vm_id,
source_file='/home/cgcs/sfc/tcp_server_multi.py',
dest_file='/root/tcp_server_multi.py')
vm_helper.scp_to_vm_from_natbox(vm_id=dest_vm_id,
source_file='/home/cgcs/sfc/tcp_client.py',
dest_file='/root/tcp_client.py')
vm_helper.scp_to_vm_from_natbox(
vm_id=dest_vm_id,
source_file='/home/cgcs/sfc/loop_tcp_client.sh',
dest_file='/root/loop_tcp_client.sh')
vm_helper.scp_to_vm_from_natbox(
vm_id=dest_vm_id,
source_file='/home/cgcs/sfc/tcp_server_multi.py',
dest_file='/root/tcp_server_multi.py')
_install_sw_packages_in_vm(source_vm_id)
_install_sw_packages_in_vm(dest_vm_id)
for sfc_vm in sfc_vm_ids:
LOG.tc_step("copy vxlan tool in sfc vm {}".format(sfc_vm))
vm_helper.scp_to_vm_from_natbox(
vm_id=sfc_vm,
source_file='/home/cgcs/sfc/vxlan_tool.py',
dest_file='/root/vxlan_tool.py')
LOG.tc_step("Create Port Pair for 3 SFC VM")
port_pair_ids = []
port_pair_id1 = _setup_port_pair(nsh_aware, ingress_port_id1,
egress_port_id1)
port_pair_ids.append(port_pair_id1)
port_pair_id2 = _setup_port_pair(nsh_aware, ingress_port_id2,
egress_port_id2)
port_pair_ids.append(port_pair_id2)
port_pair_id3 = _setup_port_pair(nsh_aware, ingress_port_id3,
egress_port_id3)
port_pair_ids.append(port_pair_id3)
LOG.tc_step(
"Create Port Pair group using 3 port pairs:{}".format(port_pair_ids))
port_pair_group_ids = []
port_pair_group_id = _setup_port_pair_groups(port_pair_ids)
port_pair_group_ids.append(port_pair_group_id)
LOG.tc_step("Create flow classifier")
flow_classifier_name = 'sfc_flow_classifier'
flow_classifier, dest_vm_internal_net_ip = _setup_flow_classifier(
flow_classifier_name, source_vm_id, dest_vm_id, protocol)
LOG.tc_step("Create Port Chain")
_setup_port_chain(port_pair_group_ids, flow_classifier, symmetric)
LOG.tc_step(
"Execute vxlan.py tool and verify {} packet received VM1 to VM2".
format(protocol))
_check_packets_forwarded_in_sfc_vm(source_vm_id,
dest_vm_id,
sfc_vm_ids,
dest_vm_internal_net_ip,
protocol,
nsh_aware,
symmetric,
load_balancing=True)
def test_modify_mtu_data_interface(mtu_range, revert_data_mtu):
"""
23) Change the MTU value of the data interface using CLI
Verify that MTU on data interfaces on all compute node can be modified by cli
The min mtu for data interface can be 1500,9000 or 9216, in which case MTU is unchangable. Need to confirm
Args:
mtu_range (str): A string that contain the mtu want to be tested
revert_data_mtu: A fixture to restore changed mtus if any to their original values
Setup:
- Nothing
Test Steps:
- lock standby controller
- modify the imtu value of the compute node
- unlock the controller
- check the compute node have expected mtu
Teardown:
- Revert data mtu
"""
hypervisors = host_helper.get_hypervisors(state='up')
if len(hypervisors) < 2:
skip("Less than two hypervisors available.")
if system_helper.is_aio_duplex():
standby = system_helper.get_standby_controller_name()
if not standby:
skip("Standby controller unavailable on CPE system. Unable to lock host")
hypervisors = [standby]
else:
if len(hypervisors) > 2:
hypervisors = random.sample(hypervisors, 2)
LOG.tc_step("Delete vms to reduce lock time")
vm_helper.delete_vms()
mtu = __get_mtu_to_mod(providernet_name='-data', mtu_range=mtu_range)
LOG.tc_step("Modify data MTU to {} for hosts: {}".format(mtu, hypervisors))
net_type = 'data'
active_controller = system_helper.get_active_controller_name()
hosts = hypervisors[:]
if active_controller in hosts:
hosts.remove(active_controller)
hosts.append(active_controller)
for host in hosts:
interfaces = get_ifs_to_mod(host, net_type, mtu)
revert_ifs = list(interfaces)
if not revert_ifs:
LOG.info('Skip host:{} because there is no interface to set MTU'.format(host))
continue
host_helper.lock_host(host, swact=True)
revert_ifs.reverse()
changed_ifs = []
for interface in revert_ifs:
LOG.tc_step('Checking the max MTU for the IF:{} on host:{}'.format(interface, host))
max_mtu, cur_mtu, nic_name = get_max_allowed_mtus(host=host, network_type=net_type, if_name=interface)
LOG.info('Checking the max MTU for the IF:{}, max MTU: {}, host:{}'.format(
interface, max_mtu or 'NOT SET', host))
expecting_pass = not max_mtu or mtu <= max_mtu
if not expecting_pass:
LOG.warn('Expecting to fail in changing MTU: changing to:{}, max-mtu:{}'.format(mtu, max_mtu))
pre_mtu = int(host_helper.get_host_interface_values(host, interface, 'imtu')[0])
LOG.tc_step('Modify MTU of IF:{} on host:{} to:{}, expeting: {}'.format(
interface, host, mtu, 'PASS' if expecting_pass else 'FAIL'))
code, res = host_helper.modify_mtu_on_interface(host, interface, mtu_val=mtu, network_type=net_type,
lock_unlock=False, fail_ok=True)
msg_result = "PASS" if expecting_pass else "FAIL"
msg = "Failed to modify data MTU, expecting to {}, \nnew MTU:{}, max MTU:{}, old MTU:{}, " \
"Return code:{}; Details: {}".format(msg_result, pre_mtu, max_mtu, pre_mtu, code, res)
if 0 == code:
if mtu != cur_mtu:
changed_ifs.append(interface)
HOSTS_IF_MODIFY_ARGS.append((host, pre_mtu, mtu, max_mtu, interface, net_type))
assert expecting_pass, msg
else:
assert not expecting_pass, msg
LOG.info('OK, modification of MTU of data interface {} as expected: {}'.format(msg_result, msg_result))
host_helper.unlock_host(host)
for interface in revert_ifs:
if interface in changed_ifs:
actual_mtu = int(host_helper.get_host_interface_values(host,
interface=interface, fields=['imtu'])[0])
assert actual_mtu == mtu, \
'Actual MTU after modification did not match expected, expected:{}, actual:{}'.format(
mtu, actual_mtu)
changed_ifs[:] = []
if not HOSTS_IF_MODIFY_ARGS:
skip('No data interface changed!')
return
HOSTS_IF_MODIFY_ARGS.reverse()
def _test_cpu_pol_dedicated_shared_coexists(vcpus_dedicated, vcpus_shared, pol_source, boot_source):
"""
Test two vms coexisting on the same host, one with the dedicated cpu property, and one with the shared cpu property.
Args:
vcpus_dedicated: Amount of vcpu(s) to allocate for the vm with the dedicated CPU_POLICY.
vcpus_shared: Amount of vcpu(s) to allocate for the vm with the shared CPU_POLICY.
pol_source: Where the CPU_POLICY is set from.
boot_source: The boot media the vm will use to boot.
Test Setups:
- Create two flavors, one for each vm.
- If using 'flavor' for pol_source, set extra specs for the CPU_POLICY.
- If using 'image' for pol_source, set ImageMetaData for the CPU_POLICY.
- If using 'volume' for boot_source, create volume from tis image.
- If using 'image' for boot_source, use tis image.
- Determine the amount of free vcpu(s) on the compute before testing.
Test Steps:
- Boot the first vm with CPU_POLICY: dedicated.
- Wait until vm is pingable from natbox.
- Check vm topology for vcpu(s).
- Determine the amount of free vcpu(s) on the compute.
- Boot the second vm with CPU_POLICY: shared.
- Wait until vm is pingable from natbox.
- Check vm topology for vcpu(s).
- Delete vms
- Determine the amount of free vcpu(s) on the compute after testing.
- Compare free vcpu(s) on the compute before and after testing, ensuring they are the same.
Test Teardown
- Delete created volumes and flavors
"""
LOG.tc_step("Getting host list")
target_hosts = host_helper.get_hypervisors(state='up')
target_host = target_hosts[0]
storage_backing = host_helper.get_host_instance_backing(host=target_host)
if 'image' in storage_backing:
storage_backing = 'local_image'
elif 'remote' in storage_backing:
storage_backing = 'remote'
image_id = glance_helper.get_image_id_from_name(GuestImages.DEFAULT['guest'], strict=True)
pre_test_cpus = host_helper.get_vcpus_for_computes(field='used_now')
collection = ['dedicated', 'shared']
vm_ids = []
for x in collection:
if x == 'dedicated':
vcpus = vcpus_dedicated
else:
vcpus = vcpus_shared
LOG.tc_step("Create {} flavor with {} vcpus".format(x, vcpus))
flavor_id = nova_helper.create_flavor(name=x, vcpus=vcpus, storage_backing=storage_backing)[1]
ResourceCleanup.add('flavor', flavor_id)
if pol_source == 'flavor':
LOG.tc_step("Set CPU_POLICY for {} flavor".format(x))
specs = {FlavorSpec.CPU_POLICY: x}
nova_helper.set_flavor(flavor_id, **specs)
else:
LOG.tc_step("Create image with CPU_POLICY: {}".format(x))
image_meta = {ImageMetadata.CPU_POLICY: x}
image_id = glance_helper.create_image(name='cpu_pol_{}'.format(x), cleanup='function', **image_meta)[1]
if boot_source == 'volume':
LOG.tc_step("Create volume from image")
source_id = cinder_helper.create_volume(name='cpu_pol_{}'.format(x), source_id=image_id)[1]
ResourceCleanup.add('volume', source_id)
else:
source_id = image_id
pre_boot_cpus = host_helper.get_vcpus_for_computes(field='used_now')
LOG.tc_step("Booting cpu_pol_{}".format(x))
vm_id = vm_helper.boot_vm(name='cpu_pol_{}'.format(x), flavor=flavor_id, source=boot_source,
source_id=source_id, avail_zone='nova', vm_host=target_host, cleanup='function')[1]
vm_ids.append(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
check_helper.check_topology_of_vm(vm_id, vcpus=vcpus, cpu_pol=x, vm_host=target_host,
prev_total_cpus=pre_boot_cpus[target_host])
LOG.tc_step("Deleting both dedicated and shared vms")
vm_helper.delete_vms(vms=vm_ids)
post_delete_cpus = host_helper.get_vcpus_for_computes(field='used_now')
assert post_delete_cpus == pre_test_cpus, "vcpu count after test does not equal vcpu count before test"
def test_sysadmin_aging_and_swact(swact):
"""
Test password aging.
Args:
Test Steps:
1 change the aging setting, forcing current password expired, new password be set and required by next login
2 verify the new password is required upon login to the floating IP
Returns:
"""
global _host_users
LOG.tc_step('Change the aging settings of sysadmin')
if 'sysadmin' != HostLinuxUser.get_user():
skip('Current User:{} is not sysadmin'.format(
HostLinuxUser.get_user()))
return
user = '******'
original_password = HostLinuxUser.get_password()
active_controller = ControllerClient.get_active_controller()
active_controller_name = system_helper.get_active_controller_name()
host = lab_info.get_lab_floating_ip()
_host_users[('active-controller', 'sysadmin')] = [original_password]
LOG.info('Closing ssh connection to the active controller\n')
active_controller.flush()
active_controller.close()
wait_time = 10
LOG.info(
'wait for {} seconds after closing the ssh connect to the active-controller'
.format(wait_time))
time.sleep(wait_time)
# command = 'chage -d 0 -M 0 sysadmin'
# this is from the test plan
# sudo passwd -e sysadmin
command = 'sudo passwd -e sysadmin'
LOG.info('changing password aging using command:\n{}'.format(command))
connect = log_in_raw(host, user, original_password)
LOG.info('sudo execute:{}\n'.format(command))
code, output = execute_sudo_cmd(connect,
command,
original_password,
expecting_fail=False)
# code, output = active_controller.exec_sudo_cmd(command)
LOG.info(
'OK, aging settings of sysadmin was successfully changed with command:\n{}\ncode:{}, output:{}'
.format(command, code, output))
LOG.tc_step('Verify new password needs to be set upon login')
exclude_list = [original_password]
# verify password was expired
new_password = security_helper.gen_linux_password(
exclude_list=exclude_list, length=PASSWORD_LEGNTH)
set_password = first_login_to_floating_ip(user, original_password,
new_password)[1]
if set_password != new_password:
message = 'first time login did not ask for new password:{}, ' \
'current password should still been in effective\n'.format(new_password, original_password)
LOG.warn(message)
assert False, message
# and reset with new password
new_password = set_password
if new_password != original_password:
_host_users[('active-controller', 'sysadmin')].append(new_password)
HostLinuxUser.set_password(new_password)
exclude_list.append(new_password)
LOG.info('OK, new password was required and logged in\n')
# reconnect after set new password
LOG.info('reconnect to the active controller')
host = lab_info.get_lab_floating_ip()
connect = log_in_raw(host, user, new_password)
cmd = 'hostname; id; date'
LOG.info('attempt to run cmd:{}\n'.format(cmd))
code, output = execute_cmd(connect, cmd)
LOG.info('output:\n{}\n, code:{}, cmd:{}\n'.format(output, code, cmd))
# perform swact and verify swact is working
wait_time = 300
LOG.info('wait for {} seconds after aging settings been modified'.format(
wait_time))
time.sleep(wait_time)
if swact == 'swact':
LOG.tc_step('Swact host')
swact_host_after_reset_sysadmin_raw(connect, active_controller_name)
LOG.info('OK, host swact')
LOG.info('Closing raw ssh connection to the active controller\n')
connect.logout()
wait_time = 180
LOG.info('wait for {} after swact and closing own ssh connection'.format(
wait_time))
time.sleep(wait_time)
# reconnect to active after swact
LOG.info('reconnect to the active controller')
host = lab_info.get_lab_floating_ip()
connect = log_in_raw(host, user, new_password)
LOG.tc_step('Restore the password ')
# restore_sysadmin_password_raw(connect, new_password, original_password, exclude_list=exclude_list)
restore_sysadmin_password_raw(connect,
current_password=new_password,
original_password="******",
exclude_list=exclude_list)
HostLinuxUser.set_password(original_password)
LOG.info('Close the connection to {} as user:{} with password:{}'.format(
host, user, original_password))
connect.close()
LOG.info(
'reconnect to the active controller using ControllerClient.get_active_controller()\n'
)
active_controller.connect()
