def _test_check_vm_disk_on_compute(storage, hosts_per_backing):
"""
Tests that existence of volumes are properly reported for lvm-backed vms.
Skip:
- Skip if no lvm-configured compute nodes available
Test steps:
- Create a flavor for a lvm-backed vms and boot vm out of that flavor
- SSH onto the node hosting the VM and do the following:
- Run ps aux and confirm that there is a qemu process
- Run sudo lvs and confirm the existence of a thin pool
- Run sudo lvs and confirm the existence of a volume for the vm
- Ensure that the "free" space shown for the hypervisor (obtained by running
"nova hypervisor-show <compute node>" and then checking the "free_disk_gb" field)
reflects the space available within the thin pool
- Delete the instance and ensure that space is returned to the hypervisor
Test Teardown:
- Delete created VM if not already done
"""
hosts_with_backing = hosts_per_backing.get(storage, [])
if not hosts_with_backing:
skip(SkipStorageBacking.NO_HOST_WITH_BACKING.format(storage))
LOG.tc_step("Create flavor and boot vm")
flavor = nova_helper.create_flavor(storage_backing=storage)[1]
ResourceCleanup.add('flavor', flavor, scope='function')
vm = vm_helper.boot_vm(source='image', flavor=flavor, cleanup='function')[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm)
vm_host = vm_helper.get_vm_host(vm)
with host_helper.ssh_to_host(vm_host) as compute_ssh:
LOG.tc_step("Look for qemu process")
compute_ssh.exec_sudo_cmd(cmd="lvs --units g")
assert check_for_qemu_process(compute_ssh), "qemu process not found when calling ps"
LOG.tc_step("Look for pool information")
thin_pool_size = get_initial_pool_space(compute_ssh, vm)
vm_vol_name = vm + '_disk'
raw_vm_volume_output = \
compute_ssh.exec_sudo_cmd(cmd="lvs --units g --noheadings -o lv_size -S lv_name={}".format(vm_vol_name))[1]
assert raw_vm_volume_output, "created vm volume not found"
vm_volume_size = float(raw_vm_volume_output.strip('<g'))
LOG.tc_step("Calculate compute free disk space and ensure that it reflects thin pool")
expected_space_left = int(thin_pool_size - vm_volume_size)
free_disk_space = get_compute_free_disk_gb(vm_host)
assert expected_space_left - 1 <= free_disk_space <= expected_space_left + 1, \
'Hypervisor-show does not reflect space within thin pool'
LOG.tc_step("Calculate free space following vm deletion (ensure volume space is returned)")
vm_helper.delete_vms(vm)
free_disk_space = get_compute_free_disk_gb(vm_host)
assert int(thin_pool_size) == free_disk_space, \
'Space is not properly returned to the hypervisor or hypervisor info does not properly reflect it'
def test_set_hosts_storage_backing_min(instance_backing, number_of_hosts):
"""
Modify hosts storage backing if needed so that system has minimal number of hosts in given instance backing
Args:
instance_backing:
number_of_hosts:
Test Steps:
- Calculate the hosts to be configured based on test params
- Configure hosts to meet given criteria
- Check number of hosts in given instance backing is as specified
"""
LOG.tc_step("Determine the hosts to configure")
hosts = host_helper.get_up_hypervisors()
hosts_len = len(hosts)
host_num_mapping = {'all': hosts_len, 'two': 2, 'one': 1}
number_of_hosts = host_num_mapping[number_of_hosts]
hosts_with_backing = host_helper.get_hosts_in_storage_backing(
instance_backing)
if len(hosts_with_backing) >= number_of_hosts:
LOG.info("Already have {} hosts in {} backing. Do nothing".format(
len(hosts_with_backing), instance_backing))
return
candidate_hosts = get_candidate_hosts(number_of_hosts=number_of_hosts)
number_to_config = number_of_hosts - len(hosts_with_backing)
hosts_to_config = list(set(candidate_hosts) -
set(hosts_with_backing))[0:number_to_config]
LOG.tc_step(
"Delete vms if any to prepare for system configuration change with best effort"
)
vm_helper.delete_vms(fail_ok=True)
LOG.tc_step("Configure following hosts to {} backing: {}".format(
hosts_to_config, instance_backing))
for host in hosts_to_config:
HostsToRecover.add(host)
host_helper.set_host_storage_backing(host=host,
inst_backing=instance_backing,
unlock=False,
wait_for_configured=False)
host_helper.unlock_hosts(hosts_to_config,
check_hypervisor_up=True,
fail_ok=False)
LOG.tc_step("Waiting for hosts in {} aggregate".format(instance_backing))
for host in hosts_to_config:
host_helper.wait_for_host_in_instance_backing(
host, storage_backing=instance_backing)
LOG.tc_step("Check number of {} hosts is at least {}".format(
instance_backing, number_of_hosts))
assert number_of_hosts <= len(host_helper.get_hosts_in_storage_backing(instance_backing)), \
"Number of {} hosts is less than {} after configuration".format(instance_backing, number_of_hosts)
def func_recover(request):
vm_helper.delete_vms()
mgmt_net_id = network_helper.get_mgmt_net_id()
def teardown():
"""
If DNS servers are not set, set them. Deprovision internal DNS.
"""
global UNRESTORED_DNS_SERVERS
global HOSTS_AFFECTED
if UNRESTORED_DNS_SERVERS:
LOG.fixture_step("Restoring DNS entries to: {}".format(UNRESTORED_DNS_SERVERS))
subnet_list = network_helper.get_subnets(network=mgmt_net_id)
set_dns_servers(subnet_list, UNRESTORED_DNS_SERVERS, fail_ok=True)
UNRESTORED_DNS_SERVERS = []
if system_helper.get_alarms(alarm_id=EventLogID.CONFIG_OUT_OF_DATE):
LOG.fixture_step("Config out-of-date alarm(s) present, check {} and lock/unlock if host config out-of-date".
format(HOSTS_AFFECTED))
for host in HOSTS_AFFECTED:
if system_helper.get_host_values(host, 'config_status')[0] == 'Config out-of-date':
LOG.info("Lock/unlock {} to clear config out-of-date status".format(host))
host_helper.lock_unlock_hosts(hosts=host)
HOSTS_AFFECTED.remove(host)
request.addfinalizer(teardown)
return mgmt_net_id
def patch_function_check(request):
vms = vm_helper.get_vms(name='patch', strict=False)
boot_vm = False if len(vms) == 2 else True
if not boot_vm:
for vm in vms:
if vm_helper.get_vm_status(vm) != VMStatus.ACTIVE or not vm_helper.ping_vms_from_natbox(vm, fail_ok=True):
boot_vm = True
break
if boot_vm:
if vms:
vm_helper.delete_vms(vms, remove_cleanup='module')
vms = []
for source in ('volume', 'image'):
vms.append(vm_helper.boot_vm(name='patch_{}'.format(source), source=source, cleanup='module')[1])
def remove_on_teardown():
LOG.info("Check vm status and delete if in bad state")
for vm_ in vms:
if vm_helper.get_vm_status(vm_) != VMStatus.ACTIVE:
vm_helper.delete_vms(vm_, remove_cleanup='module')
LOG.fixture_step("Remove test patches")
remove_test_patches()
request.addfinalizer(remove_on_teardown)
return vms
def test_db_purge():
end_time = time.time() + 7200
count = 1
while time.time() < end_time:
LOG.tc_step(
"Iteration-{}: Creating and deleting image, volume, vm".format(
count))
LOG.info("------ Creating image, volume, vm")
image_id = glance_helper.create_image(
name='glance-purge',
cleanup='function',
**{ImageMetadata.AUTO_RECOVERY: 'true'})[1]
vol_id = cinder_helper.create_volume(name='cinder-purge',
source_id=image_id)[1]
vm_id = vm_helper.boot_vm(name='nova-purge',
source='volume',
source_id=vol_id)[1]
time.sleep(60)
LOG.info("------ Deleting vm, volume, image")
vm_helper.delete_vms(vms=vm_id)
cinder_helper.delete_volumes(volumes=vol_id)
glance_helper.delete_images(images=image_id)
time.sleep(60)
count += 1
def test_delete_vms_and_vols():
"""
Delete vms and volumes on the system.
Usage: normally run before a formal test session (sanity, regression, etc) starts to ensure a clean system
"""
vm_helper.delete_vms()
cinder_helper.delete_volumes()
def teardown():
LOG.info("Delete all created vms and unlock target host(s)...")
for vm_to_del in all_vms:
vm_helper.delete_vms(vm_to_del)
nova_helper.delete_flavors(all_new_flavors)
for host_to_unlock in self.hosts_locked:
host_helper.unlock_host(host_to_unlock,
check_hypervisor_up=True)
def remove_on_teardown():
LOG.info("Check vm status and delete if in bad state")
for vm_ in vms:
if vm_helper.get_vm_status(vm_) != VMStatus.ACTIVE:
vm_helper.delete_vms(vm_, remove_cleanup='module')
LOG.fixture_step("Remove test patches")
remove_test_patches()
def clean_up():
LOG.info('Clean up: delete VMs, volumes, flavors and etc.')
for vm_type in g_vms:
if g_vms[vm_type] and 'id' in g_vms[vm_type]:
vm_id = g_vms[vm_type]['id']
if vm_id:
vm_helper.delete_vms(vm_id)
nova_helper.delete_flavors(g_flavors.values())
g_flavors.clear()
def prepare_resource(add_admin_role_module):
hypervisor = random.choice(host_helper.get_up_hypervisors())
vm_helper.delete_vms()
flavor = nova_helper.create_flavor(name='flavor-1g',
ram=1024,
cleanup='module')[1]
vol_id = cinder_helper.create_volume('vol-mem_page_size',
cleanup='module')[1]
return hypervisor, flavor, vol_id
def launch_delete_vm(flavor, end_time, end_event):
iter_ = 0
name, flv_id = flavor
while time.time() < end_time:
iter_ += 1
if end_event.is_set():
assert 0, "Another thread failed. Terminate rest."
LOG.tc_step("Iter{} - Launch and delete vm on {}".format(iter_, name))
vm_id = vm_helper.boot_vm(name=name, flavor=flv_id)[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)
time.sleep(15)
vm_helper.delete_vms(vms=vm_id)
def launch_delete_vms(flavors, end_time, end_event):
iter_ = 0
while time.time() < end_time:
if end_event.is_set():
assert 0, "Other thread failed. Terminate launch_del_vms thread"
iter_ += 1
LOG.info("Iter{} - Launch and delete vm0 on numa0 and vm1 on numa1".format(iter_))
vms = []
for name, flv_id in flavors.items():
vm_id = vm_helper.boot_vm(name=name, flavor=flv_id)[1]
vms.append(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)
time.sleep(15)
vm_helper.delete_vms(vms=vms)
def pre_backup_setup(backup_info, con_ssh):
"""
Setup before doing System Backup, including clean up existing VMs, snapshots, volumes and create volumes and VMs
for B&R test purpose.
Args:
backup_info:
- options to do system backup
con_ssh:
- current ssh connection
Return:
information of created VMs, Volumes, and Images
"""
tenant = Tenant.get('tenant1')
backup_info['tenant'] = tenant
tenant_id = keystone_helper.get_projects(field='ID',
name=tenant['user'],
con_ssh=con_ssh)[0]
LOG.info('Using tenant:{} in the pre-backup test, details:{}'.format(
tenant_id, tenant))
LOG.info('Deleting VMs for pre-backup system-wide test')
vm_helper.delete_vms()
LOG.info('Deleting Volumes snapshots if any')
cinder_helper.delete_volume_snapshots(auth_info=Tenant.get('admin'),
con_ssh=con_ssh)
LOG.info('Deleting Volumes')
volumes = cinder_helper.get_volumes()
LOG.info('-deleting volumes:{}'.format(volumes))
cinder_helper.delete_volumes(volumes, timeout=180)
LOG.info('Make sure we have glance images to backup')
image_ids = glance_helper.get_images()
LOG.info('Launching VMs')
vms_added = pb_launch_vms(con_ssh, image_ids, backup_info=backup_info)
LOG.info('Creating different sizes of VMs')
volumes_added = pb_create_volumes(con_ssh, backup_info=backup_info)
return {'vms': vms_added, 'volumes': volumes_added, 'images': image_ids}
def patch_orchestration_setup():
LOG.fixture_step("Remove test patches (if any) and check system health")
remove_test_patches()
code, failed = patching_helper.check_system_health(fail_on_disallowed_failure=False)
if code > 1:
skip('Patching cannot be run with failures: {}'.format(failed))
LOG.fixture_step("Copy test patches from build server to system")
patch_dir, patches = patching_helper.download_test_patches()
LOG.fixture_step("Delete existing vms and launch a boot-from-volume vm and a boot-from-image vm")
vm_helper.delete_vms()
cinder_helper.delete_volumes()
for source in ('volume', 'image'):
vm_id = vm_helper.boot_vm(name='patch_{}'.format(source), source=source, cleanup='module')[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
controllers, computes, storages = system_helper.get_hosts_per_personality(rtn_tuple=True)
return patches, controllers, computes, storages
def get_vm_id(vm_type, reuse=True):
global g_vms, g_flavors
LOG.info(
'Make sure the VM for the specified type exists, create if it does not'
)
if reuse and g_vms[vm_type] and 'id' in g_vms[vm_type]:
vm_id = g_vms[vm_type]['id']
LOG.info('VM exists for type:{}, vm_id:{}'.format(vm_type, vm_id))
if reuse and vm_helper.get_vm_status(vm_id=vm_id) == VMStatus.ACTIVE:
return vm_id
LOG.info('not reusing...')
vm_helper.delete_vms()
reset_vms()
if not g_flavors[vm_type]:
create_flavor(vm_type)
vm_id = create_vm_values_for_type(vm_type)
return vm_id
def test_ceph_reboot_storage_node(stx_openstack_required):
"""
us69932_tc2_ceph_mon_process_kill from us69932_ceph_monitoring.odt
Verify that ceph mon processes recover when they are killed
nodes.
Args:
- Nothing
Setup:
- Requires system with storage nodes
Test Steps:
0. Run CEPH pre-check fixture to check:
- system has storage nodes
- health of the ceph cluster is okay
- that we have OSDs provisioned
1. Delete existing VMs
2. Boot new VMs and run dd on them
3. Reboot storage node and ensure both:
- mon state goes down (if storage-0)
- OSD state goes down
4. Ensure mon and OSD state recover afterwards
5. Cleanup VMs
Potential rework:
1. Add the alarms checks for raise and clear
2. Maybe we don't want to reboot all storage nodes
What defects this addresses:
1. CGTS-2975
Update:
This test was updated for the Storage and Robustness feature.
"""
con_ssh = ControllerClient.get_active_controller()
LOG.tc_step("Delete existing VMs")
vm_helper.delete_vms()
LOG.tc_step("Boot various VMs")
vms = vm_helper.boot_vms_various_types(cleanup="function")
vm_threads = []
LOG.tc_step("SSH to VMs and write to disk")
end_event = Events("End dd in vms")
try:
for vm in vms:
vm_thread = vm_helper.write_in_vm(vm, end_event=end_event, expect_timeout=40)
vm_threads.append(vm_thread)
storage_nodes = system_helper.get_storage_nodes(con_ssh)
for host in storage_nodes:
LOG.tc_step('Reboot {}'.format(host))
HostsToRecover.add(host, scope='function')
host_helper.reboot_hosts(host, wait_for_offline=True, wait_for_reboot_finish=False)
LOG.tc_step('Check health of CEPH cluster')
ceph_healthy = True
msg = None
end_time = time.time() + 10
while time.time() < end_time:
ceph_healthy = storage_helper.is_ceph_healthy(con_ssh)
if not ceph_healthy:
break
assert not ceph_healthy, "ceph is not healthy"
LOG.info(msg)
LOG.tc_step('Check that OSDs are down')
osd_list = storage_helper.get_osds(host, con_ssh)
all_osds_up = True
up_list = osd_list.copy()
end_time = time.time() + 60
while time.time() < end_time and all_osds_up:
for osd_id in osd_list:
osd_up = storage_helper.is_osd_up(osd_id, con_ssh)
if not osd_up:
msg = 'OSD ID {} is down as expected'.format(osd_id)
LOG.info(msg)
up_list.remove(osd_id)
if len(up_list) > 0:
osd_list = up_list.copy()
else:
msg = ' All OSDs are down as expected'
LOG.info(msg)
all_osds_up = False
assert not all_osds_up, " One or more OSD(s) {} is(are) up but should be down".format(up_list)
system_helper.wait_for_host_values(host, availability='available')
LOG.tc_step('Check that OSDs are up')
osd_list = storage_helper.get_osds(host, con_ssh)
down_list = osd_list.copy()
all_osds_up = False
end_time = time.time() + 60
while time.time() < end_time and not all_osds_up:
for osd_id in osd_list:
osd_up = storage_helper.is_osd_up(osd_id, con_ssh)
if osd_up:
msg = 'OSD ID {} is up as expected'.format(osd_id)
LOG.info(msg)
down_list.remove(osd_id)
if len(down_list) > 0:
osd_list = down_list.copy()
else:
msg = ' All OSDs are up as expected'
LOG.info(msg)
all_osds_up = True
assert all_osds_up, " One or more OSD(s) {} is(are) down but should be up".format(down_list)
LOG.tc_step('Check health of CEPH cluster')
end_time = time.time() + 40
while time.time() < end_time:
ceph_healthy = storage_helper.is_ceph_healthy(con_ssh)
if ceph_healthy is True:
break
assert ceph_healthy, "ceph is not healthy"
for vm_thread in vm_threads:
assert vm_thread.res is True, "Writing in vm stopped unexpectedly"
finally:
end_event.set()
for vm_thread in vm_threads:
vm_thread.wait_for_thread_end(timeout=20)
LOG.tc_step("Delete existing VMs")
vm_helper.delete_vms()
def test_swact_100_times():
"""
Skip Condition:
- Less than two controllers on system
Test Steps:
- Boot a vm and ensure it's pingable
- Start writing from pre-existed vm before swacting
- Repeat following steps 100 times:
- ensure system has standby controller
- system host-swact
- ensure all services are active in sudo sm-dump on new active controller
- ensure pre-existed vm is still pingable from NatBox
- ensure writing did not stop on pre-existed vm
- ensure new vm can be launched in 2 minutes
- ensure newly booted vm is pingable from NatBox
- delete newly booted vm
Teardown:
- delete vms, volumes
"""
if len(system_helper.get_controllers()) < 2:
skip("Less than two controllers on system")
if not system_helper.get_standby_controller_name():
assert False, "No standby controller on system"
LOG.tc_step("Boot a vm and ensure it's pingable")
vm_base = vm_helper.boot_vm(name='pre_swact', cleanup='function')[1]
LOG.tc_step("Start writing from pre-existed vm before swacting")
end_event = Events("End write in base vm")
base_vm_thread = vm_helper.write_in_vm(vm_base, end_event=end_event, expect_timeout=40, thread_timeout=60*100)
try:
for i in range(100):
iter_str = "Swact iter{}/100 - ".format(i+1)
LOG.tc_step("{}Ensure system has standby controller".format(iter_str))
standby = system_helper.get_standby_controller_name()
assert standby
LOG.tc_step("{}Swact active controller and ensure active controller is changed".format(iter_str))
host_helper.swact_host()
LOG.tc_step("{}Check all services are up on active controller via sudo sm-dump".format(iter_str))
host_helper.wait_for_sm_dump_desired_states(controller=standby, fail_ok=False)
LOG.tc_step("{}Ensure pre-existed vm still pingable post swact".format(iter_str))
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_base, timeout=45)
time.sleep(5)
LOG.tc_step("{}Ensure writing from pre-existed vm resumes after swact".format(iter_str))
assert base_vm_thread.res is True, "Writing in pre-existed vm stopped after {}".format(iter_str.lower())
LOG.tc_step("{}Attempt to boot new vm after 2 minutes of post swact and ensure it's pingable".
format(iter_str))
time.sleep(60)
for j in range(3):
code, vm_new, msg = vm_helper.boot_vm(name='post_swact', fail_ok=True, cleanup='function')
if code == 0:
break
LOG.warning("VM failed to boot - attempt{}".format(j+1))
vm_helper.delete_vms(vms=vm_new)
assert j < 2, "No vm can be booted 2+ minutes after swact"
LOG.tc_step("{}VM{} failed to boot, wait for 30 seconds and retry".format(j+1, iter_str))
time.sleep(30)
vm_helper.wait_for_vm_pingable_from_natbox(vm_new)
LOG.tc_step("{}Delete the vm created".format(iter_str))
vm_helper.delete_vms(vms=vm_new)
except:
raise
finally:
LOG.tc_step("End the base_vm_thread")
end_event.set()
base_vm_thread.wait_for_thread_end(timeout=20)
post_standby = system_helper.get_standby_controller_name()
assert post_standby, "System does not have standby controller after last swact"
def test_modify_mtu_data_interface(mtu_range, revert_data_mtu):
"""
23) Change the MTU value of the data interface using CLI
Verify that MTU on data interfaces on all compute node can be modified by cli
The min mtu for data interface can be 1500,9000 or 9216, in which case MTU is unchangable. Need to confirm
Args:
mtu_range (str): A string that contain the mtu want to be tested
revert_data_mtu: A fixture to restore changed mtus if any to their original values
Setup:
- Nothing
Test Steps:
- lock standby controller
- modify the imtu value of the compute node
- unlock the controller
- check the compute node have expected mtu
Teardown:
- Revert data mtu
"""
hypervisors = host_helper.get_hypervisors(state='up')
if len(hypervisors) < 2:
skip("Less than two hypervisors available.")
if system_helper.is_aio_duplex():
standby = system_helper.get_standby_controller_name()
if not standby:
skip("Standby controller unavailable on CPE system. Unable to lock host")
hypervisors = [standby]
else:
if len(hypervisors) > 2:
hypervisors = random.sample(hypervisors, 2)
LOG.tc_step("Delete vms to reduce lock time")
vm_helper.delete_vms()
mtu = __get_mtu_to_mod(providernet_name='-data', mtu_range=mtu_range)
LOG.tc_step("Modify data MTU to {} for hosts: {}".format(mtu, hypervisors))
net_type = 'data'
active_controller = system_helper.get_active_controller_name()
hosts = hypervisors[:]
if active_controller in hosts:
hosts.remove(active_controller)
hosts.append(active_controller)
for host in hosts:
interfaces = get_ifs_to_mod(host, net_type, mtu)
revert_ifs = list(interfaces)
if not revert_ifs:
LOG.info('Skip host:{} because there is no interface to set MTU'.format(host))
continue
host_helper.lock_host(host, swact=True)
revert_ifs.reverse()
changed_ifs = []
for interface in revert_ifs:
LOG.tc_step('Checking the max MTU for the IF:{} on host:{}'.format(interface, host))
max_mtu, cur_mtu, nic_name = get_max_allowed_mtus(host=host, network_type=net_type, if_name=interface)
LOG.info('Checking the max MTU for the IF:{}, max MTU: {}, host:{}'.format(
interface, max_mtu or 'NOT SET', host))
expecting_pass = not max_mtu or mtu <= max_mtu
if not expecting_pass:
LOG.warn('Expecting to fail in changing MTU: changing to:{}, max-mtu:{}'.format(mtu, max_mtu))
pre_mtu = int(host_helper.get_host_interface_values(host, interface, 'imtu')[0])
LOG.tc_step('Modify MTU of IF:{} on host:{} to:{}, expeting: {}'.format(
interface, host, mtu, 'PASS' if expecting_pass else 'FAIL'))
code, res = host_helper.modify_mtu_on_interface(host, interface, mtu_val=mtu, network_type=net_type,
lock_unlock=False, fail_ok=True)
msg_result = "PASS" if expecting_pass else "FAIL"
msg = "Failed to modify data MTU, expecting to {}, \nnew MTU:{}, max MTU:{}, old MTU:{}, " \
"Return code:{}; Details: {}".format(msg_result, pre_mtu, max_mtu, pre_mtu, code, res)
if 0 == code:
if mtu != cur_mtu:
changed_ifs.append(interface)
HOSTS_IF_MODIFY_ARGS.append((host, pre_mtu, mtu, max_mtu, interface, net_type))
assert expecting_pass, msg
else:
assert not expecting_pass, msg
LOG.info('OK, modification of MTU of data interface {} as expected: {}'.format(msg_result, msg_result))
host_helper.unlock_host(host)
for interface in revert_ifs:
if interface in changed_ifs:
actual_mtu = int(host_helper.get_host_interface_values(host,
interface=interface, fields=['imtu'])[0])
assert actual_mtu == mtu, \
'Actual MTU after modification did not match expected, expected:{}, actual:{}'.format(
mtu, actual_mtu)
changed_ifs[:] = []
if not HOSTS_IF_MODIFY_ARGS:
skip('No data interface changed!')
return
HOSTS_IF_MODIFY_ARGS.reverse()
def test_gpu_passthrough(setup_alias):
"""
Test case for GPU passthrough
Test Steps:
- Create pci alias for dev type 'gpu' and 'user'
- Calculate the initial pf used in 'nova device-list'
- Create flavor with extra spec with PCI_PASSTHROUGH_ALIAS device gpu & usb
- Boot a vm with created flavor & gpu passthrough specfic centos image
- Verify the pf used increased after vm launch
Teardown:
- Delete created vm, flavor, pci_alias
"""
nova_gpu_alias, nova_usb_alias = setup_alias
# initialize parameter for basic operation
name = 'gpu_passthrough'
guest_os = 'centos_gpu'
pf = 1
LOG.tc_step("Create a flavor for GPU Passthrough")
flavor_id = nova_helper.create_flavor(name=name, root_disk=16)[1]
ResourceCleanup.add('flavor', flavor_id, scope='module')
extra_spec = {
FlavorSpec.PCI_PASSTHROUGH_ALIAS:
'{}:{},{}:{}'.format(nova_gpu_alias, pf, nova_usb_alias, pf),
FlavorSpec.CPU_POLICY:
'dedicated'
}
nova_helper.set_flavor(flavor_id, **extra_spec)
initial_gpu_pfs_used = _calculate_pf_used(nova_gpu_alias)
initial_usb_pfs_used = _calculate_pf_used(nova_usb_alias)
LOG.tc_step("Get/Create {} glance image".format(guest_os))
cleanup = None if re.search(GuestImages.TIS_GUEST_PATTERN,
guest_os) else 'module'
image_id = glance_helper.get_guest_image(guest_os=guest_os,
cleanup=cleanup)
mgmt_net_id = network_helper.get_mgmt_net_id()
tenant_net_id = network_helper.get_tenant_net_id()
mgmt_nic = {'net-id': mgmt_net_id}
tenant_nic = {'net-id': tenant_net_id}
nics = [mgmt_nic, tenant_nic]
LOG.tc_step("Boot a vm {} with pci-alias and flavor ".format(
nova_gpu_alias, flavor_id))
vm_id = vm_helper.boot_vm(name,
flavor=flavor_id,
source='image',
source_id=image_id,
nics=nics,
cleanup='function')[1]
actual_gpu_pfs_used = _calculate_pf_used(nova_gpu_alias)
expected_gpu_pfs_used = initial_gpu_pfs_used + pf
assert actual_gpu_pfs_used == expected_gpu_pfs_used, "actual gpu pci pfs is not equal to expected pci pfs"
actual_usb_pfs_used = _calculate_pf_used(nova_usb_alias)
expected_usb_pfs_used = initial_usb_pfs_used + pf
assert actual_usb_pfs_used == expected_usb_pfs_used, "actual usb pci pfs is not equal to expected pci pfs"
LOG.tc_step("Delete vm {} ".format(vm_id))
vm_helper.delete_vms(vms=vm_id, stop_first=False)
actual_gpu_pfs_used = _calculate_pf_used(nova_gpu_alias)
assert actual_gpu_pfs_used == initial_gpu_pfs_used, \
"actual gpu pci pfs is not equal to expected pci pfs after vm delete"
actual_usb_pfs_used = _calculate_pf_used(nova_usb_alias)
assert actual_usb_pfs_used == initial_usb_pfs_used, \
"actual usb pci pfs is not equal to expected pci pfs after vm delete"
LOG.tc_step(
"Deleting nova service parameter service parameters for gpu & usb")
def _test_storage_profile(personality, from_backing, to_backing):
"""
This test creates a storage profile and then applies it to a node with
identical hardware, assuming one exists.
Storage profiles do not apply on controller nodes. Storage profiles can be
applied on controller+compute nodes, compute nodes and storage nodes.
Arguments:
- personality (string) - controller, compute or storage
- from_backing (string) - image, remote or None
- to_backing (string) - image, remote or None
Test Steps:
1. Query system and determine which nodes have compatible hardware.
2. Create a storage profile on one of those nodes
3. Apply the created storage profile on a compatible node*
4. Ensure the storage profiles have been successfully applied.
* If the node is a compute node or a controller+compute, we will also change
the backend if required for additional coverage.
Returns:
- Nothing
"""
global PROFILES_TO_DELETE
PROFILES_TO_DELETE = []
# Skip if test is not applicable to hardware under test
if personality == 'controller' and not system_helper.is_aio_system():
skip("Test does not apply to controller hosts without subtype compute")
hosts = system_helper.get_hosts(personality=personality)
if not hosts:
skip("No hosts of type {} available".format(personality))
if (from_backing == "remote" or to_backing
== "remote") and not system_helper.is_storage_system():
skip("This test doesn't apply to systems without storage hosts")
LOG.tc_step("Identify hardware compatible hosts")
hash_to_hosts = get_hw_compatible_hosts(hosts)
# Pick the hardware group that has the most compatible hosts
current_size = 0
candidate_hosts = []
for value in hash_to_hosts:
candidate_size = len(hash_to_hosts[value])
if candidate_size > current_size:
current_size = candidate_size
candidate_hosts = hash_to_hosts[value]
LOG.info(
"This is the total set of candidate hosts: {}".format(candidate_hosts))
if len(candidate_hosts) < 2:
skip("Insufficient hardware compatible hosts to run test")
# Rsync lab setup dot files between controllers
con_ssh = ControllerClient.get_active_controller()
_rsync_files_to_con1(con_ssh=con_ssh, file_to_check="force.txt")
# Take the hardware compatible hosts and check if any of them already have
# the backend that we want. This will save us test time.
new_to_backing = None
if personality == "compute":
from_hosts = []
to_hosts = []
for host in candidate_hosts:
host_backing = host_helper.get_host_instance_backing(host)
if host_backing == from_backing:
from_hosts.append(host)
elif host_backing == to_backing:
to_hosts.append(host)
else:
pass
LOG.info(
"Candidate hosts that already have the right from backing {}: {}".
format(from_backing, from_hosts))
LOG.info(
"Candidate hosts that already have the right to backing {}: {}".
format(to_backing, to_hosts))
# Determine what hosts to use
if not from_hosts and to_hosts:
to_host = random.choice(to_hosts)
candidate_hosts.remove(to_host)
from_host = random.choice(candidate_hosts)
elif not to_hosts and from_hosts:
from_host = random.choice(from_hosts)
candidate_hosts.remove(from_host)
to_host = random.choice(candidate_hosts)
elif not to_hosts and not from_hosts:
to_host = random.choice(candidate_hosts)
candidate_hosts.remove(to_host)
from_host = random.choice(candidate_hosts)
else:
to_host = random.choice(to_hosts)
from_host = random.choice(from_hosts)
LOG.info("From host is: {}".format(from_host))
LOG.info("To host is: {}".format(to_host))
LOG.tc_step(
"Check from host backing and convert to {} if necessary".format(
from_backing))
host_helper.set_host_storage_backing(from_host, from_backing)
system_helper.wait_for_host_values(
from_host,
availability=HostAvailState.AVAILABLE,
timeout=120,
fail_ok=False)
LOG.tc_step(
"Check to host backing and convert to {} if necessary".format(
to_backing))
new_to_backing = host_helper.set_host_storage_backing(
to_host, to_backing)
elif personality == "controller":
# For now, we don't want to host reinstall controller-0 since it will default to
# pxeboot, but this could be examined as a possible enhancement.
from_host = "controller-0"
to_host = "controller-1"
LOG.info("From host is: {}".format(from_host))
LOG.info("To host is: {}".format(to_host))
LOG.tc_step(
"Check from host backing and convert to {} if necessary".format(
from_backing))
host_helper.set_host_storage_backing(from_host, from_backing)
LOG.tc_step(
"Check to host backing and convert to {} if necessary".format(
to_backing))
new_to_backing = host_helper.set_host_storage_backing(
to_host, to_backing)
else:
# Backing doesn't apply to storage nodes so just pick from compatible hardware
from_host = random.choice(candidate_hosts)
candidate_hosts.remove(from_host)
to_host = random.choice(candidate_hosts)
LOG.tc_step(
"Create storage and interface profiles on the from host {}".format(
from_host))
prof_name = 'storprof_{}_{}'.format(
from_host, time.strftime('%Y%m%d_%H%M%S', time.localtime()))
storage_helper.create_storage_profile(from_host, profile_name=prof_name)
PROFILES_TO_DELETE.append(prof_name)
# Deleting VMs in case the remaining host(s) cannot handle all VMs
# migrating on lock, particularly important in the case of AIO-DX systems.
LOG.tc_step(
"Delete all VMs and lock the host before applying the storage profile")
vm_helper.delete_vms()
HostsToRecover.add(to_host, scope='function')
system_helper.wait_for_host_values(from_host,
availability=HostAvailState.AVAILABLE,
timeout=120,
fail_ok=False)
system_helper.wait_for_host_values(to_host,
availability=HostAvailState.AVAILABLE,
timeout=120,
fail_ok=False)
# Negative test #1 - attempt to apply profile on unlocked host (should be rejected)
LOG.tc_step('Apply the storage-profile {} onto unlocked host:{}'.format(
prof_name, to_host))
cmd = 'host-apply-storprofile {} {}'.format(to_host, prof_name)
rc, msg = cli.system(cmd, fail_ok=True)
assert rc != 0, msg
host_helper.lock_host(to_host, swact=True)
# 3 conditions to watch for: no partitions, ready partitions and in-use
# partitions on the compute. If in-use, delete and freshly install host.
# If ready, delete all ready partitions to make room for potentially new
# partitions. If no partitions, just delete nova-local lvg.
if personality == "compute":
# Negative test #2 - attempt to apply profile onto host with existing
# nova-local (should be rejected)
LOG.tc_step(
'Apply the storage-profile {} onto host with existing nova-local:{}'
.format(prof_name, to_host))
cmd = 'host-apply-storprofile {} {}'.format(to_host, prof_name)
rc, msg = cli.system(cmd, fail_ok=True)
assert rc != 0, msg
# If we were simply switching backing (without applying a storage
# profile), the nova-local lvg deletion can be omitted according to design
LOG.tc_step("Delete nova-local lvg on to host {}".format(to_host))
cli.system("host-lvg-delete {} nova-local".format(to_host))
in_use = storage_helper.get_host_partitions(to_host, "In-Use")
if in_use:
# Negative test #3 - attempt to apply profile onto host with existing
# in-use partitions (should be rejected)
LOG.tc_step('Apply the storage-profile {} onto host with existing \
in-use partitions:{}'.format(prof_name, to_host))
cmd = 'host-apply-storprofile {} {}'.format(to_host, prof_name)
rc, msg = cli.system(cmd, fail_ok=True)
assert rc != 0, msg
LOG.tc_step(
"In-use partitions found. Must delete the host and freshly install before proceeding."
)
LOG.info("Host {} has in-use partitions {}".format(
to_host, in_use))
lab = InstallVars.get_install_var("LAB")
lab.update(create_node_dict(lab['compute_nodes'], 'compute'))
lab['boot_device_dict'] = create_node_boot_dict(lab['name'])
install_helper.open_vlm_console_thread(to_host)
LOG.tc_step("Delete the host {}".format(to_host))
cli.system("host-bulk-export")
cli.system("host-delete {}".format(to_host))
assert len(
system_helper.get_controllers()) > 1, "Host deletion failed"
cli.system("host-bulk-add hosts.xml")
system_helper.wait_for_host_values(
to_host, timeout=6000, availability=HostAvailState.ONLINE)
wait_for_disks(to_host)
ready = storage_helper.get_host_partitions(to_host, "Ready")
if ready:
LOG.tc_step(
"Ready partitions have been found. Must delete them before profile application"
)
LOG.info("Host {} has Ready partitions {}".format(to_host, ready))
for uuid in reversed(ready):
storage_helper.delete_host_partition(to_host, uuid)
# Don't bother restoring in this case since the system should be
# functional after profile is applied.
LOG.tc_step('Apply the storage-profile {} onto host:{}'.format(
prof_name, to_host))
cli.system('host-apply-storprofile {} {}'.format(to_host, prof_name))
LOG.tc_step("Unlock to host")
host_helper.unlock_host(to_host)
to_host_backing = host_helper.get_host_instance_backing(to_host)
LOG.info("To host backing was {} and is now {}".format(
new_to_backing, to_host_backing))
assert to_host_backing == from_backing, "Host backing was not changed on storage profile application"
if personality == "storage":
if not storage_helper.is_ceph_healthy():
skip("Cannot run test when ceph is not healthy")
LOG.tc_step("Delete the host {}".format(to_host))
cli.system("host-bulk-export")
cli.system("host-delete {}".format(to_host))
cli.system("host-bulk-add hosts.xml")
system_helper.wait_for_host_values(to_host,
timeout=6000,
availability=HostAvailState.ONLINE)
wait_for_disks(to_host)
LOG.tc_step('Apply the storage-profile {} onto host:{}'.format(
prof_name, to_host))
cli.system('host-apply-storprofile {} {}'.format(to_host, prof_name))
# Re-provision interfaces through lab_setup.sh
LOG.tc_step("Reprovision the host as necessary")
files = ['interfaces']
con_ssh = ControllerClient.get_active_controller()
delete_lab_setup_files(con_ssh, to_host, files)
rc, msg = install_helper.run_lab_setup()
assert rc == 0, msg
LOG.tc_step("Unlock to host")
host_helper.unlock_host(to_host)
if personality == "controller":
# Note, install helper doesn't work on all labs. Some labs don't
# display BIOS type which causes install helper to fail
lab = InstallVars.get_install_var("LAB")
lab.update(create_node_dict(lab['controller_nodes'], 'controller'))
lab['boot_device_dict'] = create_node_boot_dict(lab['name'])
install_helper.open_vlm_console_thread(to_host)
LOG.tc_step("Delete the host {}".format(to_host))
cli.system("host-bulk-export")
cli.system("host-delete {}".format(to_host))
assert len(system_helper.get_controllers()) > 1, "Host deletion failed"
cli.system("host-bulk-add hosts.xml")
system_helper.wait_for_host_values(to_host,
timeout=6000,
availability=HostAvailState.ONLINE)
wait_for_disks(to_host)
LOG.tc_step("Apply the storage-profile {} onto host:{}".format(
prof_name, to_host))
cli.system("host-apply-storprofile {} {}".format(to_host, prof_name))
# Need to re-provision everything on node through lab_setup (except storage)
LOG.tc_step("Reprovision the host as necessary")
files = [
'interfaces', 'cinder_device', 'vswitch_cpus', 'shared_cpus',
'extend_cgts_vg', 'addresses'
]
con_ssh = ControllerClient.get_active_controller()
delete_lab_setup_files(con_ssh, to_host, files)
rc, msg = install_helper.run_lab_setup()
assert rc == 0, msg
LOG.tc_step("Unlock to host")
host_helper.unlock_host(to_host)
to_host_backing = host_helper.get_host_instance_backing(to_host)
LOG.info("To host backing was {} and is now {}".format(
new_to_backing, to_host_backing))
assert to_host_backing == from_backing, "Host backing was not changed on storage profile application"
def test_set_hosts_storage_backing_equal(instance_backing, number_of_hosts):
"""
Modify hosts storage backing if needed so that system has exact number
of hosts in given instance backing
Args:
instance_backing:
number_of_hosts:
Test Steps:
- Calculate the hosts to be configured based on test params
- Configure hosts to meet given criteria
- Check number of hosts in given instance backing is as specified
"""
host_num_mapping = {'zero': 0, 'one': 1, 'two': 2}
number_of_hosts = host_num_mapping[number_of_hosts]
LOG.tc_step("Calculate the hosts to be configured based on test params")
candidate_hosts = get_candidate_hosts(number_of_hosts=number_of_hosts)
hosts_with_backing = \
host_helper.get_hosts_in_storage_backing(instance_backing)
if len(hosts_with_backing) == number_of_hosts:
LOG.info("Already have {} hosts in {} backing. Do "
"nothing".format(number_of_hosts, instance_backing))
return
elif len(hosts_with_backing) < number_of_hosts:
backing_to_config = instance_backing
number_to_config = number_of_hosts - len(hosts_with_backing)
hosts_pool = list(set(candidate_hosts) - set(hosts_with_backing))
else:
backing_to_config = 'remote' if 'image' in instance_backing else \
'local_image'
number_to_config = len(hosts_with_backing) - number_of_hosts
hosts_pool = hosts_with_backing
LOG.tc_step("Delete vms if any to prepare for system configuration "
"change with best effort")
vm_helper.delete_vms(fail_ok=True)
hosts_to_config = hosts_pool[0:number_to_config]
LOG.tc_step("Configure following hosts to {} backing: "
"{}".format(hosts_to_config, backing_to_config))
for host in hosts_to_config:
host_helper.set_host_storage_backing(host=host,
inst_backing=backing_to_config,
unlock=False,
wait_for_configured=False)
HostsToRecover.add(host)
host_helper.unlock_hosts(hosts_to_config,
check_hypervisor_up=True,
fail_ok=False)
LOG.tc_step("Waiting for hosts in {} aggregate".format(backing_to_config))
for host in hosts_to_config:
host_helper.wait_for_host_in_instance_backing(
host, storage_backing=backing_to_config)
LOG.tc_step("Check number of {} hosts is {}".format(
instance_backing, number_of_hosts))
assert number_of_hosts == \
len(host_helper.get_hosts_in_storage_backing(instance_backing)), \
"Number of {} hosts is not {} after " \
"configuration".format(instance_backing, number_of_hosts)
def _test_pci_resource_usage(vif_model_check):
"""
Create a vm under test with specified vifs for tenant network
Returns (str): id of vm under test
"""
vif_model, base_vm, flavor_id, nics_to_test, seg_id, net_type, pnet_name, extra_pcipt_net_name, extra_pcipt_net = \
vif_model_check
LOG.tc_step("Ensure core/vm quota is sufficient")
if 'sriov' in vif_model:
vm_type = 'sriov'
resource_param = 'pci_vfs_used'
max_resource = 'pci_vfs_configured'
else:
vm_type = 'pcipt'
resource_param = 'pci_pfs_used'
max_resource = 'pci_pfs_configured'
LOG.tc_step(
"Get resource usage for {} interface before booting VM(s)".format(
vif_model))
LOG.info("provider net for {} interface: {}".format(vif_model, pnet_name))
assert pnet_name, "provider network for {} interface is not found".format(
vif_model)
total_val, pre_resource_value = nova_helper.get_pci_interface_stats_for_providernet(
pnet_name, fields=(max_resource, resource_param))
LOG.info("Resource Usage {} for {}. Resource configured: {}".format(
pre_resource_value, vif_model, total_val))
expt_change = 2 if vif_model == 'pci-passthrough' and extra_pcipt_net else 1
vm_limit = int((total_val - pre_resource_value) /
expt_change) if vif_model == 'pci-passthrough' else 5
vm_helper.ensure_vms_quotas(vm_limit + 5)
vms_under_test = []
for i in range(vm_limit):
LOG.tc_step("Boot a vm with {} vif model on {} net".format(
vif_model, net_type))
vm_id = vm_helper.boot_vm(name=vif_model,
flavor=flavor_id,
cleanup='function',
nics=nics_to_test)[1]
vms_under_test.append(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id, fail_ok=False)
if vm_type == 'pcipt':
LOG.tc_step("Add vlan to pci-passthrough interface for VM.")
vm_helper.add_vlan_for_vm_pcipt_interfaces(vm_id=vm_id,
net_seg_id=seg_id)
LOG.tc_step(
"Ping vm over mgmt and {} nets from itself".format(net_type))
vm_helper.ping_vms_from_vm(to_vms=vm_id,
from_vm=vm_id,
net_types=['mgmt', net_type])
LOG.tc_step(
"Check resource usage for {} interface increased by 1".format(
vif_model))
resource_value = nova_helper.get_provider_net_info(
pnet_name, field=resource_param)
assert pre_resource_value + expt_change == resource_value, "Resource usage for {} is not increased by {}". \
format(vif_model, expt_change)
pre_resource_value = resource_value
for vm_to_del in vms_under_test:
LOG.tc_step(
"Check resource usage for {} interface reduced by 1 after deleting a vm"
.format(vif_model))
vm_helper.delete_vms(vm_to_del, check_first=False, stop_first=False)
resource_val = common.wait_for_val_from_func(
expt_val=pre_resource_value - expt_change,
timeout=30,
check_interval=3,
func=nova_helper.get_provider_net_info,
providernet_id=pnet_name,
field=resource_param)[1]
assert pre_resource_value - expt_change == resource_val, "Resource usage for {} is not reduced by {}". \
format(vif_model, expt_change)
pre_resource_value = resource_val
def test_pcipt_robustness(self, pcipt_prep):
"""
TC3_robustness: PCI-passthrough by locking and rebooting pci_vm host
Args:
pcipt_prep: test fixture to set up test environment and get proper pci nets/hosts/seg_id
Setups:
- select a providernet with pcipt interfaces configured
- get pci hosts configured with same above providernet
- get one network under above providernet (or two for CX4 nic)
Test Steps:
- Boot 2 pcipt vms with pci-passthrough vif over selected network
- Verify resource usage for providernet is increased as expected
- Lock pci_vm host and ensure vm migrated to other host (or fail to lock if no other pcipt host available)
- (Delete above tested pcipt vm if only two pcipt hosts available)
- Lock host for another pcipt vm, and lock is successful
- Verify vms' pci-pt interfaces reachable and resource usage for pnet as expected
- 'sudo reboot -f' pci_vm host, and ensure vm evacuated or up on same host if no other pcipt host available
- Repeat above step for another pcipt vm
- Verify vms' pci-pt interfaces reachable and resource usage for pnet unchanged
Teardown:
- Delete vms, volumes, flavor created
- Recover hosts if applicable
"""
net_type, pci_net_name, pci_hosts, pnet_id, nics, min_vcpu_host, seg_id, vm_num, vm_vcpus, pfs_use_init = \
pcipt_prep
vif_model = 'pci-passthrough'
# Create flavor with calculated vcpu number
LOG.fixture_step(
"Create a flavor with dedicated cpu policy and {} vcpus".format(
vm_vcpus))
flavor_id = nova_helper.create_flavor(
name='dedicated_{}vcpu'.format(vm_vcpus), ram=1024,
vcpus=vm_vcpus)[1]
ResourceCleanup.add('flavor', flavor_id, scope='module')
extra_specs = {
FlavorSpec.CPU_POLICY: 'dedicated',
}
# FlavorSpec.PCI_NUMA_AFFINITY: 'preferred'} # LP1854516
nova_helper.set_flavor(flavor=flavor_id, **extra_specs)
# Boot vms with 2 {} vifs each, and wait for pingable
LOG.tc_step("Boot {} vms with 2 {} vifs each".format(
vm_num, vif_model))
vms = []
for i in range(vm_num):
LOG.info("Booting pci-passthrough vm{}".format(i + 1))
vm_id = vm_helper.boot_vm(flavor=flavor_id,
nics=nics,
cleanup='function')[1]
vms.append(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
vm_helper.add_vlan_for_vm_pcipt_interfaces(vm_id,
seg_id,
init_conf=True)
# TODO: feature unavailable atm. Update required
# pfs_use_post_boot = nova_helper.get_provider_net_info(pnet_id, field='pci_pfs_used')
# resource_change = 2 if isinstance(seg_id, dict) else 1
# assert pfs_use_post_boot - pfs_use_init == vm_num * resource_change, "Number of PCI pfs used is not as expected"
check_vm_pci_interface(vms=vms, net_type=net_type)
HostsToRecover.add(pci_hosts)
# pfs_use_pre_action = pfs_use_post_boot
iter_count = 2 if len(pci_hosts) < 3 else 1
for i in range(iter_count):
if i == 1:
LOG.tc_step(
"Delete a pcipt vm and test lock and reboot pcipt host again for success pass"
)
vm_helper.delete_vms(vms=vms[1])
vms.pop()
# TODO: feature unavailable atm. Update required
# pfs_use_pre_action -= resource_change
# common.wait_for_val_from_func(expt_val=pfs_use_pre_action, timeout=30, check_interval=3,
# func=nova_helper.get_provider_net_info,
# providernet_id=pnet_id, field='pci_pfs_used')
LOG.tc_step("Test lock {} vms hosts started - iter{}".format(
vif_model, i + 1))
for vm in vms:
pre_lock_host = vm_helper.get_vm_host(vm)
assert pre_lock_host in pci_hosts, "VM is not booted on pci_host"
LOG.tc_step("Lock host of {} vms: {}".format(
vif_model, pre_lock_host))
code, output = host_helper.lock_host(host=pre_lock_host,
check_first=False,
swact=True,
fail_ok=True)
post_lock_host = vm_helper.get_vm_host(vm)
assert post_lock_host in pci_hosts, "VM is not on pci host after migrating"
if len(pci_hosts) < 3 and i == 0:
assert 5 == code, "Expect host-lock fail due to migration of vm failure. Actual: {}".format(
output)
assert pre_lock_host == post_lock_host, "VM host should not change when no other host to migrate to"
else:
assert 0 == code, "Expect host-lock successful. Actual: {}".format(
output)
assert pre_lock_host != post_lock_host, "VM host did not change"
LOG.tc_step("Unlock {}".format(pre_lock_host))
check_vm_pci_interface(vms, net_type=net_type)
host_helper.unlock_host(pre_lock_host, available_only=True)
# TODO: feature unavailable atm. Update required
# pfs_use_post_lock = nova_helper.get_provider_net_info(pnet_id, field='pci_pfs_used')
# assert pfs_use_pre_action == pfs_use_post_lock, "Number of PCI pfs used after host-lock is not as expected"
LOG.tc_step("Test evacuate {} vms started - iter{}".format(
vif_model, i + 1))
for vm in vms:
pre_evac_host = vm_helper.get_vm_host(vm)
LOG.tc_step(
"Reboot {} and ensure {} vm are evacuated when applicable".
format(pre_evac_host, vif_model))
code, output = vm_helper.evacuate_vms(pre_evac_host,
vm,
fail_ok=True,
wait_for_host_up=True)
if len(pci_hosts) < 3 and i == 0:
assert 1 == code, "Expect vm stay on same host due to migration fail. Actual:{}".format(
output)
vm_helper.wait_for_vm_status(vm_id=vm)
else:
assert 0 == code, "Expect vm evacuated to other host. Actual: {}".format(
output)
post_evac_host = vm_helper.get_vm_host(vm)
assert post_evac_host in pci_hosts, "VM is not on pci host after evacuation"
check_vm_pci_interface(vms, net_type=net_type)
def test_idle_kpi(collect_kpi):
if not collect_kpi:
skip("KPI only test. Skip due to kpi collection is not enabled")
LOG.tc_step("Delete vms and volumes on system if any")
vm_helper.delete_vms()
is_aio = system_helper.is_aio_system()
active_con = system_helper.get_active_controller_name()
con_ssh = ControllerClient.get_active_controller()
cpu_arg = ''
if is_aio:
LOG.info("AIO system found, check platform cores only")
cpu_arg = ' -P '
platform_cores_per_proc = host_helper.get_host_cpu_cores_for_function(
hostname=active_con,
func='Platform',
core_type='log_core',
thread=None,
con_ssh=con_ssh)
platform_cpus = []
for proc in platform_cores_per_proc:
platform_cpus += platform_cores_per_proc[proc]
cpu_arg += ','.join([str(val) for val in platform_cpus])
LOG.tc_step(
"Sleep for 5 minutes, then monitor for cpu and memory usage every 10 seconds for 5 minutes"
)
time.sleep(300)
output = con_ssh.exec_cmd(
'sar -u{} 10 30 -r | grep --color=never "Average"'.format(cpu_arg),
expect_timeout=600,
fail_ok=False)[1]
# Sample output:
# controller-1:~$ sar -u -P 0,1 1 3 -r | grep Average
# Average: CPU %user %nice %system %iowait %steal %idle
# Average: 0 8.52 0.00 4.92 1.97 0.00 84.59
# Average: 1 14.19 0.00 4.73 0.00 0.00 81.08
# Average: kbmemfree kbmemused %memused kbbuffers kbcached kbcommit %commit kbactive kbinact kbdirty
# Average: 105130499 26616873 20.20 203707 782956 63556293 48.24 24702756 529517 579
lines = output.splitlines()
start_index = 0
for i in range(len(lines)):
if lines(i).startswith('Average:'):
start_index = i
break
lines = lines[start_index:]
# Parse mem usage stats
mem_vals = lines.pop(-1).split()
mem_headers = lines.pop(-1).split()
mem_usage_index = mem_headers.index('%memused')
mem_usage = float(mem_vals[mem_usage_index])
# Parse cpu usage stats
cpu_headers = lines.pop(0).split()
cpu_lines = [line.split() for line in lines]
idle_cpu_index = cpu_headers.index('%idle')
cpus_idle = [float(cpu_vals[idle_cpu_index]) for cpu_vals in cpu_lines]
avg_cpu_idle = sum(cpus_idle) / len(cpu_lines)
avg_cpu_usage = round(100 - avg_cpu_idle, 4)
cpu_kpi_name = Idle.NAME_CPU
mem_kpi_name = Idle.NAME_MEM
if not container_helper.is_stx_openstack_deployed():
cpu_kpi_name += '_platform'
mem_kpi_name += '_platform'
kpi_log_parser.record_kpi(local_kpi_file=collect_kpi,
kpi_name=cpu_kpi_name,
kpi_val=avg_cpu_usage,
uptime=5,
unit='Percentage',
fail_ok=False)
kpi_log_parser.record_kpi(local_kpi_file=collect_kpi,
kpi_name=mem_kpi_name,
kpi_val=mem_usage,
uptime=5,
unit='Percentage',
fail_ok=False)
def test_neutron_port_security(setup_port_security, port_security):
"""
Test neutron port security enabled/disabled with IP spoofing
Args:
n/a
Pre-requisites:
- System should be have ml2 driver capable
Setups:
- Enable Extension ml2 driver in system if its not already enabled
- Enable Port Security in the network if its not already enabled
Test Steps:
- Set port_security on existing neutron networks
- Boot 2 vms to test where userdata sets a static ip that is different than nova show
- Verify IP Spoofing fails when port security is enabled, and vise versa
- Delete spoofed vms
- Boot another 2 vms without userdata
- Verify ping between VMs work when without ip spoofing attach
- Change vm2 mac address and verify IP spoofing fails only when port security is enabled
- Revert vm2 mac address and verify ping between vms work again
Teardown:
- Delete created vms, volumes, etc
"""
internal_net_id, nics = setup_port_security
port_security_enabled = True if port_security == 'enabled' else False
LOG.tc_step(
"Ensure port security is {} on neutron networks".format(port_security))
internal_net_port_security = eval(
network_helper.get_network_values(internal_net_id,
'port_security_enabled')[0])
if internal_net_port_security is not port_security_enabled:
LOG.info('Set port security to {} on existing neutron networks'.format(
port_security))
networks = network_helper.get_networks(auth_info=Tenant.get('admin'))
for net in networks:
network_helper.set_network(
net_id=net, enable_port_security=port_security_enabled)
# Test IP protection
LOG.tc_step(
"Launch two VMs with port security {} with mismatch IP in userdata than neutron port"
.format(port_security))
vms = []
for i in (1, 2):
user_data = '{}/vm{}-userdata.txt'.format(StxPath.USERDATA, i)
vm_name = 'vm{}_mismatch_ip_ps_{}'.format(i, port_security)
vm = vm_helper.boot_vm(name=vm_name,
nics=nics,
cleanup='function',
user_data=user_data)[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm)
vms.append(vm)
vm1, vm2 = vms
vm_helper.ping_vms_from_vm(to_vms=vm2,
from_vm=vm1,
net_types=['mgmt'],
retry=10)
vm2_ip = '10.1.0.2'
expt_res = 'fails' if port_security_enabled else 'succeeds'
LOG.tc_step(
"With port security {}, verify ping over internal net {} with mismatch IPs"
.format(port_security, expt_res))
packet_loss_rate = _ping_server(vm1,
ip_addr=vm2_ip,
fail_ok=port_security_enabled)
if port_security_enabled:
assert packet_loss_rate == 100, "IP spoofing succeeded when port security is enabled"
LOG.info("Delete VMs with mismatch IPs")
vm_helper.delete_vms(vms)
# Test MAC protection
LOG.tc_step(
"Launch two VMs without IP Spoofing and check ping between vms works")
vms = []
for i in (1, 2):
vm = vm_helper.boot_vm(name='vm{}_ps_{}'.format(i, port_security),
nics=nics,
cleanup='function')[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm)
vms.append(vm)
vm1, vm2 = vms
vm_helper.ping_vms_from_vm(vm2,
from_vm=vm1,
net_types=['mgmt', 'internal'])
LOG.tc_step(
"With port security {}, change VM mac address and ensure ping over internal net {}"
.format(port_security, expt_res))
origin_mac_addr = network_helper.get_ports(server=vm2,
network=internal_net_id,
field='MAC Address')[0]
vm2_ip = network_helper.get_internal_ips_for_vms(vm2)[0]
new_mac_addr = _change_mac_address(vm2, origin_mac_addr)
packet_loss_rate = _ping_server(vm1,
ip_addr=vm2_ip,
fail_ok=port_security_enabled)
if port_security_enabled:
assert packet_loss_rate == 100, "IP spoofing succeeded when port security is enabled"
LOG.tc_step(
"With port security {}, revert VM mac address and ensure ping over internal net succeeds"
.format(port_security))
_change_mac_address(vm2, new_mac_addr, origin_mac_addr)
_ping_server(vm1, ip_addr=vm2_ip, fail_ok=False)
def test_delete_vm():
for i in range(20):
vm_id = vm_helper.boot_vm('del_one', source='image')[1]
vm_helper.delete_vms(vm_id, stop_first=False, delete_volumes=False)
def test_lock_stor_check_osds_down(stx_openstack_required, host):
"""
This test is adapted from
us69932_tc3_ceph_mon_maintenance_operations from us69932_ceph_monitoring.odt
The goal of this test is to check that all OSDs go down on a locked storage
node. There are two variants:
1. Lock 'storage-0' which is a ceph monitor
2. Lock a storage node that is not 'storage-0', i.e. not a ceph monitor
Args:
- None
Setup:
- Requires system with storage nodes
Test Steps:
1. Lock storage node
2. Check
- CEPH cluster is in HEALTH_WARN
- Ensure all OSDs on the locked storage node are down
- Check that the appropriate alarms are raised:
3. Unlock storage node
- ensure CEPH is HEALTH_OK
- ensure all OSDs on unlocked node are up
- Check that alarms are cleared
Note: If the storage node to be locked is monitor, we also expect to see
the mon down alarm.
What defects this addresses:
1. CGTS-2609 - Ceph processes fail to start after storage node reboot
Notes:
- Updated test to write to disk to add I/O load on system
"""
con_ssh = ControllerClient.get_active_controller()
if host == 'any':
storage_nodes = system_helper.get_hosts(personality='storage')
LOG.info('System has {} storage nodes:'.format(storage_nodes))
storage_nodes.remove('storage-0')
node_id = random.randint(0, len(storage_nodes) - 1)
host = storage_nodes[node_id]
LOG.tc_step("Delete existing VMs")
vm_helper.delete_vms()
LOG.tc_step("Boot various VMs")
vms = vm_helper.boot_vms_various_types(cleanup="function")
vm_threads = []
LOG.tc_step("SSH to VMs and write to disk")
end_event = Events("End dd in vms")
try:
for vm in vms:
vm_thread = vm_helper.write_in_vm(vm, end_event=end_event, expect_timeout=40)
vm_threads.append(vm_thread)
LOG.tc_step('Lock storage node {}'.format(host))
HostsToRecover.add(host)
host_helper.lock_host(host, check_first=False)
LOG.tc_step('Determine the storage group for host {}'.format(host))
storage_group, msg = storage_helper.get_storage_group(host)
LOG.info(msg)
LOG.tc_step('Check that host lock alarm is raised when {} is locked'.format(host))
assert system_helper.wait_for_alarm(alarm_id=EventLogID.HOST_LOCK, entity_id=host, strict=False)[0], \
"Alarm {} not raised".format(EventLogID.HOST_LOCK)
LOG.tc_step('Check health of CEPH cluster')
ceph_healthy = storage_helper.is_ceph_healthy(con_ssh)
assert not ceph_healthy
LOG.tc_step('Check that OSDs are down')
osd_list = storage_helper.get_osds(host, con_ssh)
for osd_id in osd_list:
osd_up = storage_helper.is_osd_up(osd_id, con_ssh)
msg = 'OSD ID {} is up but should be down'.format(osd_id)
assert not osd_up, msg
msg = 'OSD ID {} is down as expected'.format(osd_id)
LOG.info(msg)
LOG.tc_step('Check that loss of replication alarm is raised')
assert system_helper.wait_for_alarm(alarm_id=EventLogID.STORAGE_LOR)[0], \
"Alarm {} not raised".format(EventLogID.STORAGE_LOR)
LOG.tc_step('Check that ceph is in health warn')
assert system_helper.wait_for_alarm(alarm_id=EventLogID.STORAGE_ALARM_COND)[0], \
"Alarm {} not raised".format(EventLogID.STORAGE_ALARM_COND)
# We're waiting 5 minutes for ceph rebalancing to be performed
# DO NOT REMOVE. This is part of the test.
time.sleep(300)
LOG.tc_step('Unlock storage node')
rtn_code, out = host_helper.unlock_host(host)
assert rtn_code == 0, out
health = False
end_time = time.time() + 40
while time.time() < end_time:
health = storage_helper.is_ceph_healthy(con_ssh)
if health is True:
break
assert health, "Ceph did not become healthy"
LOG.tc_step('Check that host lock alarm is cleared when {} is unlocked'.format(host))
assert system_helper.wait_for_alarm_gone(EventLogID.HOST_LOCK, entity_id=host, strict=False), \
"Alarm {} not cleared".format(EventLogID.HOST_LOCK)
LOG.tc_step('Check that the replication group alarm is cleared')
assert system_helper.wait_for_alarm_gone(EventLogID.STORAGE_LOR), \
"Alarm {} not cleared".format(EventLogID.STORAGE_LOR)
LOG.tc_step('Check that the Storage Alarm Condition is cleared')
assert system_helper.wait_for_alarm_gone(EventLogID.STORAGE_ALARM_COND), \
"Alarm {} not cleared".format(EventLogID.STORAGE_ALARM_COND)
LOG.tc_step('Check OSDs are up after unlock')
for osd_id in osd_list:
osd_up = storage_helper.is_osd_up(osd_id, con_ssh)
msg = 'OSD ID {} should be up but is not'.format(osd_id)
assert osd_up, msg
LOG.tc_step('Check health of CEPH cluster')
end_time = time.time() + 40
while time.time() < end_time:
ceph_healthy = storage_helper.is_ceph_healthy(con_ssh)
if ceph_healthy is True:
break
for vm_thread in vm_threads:
assert vm_thread.res is True, "Writing in vm stopped unexpectedly"
finally:
# wait_for_thread_end needs to be called even if test failed in the middle, otherwise thread will not end
end_event.set()
for vm_thread in vm_threads:
vm_thread.wait_for_thread_end(timeout=20)
LOG.tc_step("Delete existing VMs")
vm_helper.delete_vms()
def _test_cpu_pol_dedicated_shared_coexists(vcpus_dedicated, vcpus_shared, pol_source, boot_source):
"""
Test two vms coexisting on the same host, one with the dedicated cpu property, and one with the shared cpu property.
Args:
vcpus_dedicated: Amount of vcpu(s) to allocate for the vm with the dedicated CPU_POLICY.
vcpus_shared: Amount of vcpu(s) to allocate for the vm with the shared CPU_POLICY.
pol_source: Where the CPU_POLICY is set from.
boot_source: The boot media the vm will use to boot.
Test Setups:
- Create two flavors, one for each vm.
- If using 'flavor' for pol_source, set extra specs for the CPU_POLICY.
- If using 'image' for pol_source, set ImageMetaData for the CPU_POLICY.
- If using 'volume' for boot_source, create volume from tis image.
- If using 'image' for boot_source, use tis image.
- Determine the amount of free vcpu(s) on the compute before testing.
Test Steps:
- Boot the first vm with CPU_POLICY: dedicated.
- Wait until vm is pingable from natbox.
- Check vm topology for vcpu(s).
- Determine the amount of free vcpu(s) on the compute.
- Boot the second vm with CPU_POLICY: shared.
- Wait until vm is pingable from natbox.
- Check vm topology for vcpu(s).
- Delete vms
- Determine the amount of free vcpu(s) on the compute after testing.
- Compare free vcpu(s) on the compute before and after testing, ensuring they are the same.
Test Teardown
- Delete created volumes and flavors
"""
LOG.tc_step("Getting host list")
target_hosts = host_helper.get_hypervisors(state='up')
target_host = target_hosts[0]
storage_backing = host_helper.get_host_instance_backing(host=target_host)
if 'image' in storage_backing:
storage_backing = 'local_image'
elif 'remote' in storage_backing:
storage_backing = 'remote'
image_id = glance_helper.get_image_id_from_name(GuestImages.DEFAULT['guest'], strict=True)
pre_test_cpus = host_helper.get_vcpus_for_computes(field='used_now')
collection = ['dedicated', 'shared']
vm_ids = []
for x in collection:
if x == 'dedicated':
vcpus = vcpus_dedicated
else:
vcpus = vcpus_shared
LOG.tc_step("Create {} flavor with {} vcpus".format(x, vcpus))
flavor_id = nova_helper.create_flavor(name=x, vcpus=vcpus, storage_backing=storage_backing)[1]
ResourceCleanup.add('flavor', flavor_id)
if pol_source == 'flavor':
LOG.tc_step("Set CPU_POLICY for {} flavor".format(x))
specs = {FlavorSpec.CPU_POLICY: x}
nova_helper.set_flavor(flavor_id, **specs)
else:
LOG.tc_step("Create image with CPU_POLICY: {}".format(x))
image_meta = {ImageMetadata.CPU_POLICY: x}
image_id = glance_helper.create_image(name='cpu_pol_{}'.format(x), cleanup='function', **image_meta)[1]
if boot_source == 'volume':
LOG.tc_step("Create volume from image")
source_id = cinder_helper.create_volume(name='cpu_pol_{}'.format(x), source_id=image_id)[1]
ResourceCleanup.add('volume', source_id)
else:
source_id = image_id
pre_boot_cpus = host_helper.get_vcpus_for_computes(field='used_now')
LOG.tc_step("Booting cpu_pol_{}".format(x))
vm_id = vm_helper.boot_vm(name='cpu_pol_{}'.format(x), flavor=flavor_id, source=boot_source,
source_id=source_id, avail_zone='nova', vm_host=target_host, cleanup='function')[1]
vm_ids.append(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
check_helper.check_topology_of_vm(vm_id, vcpus=vcpus, cpu_pol=x, vm_host=target_host,
prev_total_cpus=pre_boot_cpus[target_host])
LOG.tc_step("Deleting both dedicated and shared vms")
vm_helper.delete_vms(vms=vm_ids)
post_delete_cpus = host_helper.get_vcpus_for_computes(field='used_now')
assert post_delete_cpus == pre_test_cpus, "vcpu count after test does not equal vcpu count before test"
def test_evacuate_shared_cpu_vm(self, target_hosts, add_shared_cpu, add_admin_role_func):
"""
Test that instance with shared vcpu can be evacuated and that the vm still has shared vcpu after evacuation
Setup:
- Configure at least two computes to have shared cpus via
'system host-cpu-modify -f shared p0=1,p1=1 <hostname>' (module)
Test Steps:
- Create 2 VMs with shared vcpu
- Boot a vm for each of the created flavors
- Ensure all vms are booted successfully and validate the shared vcpus
- Evacuate the vms
- Ensure evacuation is successful and validate the shared vcpus
Teardown:
- Delete created vms and flavors
- Set shared cpus to 0 (default setting) on the compute node under test (module)
"""
storage_backing, shared_cpu_hosts, max_vcpus_per_proc = add_shared_cpu
vm_helper.delete_vms()
prev_total_vcpus = host_helper.get_vcpus_for_computes()
target_host = shared_cpu_hosts[0]
vms = []
vcpus = 2
shared_vcpu = 1
pcpus = vcpus - shared_vcpu
expt_increase = 0
LOG.tc_step("Create two 2 vcpu VMs each with 1 shared vcpu")
flv_id = create_shared_flavor(vcpus=vcpus, shared_vcpu=shared_vcpu, storage_backing=storage_backing)
for _ in range(2):
vm_id = vm_helper.boot_vm(name='shared_cpu', flavor=flv_id, fail_ok=False, avail_zone='nova',
vm_host=target_host, cleanup='function')[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
expt_increase += pcpus
LOG.tc_step("Check vm {} numa node setting via vm-topology".format(vm_id))
check_shared_vcpu(vm=vm_id, shared_vcpu=shared_vcpu, vcpus=vcpus, prev_total_vcpus=prev_total_vcpus,
expt_increase=expt_increase)
vms.append(vm_id)
LOG.tc_step("Evacuate vms")
vm_helper.evacuate_vms(target_host, vms_to_check=vms, ping_vms=True)
vm_hosts = []
LOG.tc_step("Check shared vcpus and numa settings for vms after evacuation")
for vm_ in vms:
vm_host = vm_helper.get_vm_host(vm_id=vm_)
vm_hosts.append(vm_host)
if len(list(set(vm_hosts))) == 1:
post_evac_expt_increase = pcpus * 2
else:
post_evac_expt_increase = pcpus
for vm_ in vms:
check_shared_vcpu(vm=vm_, expt_increase=post_evac_expt_increase,
prev_total_vcpus=prev_total_vcpus, shared_vcpu=shared_vcpu, vcpus=vcpus)
def test_attempt_to_delete_volume_associated_with_snapshot():
"""
This is a negative test to verify that volumes with associated snapshots
cannot be deleted.
Test Steps:
1. Create a volume
2. Launch a VM with that volume
3. Create a snapshot based on that VM
4. Delete the VM, leaving behind the volume and snapshot
5. Attempt to delete volume. Rejeted.
6. Delete the snapshot.
7. Delete the volume.
Teardown:
1. Delete VMs
2. Delete volumes
3. Delete snapshots
4. Delete images
Possible Improvements:
1. Could update test to use non-raw images, but determining size of of
image is more complex if the original file is no longer on the filesystem.
"""
LOG.tc_step("Get available images")
image_list = glance_helper.get_images()
if len(image_list) == 0:
skip("The test requires some images to be present")
# Filter out zero-sized images and non-raw images (latter is lazy)
image_uuid = vol_size = None
for image in image_list:
image_uuid = image
image_prop_s, image_prop_d = glance_helper.get_image_values(
image_uuid, ("size", "disk_format"))
if str(image_prop_s) == "0" or image_prop_d != "raw":
continue
else:
divisor = 1024 * 1024 * 1024
image_size = int(image_prop_s)
vol_size = int(math.ceil(image_size / divisor))
break
else:
skip("No usable images found")
LOG.tc_step("Create a cinder bootable volume")
# Check if lab has emc-vnx volume types. Use volume type = iscsi;
# Creating snapshot with emc-vnx(EMS San)
# is not supported yet.
volume_types = cinder_helper.get_volume_types(field='Name')
vol_type = 'iscsi' if any('emc' in t for t in volume_types) else None
vol_id = cinder_helper.create_volume(source_id=image_uuid,
vol_type=vol_type,
size=vol_size,
fail_ok=False,
cleanup='function')[1]
LOG.tc_step("Boot VM using newly created bootable volume")
vm_id = vm_helper.boot_vm(source="volume",
source_id=vol_id,
cleanup='function')[1]
assert vm_id, "Failed to boot VM"
# nova image-create generates a glance image of 0 size
# real snapshot is stored in cinder
LOG.tc_step("Create a snapshot based on that VM")
vm_name = vm_helper.get_vm_name_from_id(vm_id)
snapshot_name = vm_name + "_snapshot"
code, image_id, snapshot_id = vm_helper.create_image_from_vm(
vm_id,
image_name=snapshot_name,
cleanup='function',
expt_cinder_snapshot=True)
# We're deleting the VM, but leaving the volume and the snapshot
LOG.tc_step("Delete the VM")
vm_helper.delete_vms(vms=vm_id, fail_ok=False)
LOG.tc_step("Attempting to delete the volume with associated snapshot")
rc, out = cinder_helper.delete_volumes(vol_id, fail_ok=True)
assert rc == 1, "Volume deletion was expected to fail but instead succeeded"
LOG.tc_step("Delete the snapshot")
cinder_helper.delete_volume_snapshots(snapshot_id, fail_ok=False)
LOG.tc_step("Re-attempt volume deletion")
# This step has been failing on ip33-36 and sm-1 due to volume delete
# rejected. After a minute or so,
# it was accepted though.
cinder_helper.delete_volumes(vol_id, fail_ok=False)
