def modify_system_backing(request):
"""
Issues in this fixture:
- Hardcoded compute name
- compute could be in bad state
- Does not work for CPE lab
- Lock unlock 6 times (could be only 3 times)
- Did not check original storage backing (lab could be configured with local_lvm by default)
"""
hostname, storage_backing = request.param
LOG.fixture_step("Modify {} storage backing to {}".format(
hostname, storage_backing))
host_helper.lock_host(hostname)
host_helper.set_host_storage_backing(hostname,
inst_backing=storage_backing,
lock=False)
host_helper.unlock_hosts(hostname)
def revert_host():
LOG.fixture_step(
"Revert {} storage backing to local_image".format(hostname))
host_helper.lock_host(hostname)
host_helper.set_host_storage_backing(hostname,
inst_backing='local_image',
lock=False)
host_helper.unlock_hosts(hostname)
request.addfinalizer(revert_host)
return storage_backing
def test_set_hosts_storage_backing_min(instance_backing, number_of_hosts):
"""
Modify hosts storage backing if needed so that system has minimal number of hosts in given instance backing
Args:
instance_backing:
number_of_hosts:
Test Steps:
- Calculate the hosts to be configured based on test params
- Configure hosts to meet given criteria
- Check number of hosts in given instance backing is as specified
"""
LOG.tc_step("Determine the hosts to configure")
hosts = host_helper.get_up_hypervisors()
hosts_len = len(hosts)
host_num_mapping = {'all': hosts_len, 'two': 2, 'one': 1}
number_of_hosts = host_num_mapping[number_of_hosts]
hosts_with_backing = host_helper.get_hosts_in_storage_backing(
instance_backing)
if len(hosts_with_backing) >= number_of_hosts:
LOG.info("Already have {} hosts in {} backing. Do nothing".format(
len(hosts_with_backing), instance_backing))
return
candidate_hosts = get_candidate_hosts(number_of_hosts=number_of_hosts)
number_to_config = number_of_hosts - len(hosts_with_backing)
hosts_to_config = list(set(candidate_hosts) -
set(hosts_with_backing))[0:number_to_config]
LOG.tc_step(
"Delete vms if any to prepare for system configuration change with best effort"
)
vm_helper.delete_vms(fail_ok=True)
LOG.tc_step("Configure following hosts to {} backing: {}".format(
hosts_to_config, instance_backing))
for host in hosts_to_config:
HostsToRecover.add(host)
host_helper.set_host_storage_backing(host=host,
inst_backing=instance_backing,
unlock=False,
wait_for_configured=False)
host_helper.unlock_hosts(hosts_to_config,
check_hypervisor_up=True,
fail_ok=False)
LOG.tc_step("Waiting for hosts in {} aggregate".format(instance_backing))
for host in hosts_to_config:
host_helper.wait_for_host_in_instance_backing(
host, storage_backing=instance_backing)
LOG.tc_step("Check number of {} hosts is at least {}".format(
instance_backing, number_of_hosts))
assert number_of_hosts <= len(host_helper.get_hosts_in_storage_backing(instance_backing)), \
"Number of {} hosts is less than {} after configuration".format(instance_backing, number_of_hosts)
def revert_host():
LOG.fixture_step(
"Revert {} storage backing to local_image".format(hostname))
host_helper.lock_host(hostname)
host_helper.set_host_storage_backing(hostname,
inst_backing='local_image',
lock=False)
host_helper.unlock_hosts(hostname)
def revert_host():
LOG.fixture_step("Revert {} storage backing to {} if needed".format(
target_host, original_backing))
host_helper.set_host_storage_backing(target_host,
inst_backing=original_backing,
check_first=True,
lock=True,
unlock=True)
def cleanup():
if not system_helper.is_storage_system():
skip("This test requires a storage system")
profiles_created = self._pop_cleanup_list('profile')
old_new_types = self._pop_cleanup_list('local_storage_type')
# Add hosts to module level recovery fixture in case of modify or unlock fail in following class level
# recovery attempt.
for item in old_new_types:
HostsToRecover.add(item[0], scope='module')
exceptions = []
try:
LOG.fixture_step("(class) Delete created storage profiles")
while profiles_created:
storage_helper.delete_storage_profile(
profile=profiles_created.pop())
except Exception as e:
LOG.exception(e)
exceptions.append(e)
try:
LOG.fixture_step(
"(class) Revert local storage backing for {}".format(
old_new_types))
while old_new_types:
host_to_revert, old_type, _ = old_new_types.pop()
LOG.info("Revert {} local storage to {}".format(
host_to_revert, old_type))
host_helper.set_host_storage_backing(host=host_to_revert,
inst_backing=old_type,
unlock=True)
except Exception as e:
LOG.exception(e)
exceptions.append(e)
assert not exceptions, "Failure occurred. Errors: {}".format(
exceptions)
def test_something(host_to_modify, storage_backing):
"""
Test parametrize the test function instead of the test fixture.
Args:
host_to_modify (str): fixture that returns the hostname under test
storage_backing (str): storage backing to configure
Setups:
- Select a host and record the storage backing before test starts (module)
Test Steps:
- Modify host storage backing to given storage backing if not already on it
- Create a flavor with specified storage backing
- Boot vm from above flavor
Teardown:
- Delete created vm, volume, flavor
- Modify host storage backing to its original config if not already on it (module)
"""
# Modify host storage backing withc check_first=True, so it will not modify if already in that backing
# if lock_host() has to be done inside test case, set swact=True, so it will handle CPE case
LOG.tc_step(
"Modify {} storage backing to {} if not already has the matching storage backing"
.format(host_to_modify, storage_backing))
host_helper.set_host_storage_backing(host_to_modify,
inst_backing=storage_backing,
check_first=True,
lock=True,
unlock=True)
LOG.tc_step("Create a flavor with specified storage backing")
flv_id = nova_helper.create_flavor(name='test_avoid_flv',
storage_backing=storage_backing)[1]
ResourceCleanup.add(resource_type='flavor', resource_id=flv_id)
LOG.tc_step("Boot vm from above flavor")
vm_id = vm_helper.boot_vm(name='test_avoid_vm', flavor=flv_id)[1]
ResourceCleanup.add(resource_type='vm', resource_id=vm_id)
def prepare_hosts(request):
"""
Setup:
Attempt to convert all computes to expected storage backing.
Skip test if unsuccessful.
Args:
request: expected host storage backing to run the test
Returns: hosts storage backing
Teardown:
Restore hosts to original state
"""
expected_storage_backing = request.param
avail_hosts = host_helper.get_hosts_in_storage_backing(
storage_backing=expected_storage_backing)
all_hosts = host_helper.get_hypervisors()
modified_hosts = {}
locked_hosts = []
avail_num = len(avail_hosts)
# Try to convert all available hypervisor hosts to the expected storage backing
for host in all_hosts:
if host not in avail_hosts:
original_storage = host_helper.get_host_instance_backing(host)
return_code, msg = host_helper.set_host_storage_backing(
host=host, inst_backing=expected_storage_backing, fail_ok=True)
if return_code == 0:
avail_num += 1
modified_hosts[host] = original_storage
elif return_code == 1: # Host locked, but cannot modify to the expected storage backing
locked_hosts.append(host)
else:
skip("Host {} cannot be locked. Error: {}".format(host, msg))
# Skip test if config failed
if avail_num < 2:
skip("Less than two hosts are successfully modified to {} backing".
format(expected_storage_backing))
# Teardown to restore hosts to original storage backing
def restore_hosts():
LOG.debug("Modifying hosts backing to original states..")
host_helper.unlock_hosts(locked_hosts)
for host in modified_hosts:
host_helper.set_host_storage_backing(host, modified_hosts[host])
request.addfinalizer(restore_hosts())
return request.param
def restore_hosts():
LOG.debug("Modifying hosts backing to original states..")
host_helper.unlock_hosts(locked_hosts)
for host in modified_hosts:
host_helper.set_host_storage_backing(host, modified_hosts[host])
def setup_local_storage(self, request, get_target_host):
local_storage = request.param
host = get_target_host
def cleanup():
if not system_helper.is_storage_system():
skip("This test requires a storage system")
profiles_created = self._pop_cleanup_list('profile')
old_new_types = self._pop_cleanup_list('local_storage_type')
# Add hosts to module level recovery fixture in case of modify or unlock fail in following class level
# recovery attempt.
for item in old_new_types:
HostsToRecover.add(item[0], scope='module')
exceptions = []
try:
LOG.fixture_step("(class) Delete created storage profiles")
while profiles_created:
storage_helper.delete_storage_profile(
profile=profiles_created.pop())
except Exception as e:
LOG.exception(e)
exceptions.append(e)
try:
LOG.fixture_step(
"(class) Revert local storage backing for {}".format(
old_new_types))
while old_new_types:
host_to_revert, old_type, _ = old_new_types.pop()
LOG.info("Revert {} local storage to {}".format(
host_to_revert, old_type))
host_helper.set_host_storage_backing(host=host_to_revert,
inst_backing=old_type,
unlock=True)
except Exception as e:
LOG.exception(e)
exceptions.append(e)
assert not exceptions, "Failure occurred. Errors: {}".format(
exceptions)
request.addfinalizer(cleanup)
origin_lvg = host_helper.get_host_instance_backing(host)
if origin_lvg != local_storage:
self._add_to_cleanup_list(to_cleanup=(host, origin_lvg,
local_storage),
cleanup_type='local_storage_type')
LOG.fixture_step(
"(class) Set {} local storage backing to {}".format(
host, local_storage))
host_helper.set_host_storage_backing(host,
inst_backing=local_storage,
check_first=False)
return local_storage, host
def _test_storage_profile(personality, from_backing, to_backing):
"""
This test creates a storage profile and then applies it to a node with
identical hardware, assuming one exists.
Storage profiles do not apply on controller nodes. Storage profiles can be
applied on controller+compute nodes, compute nodes and storage nodes.
Arguments:
- personality (string) - controller, compute or storage
- from_backing (string) - image, remote or None
- to_backing (string) - image, remote or None
Test Steps:
1. Query system and determine which nodes have compatible hardware.
2. Create a storage profile on one of those nodes
3. Apply the created storage profile on a compatible node*
4. Ensure the storage profiles have been successfully applied.
* If the node is a compute node or a controller+compute, we will also change
the backend if required for additional coverage.
Returns:
- Nothing
"""
global PROFILES_TO_DELETE
PROFILES_TO_DELETE = []
# Skip if test is not applicable to hardware under test
if personality == 'controller' and not system_helper.is_aio_system():
skip("Test does not apply to controller hosts without subtype compute")
hosts = system_helper.get_hosts(personality=personality)
if not hosts:
skip("No hosts of type {} available".format(personality))
if (from_backing == "remote" or to_backing
== "remote") and not system_helper.is_storage_system():
skip("This test doesn't apply to systems without storage hosts")
LOG.tc_step("Identify hardware compatible hosts")
hash_to_hosts = get_hw_compatible_hosts(hosts)
# Pick the hardware group that has the most compatible hosts
current_size = 0
candidate_hosts = []
for value in hash_to_hosts:
candidate_size = len(hash_to_hosts[value])
if candidate_size > current_size:
current_size = candidate_size
candidate_hosts = hash_to_hosts[value]
LOG.info(
"This is the total set of candidate hosts: {}".format(candidate_hosts))
if len(candidate_hosts) < 2:
skip("Insufficient hardware compatible hosts to run test")
# Rsync lab setup dot files between controllers
con_ssh = ControllerClient.get_active_controller()
_rsync_files_to_con1(con_ssh=con_ssh, file_to_check="force.txt")
# Take the hardware compatible hosts and check if any of them already have
# the backend that we want. This will save us test time.
new_to_backing = None
if personality == "compute":
from_hosts = []
to_hosts = []
for host in candidate_hosts:
host_backing = host_helper.get_host_instance_backing(host)
if host_backing == from_backing:
from_hosts.append(host)
elif host_backing == to_backing:
to_hosts.append(host)
else:
pass
LOG.info(
"Candidate hosts that already have the right from backing {}: {}".
format(from_backing, from_hosts))
LOG.info(
"Candidate hosts that already have the right to backing {}: {}".
format(to_backing, to_hosts))
# Determine what hosts to use
if not from_hosts and to_hosts:
to_host = random.choice(to_hosts)
candidate_hosts.remove(to_host)
from_host = random.choice(candidate_hosts)
elif not to_hosts and from_hosts:
from_host = random.choice(from_hosts)
candidate_hosts.remove(from_host)
to_host = random.choice(candidate_hosts)
elif not to_hosts and not from_hosts:
to_host = random.choice(candidate_hosts)
candidate_hosts.remove(to_host)
from_host = random.choice(candidate_hosts)
else:
to_host = random.choice(to_hosts)
from_host = random.choice(from_hosts)
LOG.info("From host is: {}".format(from_host))
LOG.info("To host is: {}".format(to_host))
LOG.tc_step(
"Check from host backing and convert to {} if necessary".format(
from_backing))
host_helper.set_host_storage_backing(from_host, from_backing)
system_helper.wait_for_host_values(
from_host,
availability=HostAvailState.AVAILABLE,
timeout=120,
fail_ok=False)
LOG.tc_step(
"Check to host backing and convert to {} if necessary".format(
to_backing))
new_to_backing = host_helper.set_host_storage_backing(
to_host, to_backing)
elif personality == "controller":
# For now, we don't want to host reinstall controller-0 since it will default to
# pxeboot, but this could be examined as a possible enhancement.
from_host = "controller-0"
to_host = "controller-1"
LOG.info("From host is: {}".format(from_host))
LOG.info("To host is: {}".format(to_host))
LOG.tc_step(
"Check from host backing and convert to {} if necessary".format(
from_backing))
host_helper.set_host_storage_backing(from_host, from_backing)
LOG.tc_step(
"Check to host backing and convert to {} if necessary".format(
to_backing))
new_to_backing = host_helper.set_host_storage_backing(
to_host, to_backing)
else:
# Backing doesn't apply to storage nodes so just pick from compatible hardware
from_host = random.choice(candidate_hosts)
candidate_hosts.remove(from_host)
to_host = random.choice(candidate_hosts)
LOG.tc_step(
"Create storage and interface profiles on the from host {}".format(
from_host))
prof_name = 'storprof_{}_{}'.format(
from_host, time.strftime('%Y%m%d_%H%M%S', time.localtime()))
storage_helper.create_storage_profile(from_host, profile_name=prof_name)
PROFILES_TO_DELETE.append(prof_name)
# Deleting VMs in case the remaining host(s) cannot handle all VMs
# migrating on lock, particularly important in the case of AIO-DX systems.
LOG.tc_step(
"Delete all VMs and lock the host before applying the storage profile")
vm_helper.delete_vms()
HostsToRecover.add(to_host, scope='function')
system_helper.wait_for_host_values(from_host,
availability=HostAvailState.AVAILABLE,
timeout=120,
fail_ok=False)
system_helper.wait_for_host_values(to_host,
availability=HostAvailState.AVAILABLE,
timeout=120,
fail_ok=False)
# Negative test #1 - attempt to apply profile on unlocked host (should be rejected)
LOG.tc_step('Apply the storage-profile {} onto unlocked host:{}'.format(
prof_name, to_host))
cmd = 'host-apply-storprofile {} {}'.format(to_host, prof_name)
rc, msg = cli.system(cmd, fail_ok=True)
assert rc != 0, msg
host_helper.lock_host(to_host, swact=True)
# 3 conditions to watch for: no partitions, ready partitions and in-use
# partitions on the compute. If in-use, delete and freshly install host.
# If ready, delete all ready partitions to make room for potentially new
# partitions. If no partitions, just delete nova-local lvg.
if personality == "compute":
# Negative test #2 - attempt to apply profile onto host with existing
# nova-local (should be rejected)
LOG.tc_step(
'Apply the storage-profile {} onto host with existing nova-local:{}'
.format(prof_name, to_host))
cmd = 'host-apply-storprofile {} {}'.format(to_host, prof_name)
rc, msg = cli.system(cmd, fail_ok=True)
assert rc != 0, msg
# If we were simply switching backing (without applying a storage
# profile), the nova-local lvg deletion can be omitted according to design
LOG.tc_step("Delete nova-local lvg on to host {}".format(to_host))
cli.system("host-lvg-delete {} nova-local".format(to_host))
in_use = storage_helper.get_host_partitions(to_host, "In-Use")
if in_use:
# Negative test #3 - attempt to apply profile onto host with existing
# in-use partitions (should be rejected)
LOG.tc_step('Apply the storage-profile {} onto host with existing \
in-use partitions:{}'.format(prof_name, to_host))
cmd = 'host-apply-storprofile {} {}'.format(to_host, prof_name)
rc, msg = cli.system(cmd, fail_ok=True)
assert rc != 0, msg
LOG.tc_step(
"In-use partitions found. Must delete the host and freshly install before proceeding."
)
LOG.info("Host {} has in-use partitions {}".format(
to_host, in_use))
lab = InstallVars.get_install_var("LAB")
lab.update(create_node_dict(lab['compute_nodes'], 'compute'))
lab['boot_device_dict'] = create_node_boot_dict(lab['name'])
install_helper.open_vlm_console_thread(to_host)
LOG.tc_step("Delete the host {}".format(to_host))
cli.system("host-bulk-export")
cli.system("host-delete {}".format(to_host))
assert len(
system_helper.get_controllers()) > 1, "Host deletion failed"
cli.system("host-bulk-add hosts.xml")
system_helper.wait_for_host_values(
to_host, timeout=6000, availability=HostAvailState.ONLINE)
wait_for_disks(to_host)
ready = storage_helper.get_host_partitions(to_host, "Ready")
if ready:
LOG.tc_step(
"Ready partitions have been found. Must delete them before profile application"
)
LOG.info("Host {} has Ready partitions {}".format(to_host, ready))
for uuid in reversed(ready):
storage_helper.delete_host_partition(to_host, uuid)
# Don't bother restoring in this case since the system should be
# functional after profile is applied.
LOG.tc_step('Apply the storage-profile {} onto host:{}'.format(
prof_name, to_host))
cli.system('host-apply-storprofile {} {}'.format(to_host, prof_name))
LOG.tc_step("Unlock to host")
host_helper.unlock_host(to_host)
to_host_backing = host_helper.get_host_instance_backing(to_host)
LOG.info("To host backing was {} and is now {}".format(
new_to_backing, to_host_backing))
assert to_host_backing == from_backing, "Host backing was not changed on storage profile application"
if personality == "storage":
if not storage_helper.is_ceph_healthy():
skip("Cannot run test when ceph is not healthy")
LOG.tc_step("Delete the host {}".format(to_host))
cli.system("host-bulk-export")
cli.system("host-delete {}".format(to_host))
cli.system("host-bulk-add hosts.xml")
system_helper.wait_for_host_values(to_host,
timeout=6000,
availability=HostAvailState.ONLINE)
wait_for_disks(to_host)
LOG.tc_step('Apply the storage-profile {} onto host:{}'.format(
prof_name, to_host))
cli.system('host-apply-storprofile {} {}'.format(to_host, prof_name))
# Re-provision interfaces through lab_setup.sh
LOG.tc_step("Reprovision the host as necessary")
files = ['interfaces']
con_ssh = ControllerClient.get_active_controller()
delete_lab_setup_files(con_ssh, to_host, files)
rc, msg = install_helper.run_lab_setup()
assert rc == 0, msg
LOG.tc_step("Unlock to host")
host_helper.unlock_host(to_host)
if personality == "controller":
# Note, install helper doesn't work on all labs. Some labs don't
# display BIOS type which causes install helper to fail
lab = InstallVars.get_install_var("LAB")
lab.update(create_node_dict(lab['controller_nodes'], 'controller'))
lab['boot_device_dict'] = create_node_boot_dict(lab['name'])
install_helper.open_vlm_console_thread(to_host)
LOG.tc_step("Delete the host {}".format(to_host))
cli.system("host-bulk-export")
cli.system("host-delete {}".format(to_host))
assert len(system_helper.get_controllers()) > 1, "Host deletion failed"
cli.system("host-bulk-add hosts.xml")
system_helper.wait_for_host_values(to_host,
timeout=6000,
availability=HostAvailState.ONLINE)
wait_for_disks(to_host)
LOG.tc_step("Apply the storage-profile {} onto host:{}".format(
prof_name, to_host))
cli.system("host-apply-storprofile {} {}".format(to_host, prof_name))
# Need to re-provision everything on node through lab_setup (except storage)
LOG.tc_step("Reprovision the host as necessary")
files = [
'interfaces', 'cinder_device', 'vswitch_cpus', 'shared_cpus',
'extend_cgts_vg', 'addresses'
]
con_ssh = ControllerClient.get_active_controller()
delete_lab_setup_files(con_ssh, to_host, files)
rc, msg = install_helper.run_lab_setup()
assert rc == 0, msg
LOG.tc_step("Unlock to host")
host_helper.unlock_host(to_host)
to_host_backing = host_helper.get_host_instance_backing(to_host)
LOG.info("To host backing was {} and is now {}".format(
new_to_backing, to_host_backing))
assert to_host_backing == from_backing, "Host backing was not changed on storage profile application"
def test_set_hosts_storage_backing_equal(instance_backing, number_of_hosts):
"""
Modify hosts storage backing if needed so that system has exact number
of hosts in given instance backing
Args:
instance_backing:
number_of_hosts:
Test Steps:
- Calculate the hosts to be configured based on test params
- Configure hosts to meet given criteria
- Check number of hosts in given instance backing is as specified
"""
host_num_mapping = {'zero': 0, 'one': 1, 'two': 2}
number_of_hosts = host_num_mapping[number_of_hosts]
LOG.tc_step("Calculate the hosts to be configured based on test params")
candidate_hosts = get_candidate_hosts(number_of_hosts=number_of_hosts)
hosts_with_backing = \
host_helper.get_hosts_in_storage_backing(instance_backing)
if len(hosts_with_backing) == number_of_hosts:
LOG.info("Already have {} hosts in {} backing. Do "
"nothing".format(number_of_hosts, instance_backing))
return
elif len(hosts_with_backing) < number_of_hosts:
backing_to_config = instance_backing
number_to_config = number_of_hosts - len(hosts_with_backing)
hosts_pool = list(set(candidate_hosts) - set(hosts_with_backing))
else:
backing_to_config = 'remote' if 'image' in instance_backing else \
'local_image'
number_to_config = len(hosts_with_backing) - number_of_hosts
hosts_pool = hosts_with_backing
LOG.tc_step("Delete vms if any to prepare for system configuration "
"change with best effort")
vm_helper.delete_vms(fail_ok=True)
hosts_to_config = hosts_pool[0:number_to_config]
LOG.tc_step("Configure following hosts to {} backing: "
"{}".format(hosts_to_config, backing_to_config))
for host in hosts_to_config:
host_helper.set_host_storage_backing(host=host,
inst_backing=backing_to_config,
unlock=False,
wait_for_configured=False)
HostsToRecover.add(host)
host_helper.unlock_hosts(hosts_to_config,
check_hypervisor_up=True,
fail_ok=False)
LOG.tc_step("Waiting for hosts in {} aggregate".format(backing_to_config))
for host in hosts_to_config:
host_helper.wait_for_host_in_instance_backing(
host, storage_backing=backing_to_config)
LOG.tc_step("Check number of {} hosts is {}".format(
instance_backing, number_of_hosts))
assert number_of_hosts == \
len(host_helper.get_hosts_in_storage_backing(instance_backing)), \
"Number of {} hosts is not {} after " \
"configuration".format(instance_backing, number_of_hosts)