def add_1g_and_4k_pages(self, request, config_host_class,
skip_for_one_proc, add_cgcsauto_zone,
add_admin_role_module):
storage_backing, candidate_hosts = host_helper.get_storage_backing_with_max_hosts(
)
if len(candidate_hosts) < 2:
skip("Less than two up hosts have same storage backing")
vm_helper.delete_vms()
LOG.fixture_step(
"Check mempage configs for hypervisors and select host to use or "
"configure")
hosts_selected, hosts_to_configure = get_hosts_to_configure(
candidate_hosts)
if set(hosts_to_configure) != {None}:
def _modify(host):
is_1g = True if hosts_selected.index(host) == 0 else False
proc1_kwargs = \
{'gib_1g': VM_MEM_GIB, 'gib_4k_range': (None, VM_MEM_GIB)} if is_1g else \
{'gib_1g': 0, 'gib_4k_range': (10, None)}
kwargs = {
'gib_1g': 0,
'gib_4k_range': (None, VM_MEM_GIB)
}, proc1_kwargs
actual_mems = host_helper._get_actual_mems(host=host)
LOG.fixture_step(
"Modify {} proc0 to have 0 of 1G pages and <{}GiB of 4K pages"
.format(host, VM_MEM_GIB))
host_helper.modify_host_memory(host,
proc=0,
actual_mems=actual_mems,
**kwargs[0])
LOG.fixture_step(
"Modify {} proc1 to have >={}GiB of {} pages".format(
host, VM_MEM_GIB, '1G' if is_1g else '4k'))
host_helper.modify_host_memory(host,
proc=1,
actual_mems=actual_mems,
**kwargs[1])
for host_to_config in hosts_to_configure:
if host_to_config:
config_host_class(host=host_to_config, modify_func=_modify)
LOG.fixture_step(
"Check mem pages for {} are modified and updated successfully"
.format(host_to_config))
host_helper.wait_for_memory_update(host=host_to_config)
LOG.fixture_step(
"Check host memories for {} after mem config completed".format(
hosts_selected))
_, hosts_unconfigured = get_hosts_to_configure(hosts_selected)
assert not hosts_unconfigured[0], \
"Failed to configure {}. Expt: proc0:1g<{},4k<{}gib; proc1:1g>={},4k<{}gib".format(
hosts_unconfigured[0], VM_MEM_GIB, VM_MEM_GIB, VM_MEM_GIB, VM_MEM_GIB)
assert not hosts_unconfigured[1], \
"Failed to configure {}. Expt: proc0:1g<{},4k<{}gib; proc1:1g<{},4k>={}gib".format(
hosts_unconfigured[1], VM_MEM_GIB, VM_MEM_GIB, VM_MEM_GIB, VM_MEM_GIB)
LOG.fixture_step('(class) Add hosts to cgcsauto aggregate: {}'.format(
hosts_selected))
nova_helper.add_hosts_to_aggregate(aggregate='cgcsauto',
hosts=hosts_selected)
def remove_host_from_zone():
LOG.fixture_step(
'(class) Remove hosts from cgcsauto aggregate: {}'.format(
hosts_selected))
nova_helper.remove_hosts_from_aggregate(aggregate='cgcsauto',
check_first=False)
request.addfinalizer(remove_host_from_zone)
return hosts_selected, storage_backing
def test_dvr_vms_network_connection(vms_num, srv_grp_policy, server_groups,
router_info):
"""
Test vms East West connection by pinging vms' data network from vm
Args:
vms_num (int): number of vms to boot
srv_grp_policy (str): affinity to boot vms on same host,
anti-affinity to boot vms on different hosts
server_groups: test fixture to return affinity and anti-affinity
server groups
router_info (str): id of tenant router
Skip Conditions:
- Only one nova host on the system
Setups:
- Enable DVR (module)
Test Steps
- Update router to distributed if not already done
- Boot given number of vms with specific server group policy to
schedule vms on same or different host(s)
- Ping vms' over data and management networks from one vm to test NS
and EW traffic
Teardown:
- Delete vms
- Revert router to
"""
# Increase instance quota count if needed
current_vms = len(vm_helper.get_vms(strict=False))
quota_needed = current_vms + vms_num
vm_helper.ensure_vms_quotas(quota_needed)
if srv_grp_policy == 'anti-affinity' and len(
host_helper.get_up_hypervisors()) == 1:
skip("Only one nova host on the system.")
LOG.tc_step("Update router to distributed if not already done")
router_id = router_info
is_dvr = network_helper.get_router_values(router_id,
fields='distributed',
auth_info=Tenant.get('admin'))[0]
if not is_dvr:
network_helper.set_router_mode(router_id, distributed=True)
LOG.tc_step("Boot {} vms with server group policy {}".format(
vms_num, srv_grp_policy))
affinity_grp, anti_affinity_grp = server_groups(soft=True)
srv_grp_id = affinity_grp if srv_grp_policy == 'affinity' else \
anti_affinity_grp
vms = []
tenant_net_id = network_helper.get_tenant_net_id()
mgmt_net_id = network_helper.get_mgmt_net_id()
internal_net_id = network_helper.get_internal_net_id()
internal_vif = {'net-id': internal_net_id}
if system_helper.is_avs():
internal_vif['vif-model'] = 'avp'
nics = [{'net-id': mgmt_net_id}, {'net-id': tenant_net_id}, internal_vif]
for i in range(vms_num):
vol = cinder_helper.create_volume()[1]
ResourceCleanup.add(resource_type='volume', resource_id=vol)
vm_id = \
vm_helper.boot_vm('dvr_ew_traffic', source='volume', source_id=vol,
nics=nics, cleanup='function',
hint={'group': srv_grp_id})[1]
vms.append(vm_id)
LOG.tc_step("Wait for vm {} pingable from NatBox".format(vm_id))
vm_helper.wait_for_vm_pingable_from_natbox(vm_id, fail_ok=False)
from_vm = vms[0]
LOG.tc_step("Ping vms over management and data networks from vm {}, and "
"verify ping successful.".format(from_vm))
vm_helper.ping_vms_from_vm(to_vms=vms,
from_vm=from_vm,
fail_ok=False,
net_types=['data', 'mgmt', 'internal'])
def check_volumes_spaces(con_ssh):
from keywords import cinder_helper
LOG.info('Checking cinder volumes and space usage')
usage_threshold = 0.70
free_space, total_space, unit = cinder_helper.get_lvm_usage(con_ssh)
if total_space and free_space < usage_threshold * total_space:
if total_space:
LOG.info(
'cinder LVM over-used: free:{}, total:{}, ration:{}%'.format(
free_space, total_space, free_space / total_space * 100))
LOG.info('Deleting known LVM alarms')
expected_reason = r'Cinder LVM .* Usage threshold exceeded; threshold: (\d+(\.\d+)?)%, actual: (\d+(\.\d+)?)%'
expected_entity = 'host=controller'
value_titles = ('UUID', 'Alarm ID', 'Reason Text', 'Entity ID')
lvm_pool_usage = system_helper.get_alarms(fields=value_titles,
con_ssh=con_ssh)
if not lvm_pool_usage:
LOG.warn('Cinder LVM pool is used up to 75%, but no alarm for it')
else:
if len(lvm_pool_usage) > 1:
LOG.warn(
'More than one alarm existing for Cinder LVM over-usage')
elif len(lvm_pool_usage) < 1:
LOG.warn('No LVM cinder over-used alarms, got:{}'.format(
lvm_pool_usage))
for lvm_alarm in lvm_pool_usage:
alarm_uuid, alarm_id, reason_text, entity_id = lvm_alarm.split(
'::::')
if re.match(expected_reason, reason_text) and re.search(
expected_entity, entity_id):
LOG.info('Expected alarm:{}, reason:{}'.format(
alarm_uuid, reason_text))
LOG.info('Deleting it')
system_helper.delete_alarms(alarms=alarm_uuid)
def delete_resources(resource_names=None,
select_all=None,
resource_types='pod',
namespace=None,
recursive=None,
labels=None,
con_ssh=None,
fail_ok=False,
post_check=True,
check_both_controllers=True):
"""
Delete pods via kubectl delete
Args:
resource_names (None|str|list|tuple):
select_all (None|bool):
resource_types (str|list|tuple):
namespace (None|str):
recursive (bool):
labels (None|dict):
con_ssh:
fail_ok:
post_check (bool): Whether to check if resources are gone after deletion
check_both_controllers (bool):
Returns (tuple):
(0, None) # pods successfully deleted
(1, <std_err>)
(2, <undeleted_resources>(list of dict)) # pod(s) still exist in
kubectl after deletion
(3, <undeleted_resources_on_other_controller>(list of dict)) #
pod(s) still exist on the other controller
"""
arg_dict = {
'--all': select_all,
'-l': labels,
'--recursive': recursive,
}
arg_str = common.parse_args(args_dict=arg_dict, vals_sep=',')
if resource_types:
if isinstance(resource_types, str):
resource_types = [resource_types]
arg_str = '{} {}'.format(','.join(resource_types), arg_str).strip()
if resource_names:
if isinstance(resource_names, str):
resource_names = [resource_names]
arg_str = '{} {}'.format(arg_str, ' '.join(resource_names))
if not con_ssh:
con_ssh = ControllerClient.get_active_controller()
code, output = exec_kube_cmd(sub_cmd='delete',
args=arg_str,
con_ssh=con_ssh,
fail_ok=fail_ok)
if code > 0:
return 1, output
if post_check:
def __wait_for_resources_gone(ssh_client):
final_remaining = []
if resource_types:
for resource_type in resource_types:
res, remaining_res = wait_for_resources_gone(
resource_names=resource_names,
resource_type=resource_type,
namespace=namespace,
con_ssh=ssh_client,
fail_ok=fail_ok)
if not res:
final_remaining += remaining_res
else:
res, final_remaining = wait_for_resources_gone(
resource_names=resource_names,
namespace=namespace,
con_ssh=ssh_client,
fail_ok=fail_ok)
return final_remaining
LOG.info("Check pod is not running on current host")
remaining = __wait_for_resources_gone(con_ssh)
if remaining:
return 2, remaining
if check_both_controllers and not system_helper.is_aio_simplex(
con_ssh=con_ssh):
LOG.info("Check pod is running on the other controller as well")
con_name = 'controller-1' if \
con_ssh.get_hostname() == 'controller-0' else 'controller-0'
from keywords import host_helper
with host_helper.ssh_to_host(hostname=con_name,
con_ssh=con_ssh) as other_con:
remaining = __wait_for_resources_gone(other_con)
if remaining:
return 3, remaining
LOG.info("{} are successfully removed.".format(resource_names))
return 0, None
def test_ping_between_two_vms(guest_os, vm1_vifs, vm2_vifs):
"""
Ping between two vms with given vif models
Test Steps:
- Create a favor with dedicated cpu policy and proper root disk size
- Create a volume from guest image under test with proper size
- Boot two vms with given vif models from above volume and flavor
- Ping VMs from NatBox and between two vms
Test Teardown:
- Delete vms, volumes, flavor, glance image created
"""
if guest_os == 'default':
guest_os = GuestImages.DEFAULT['guest']
reuse = False if 'e1000' in vm1_vifs or 'e1000' in vm2_vifs else True
cleanup = 'function' if not reuse or 'ubuntu' in guest_os else None
image_id = glance_helper.get_guest_image(guest_os, cleanup=cleanup,
use_existing=reuse)
LOG.tc_step("Create a favor dedicated cpu policy")
flavor_id = nova_helper.create_flavor(name='dedicated', guest_os=guest_os,
cleanup='function')[1]
nova_helper.set_flavor(flavor_id, **{FlavorSpec.CPU_POLICY: 'dedicated'})
mgmt_net_id = network_helper.get_mgmt_net_id()
tenant_net_id = network_helper.get_tenant_net_id()
internal_net_id = network_helper.get_internal_net_id()
net_ids = (mgmt_net_id, tenant_net_id, internal_net_id)
vms = []
for vifs_for_vm in (vm1_vifs, vm2_vifs):
# compose vm nics
nics = _compose_nics(vifs_for_vm, net_ids=net_ids, image_id=image_id,
guest_os=guest_os)
net_types = ['mgmt', 'data', 'internal'][:len(nics)]
LOG.tc_step("Create a volume from {} image".format(guest_os))
vol_id = cinder_helper.create_volume(name='vol-{}'.format(guest_os),
source_id=image_id,
guest_image=guest_os,
cleanup='function')[1]
LOG.tc_step(
"Boot a {} vm with {} vifs from above flavor and volume".format(
guest_os, vifs_for_vm))
vm_id = vm_helper.boot_vm('{}_vifs'.format(guest_os), flavor=flavor_id,
cleanup='function',
source='volume', source_id=vol_id, nics=nics,
guest_os=guest_os)[1]
LOG.tc_step("Ping VM {} from NatBox(external network)".format(vm_id))
vm_helper.wait_for_vm_pingable_from_natbox(vm_id, fail_ok=False)
vms.append(vm_id)
LOG.tc_step(
"Ping between two vms over management, data, and internal networks")
vm_helper.ping_vms_from_vm(to_vms=vms[0], from_vm=vms[1],
net_types=net_types)
vm_helper.ping_vms_from_vm(to_vms=vms[1], from_vm=vms[0],
net_types=net_types)
def get_openstack_pods(field='Name',
namespace='openstack',
application=None,
component=None,
pod_names=None,
extra_labels=None,
field_selectors=None,
exclude_label=False,
fail_ok=False,
con_ssh=None,
strict=True,
exclude=False,
**kwargs):
"""
Get openstack pods via kubectl get pods
Note that pod labels can be found via kubectl get pods -n <namespace>
--show-labels
Args:
field (str|list|tuple):
namespace:
application (str|None): label: application
component (str|None): label: component
pod_names
extra_labels (str|None):
field_selectors (str|list|tuple|dict|None):
exclude_label
fail_ok:
con_ssh:
exclude:
strict:
**kwargs:
Returns (list):
"""
if pod_names:
labels = None
else:
connector = '!=' if exclude_label else '='
labels = []
if application:
labels.append('application{}{}'.format(connector, application))
if component:
labels.append('component{}{}'.format(connector, component))
if extra_labels:
labels.append(extra_labels)
labels = ','.join(labels)
pods = get_pods(pod_names=pod_names,
field=field,
namespace=namespace,
labels=labels,
fail_ok=fail_ok,
field_selectors=field_selectors,
strict=strict,
exclude=exclude,
con_ssh=con_ssh,
**kwargs)
if not pods:
msg = "No pods found for namespace - {} with selectors: {}".format(
namespace, labels)
LOG.info(msg)
if not fail_ok:
raise exceptions.KubeError(msg)
return pods
def apply_pod(file_path,
pod_name,
namespace=None,
recursive=None,
select_all=None,
labels=None,
con_ssh=None,
fail_ok=False,
check_both_controllers=True):
"""
Apply a pod from given file via kubectl apply
Args:
file_path (str):
pod_name (str):
namespace (None|str):
recursive (None|bool):
select_all (None|bool):
labels (dict|str|list|tuple|None): key value pairs
con_ssh:
fail_ok:
check_both_controllers (bool):
Returns (tuple):
(0, <pod_info>(dict))
(1, <std_err>)
(2, <pod_info>) # pod is not running after apply
(3, <pod_info>) # pod if not running on the other controller after
apply
"""
arg_dict = {
'--all': select_all,
'-l': labels,
'--recursive': recursive,
}
arg_str = common.parse_args(args_dict=arg_dict, vals_sep=',')
arg_str += ' -f {}'.format(file_path)
if not con_ssh:
con_ssh = ControllerClient.get_active_controller()
code, output = exec_kube_cmd(sub_cmd='apply',
args=arg_str,
con_ssh=con_ssh,
fail_ok=fail_ok)
if code > 0:
return 1, output
LOG.info("Check pod is running on current host")
res = wait_for_pods_status(pod_names=pod_name,
namespace=namespace,
status=PodStatus.RUNNING,
con_ssh=con_ssh,
fail_ok=fail_ok)
if not res:
return 2, "Pod {} is not running after apply on active " \
"controller".format(pod_name)
if check_both_controllers and not system_helper.is_aio_simplex(
con_ssh=con_ssh):
LOG.info("Check pod is running on the other controller as well")
con_name = 'controller-1' if con_ssh.get_hostname() == 'controller-0' \
else 'controller-0'
from keywords import host_helper
with host_helper.ssh_to_host(hostname=con_name,
con_ssh=con_ssh) as other_con:
res, pods_info = wait_for_pods_status(pod_names=pod_name,
namespace=namespace,
con_ssh=other_con,
fail_ok=fail_ok)
if not res:
return 3, "Pod {} is not running after apply on standby " \
"controller".format(pod_name)
LOG.info("{} pod is successfully applied and running".format(pod_name))
return 0, pod_name
def test_vm_numa_node_settings(vcpus, numa_nodes, numa_node0, numa_node1,
no_simplex, check_numa_num):
"""
Test NUMA nodes settings in flavor extra specs are successfully applied to a vm
Args:
vcpus (int): Number of vcpus to set when creating flavor
numa_nodes (int): Number of NUMA nodes to set in flavor extra specs
numa_node0 (int): node.0 value in flavor extra specs
numa_node1 (int): node.1 value in flavor extra specs
Test Steps:
- Create a flavor with given number of vcpus specified
- Add numa_nodes related extra specs
- Boot a vm with flavor
- Run vm-topology
- Verify vcpus, numa nodes, cpulist for specific vm reflects the settings in flavor
- Ensure that all virtual NICs are associated with guest virtual numa node 0 (tests TC5069)
Teardown:
- Delete created vm, volume, and flavor
"""
if check_numa_num < numa_nodes:
skip("Number of processors - {} is less than required numa nodes - {}".
format(check_numa_num, numa_nodes))
LOG.tc_step("Create flavor with {} vcpus".format(vcpus))
flavor = nova_helper.create_flavor('numa_vm', vcpus=vcpus)[1]
ResourceCleanup.add('flavor', flavor, scope='function')
extra_specs = {
FlavorSpec.CPU_POLICY: 'dedicated',
FlavorSpec.NUMA_NODES: numa_nodes,
FlavorSpec.NUMA_0: numa_node0
}
if numa_node1 is not None:
extra_specs[FlavorSpec.NUMA_1] = numa_node1
LOG.tc_step("Set following extra specs for flavor {}: {}.".format(
extra_specs, flavor))
nova_helper.set_flavor(flavor, **extra_specs)
LOG.tc_step("Boot vm with flavor {}.".format(flavor))
vm_id = vm_helper.boot_vm(flavor=flavor, cleanup='function')[1]
LOG.tc_step("Verify cpu info for vm {} via vm-topology.".format(vm_id))
nova_tab, libvirt_tab = system_helper.get_vm_topology_tables(
'servers', 'libvirt')
# Filter out the line for vm under test
nova_tab = table_parser.filter_table(nova_tab, ID=vm_id)
libvirt_tab = table_parser.filter_table(libvirt_tab, uuid=vm_id)
instance_topology = table_parser.get_column(nova_tab,
'instance_topology')[0]
cpulist = table_parser.get_column(libvirt_tab, 'cpulist')[0]
if '-' in cpulist:
cpulist = cpulist.split(sep='-')
cpulist_len = int(cpulist[1]) - int(cpulist[0]) + 1
else:
cpulist_len = len(cpulist.split(sep=','))
vcpus_libvirt = int(table_parser.get_column(libvirt_tab, 'vcpus')[0])
nodelist = table_parser.get_column(libvirt_tab, 'nodelist')[0]
if isinstance(instance_topology, str):
instance_topology = [instance_topology]
# Each numa node will have an entry for given instance, thus number of entries should be the same as number of
# numa nodes for the vm
assert numa_nodes == len(instance_topology), \
"Number of numa node entries for vm {} is different than number of NUMA nodes set in flavor".format(vm_id)
expected_node_vals = [
int(val) for val in [numa_node0, numa_node1] if val is not None
]
actual_node_vals = []
for actual_node_info in instance_topology:
actual_node_val = int(
re.findall(InstanceTopology.NODE, actual_node_info)[0])
actual_node_vals.append(actual_node_val)
assert expected_node_vals == actual_node_vals, \
"Individual NUMA node value(s) for vm {} is different than numa_node setting in flavor".format(vm_id)
assert vcpus == vcpus_libvirt, \
"Number of vcpus for vm {} in libvirt view is different than what's set in flavor.".format(vm_id)
assert vcpus == cpulist_len, \
"Number of entries in cpulist for vm {} in libvirt view is different than number of vcpus set in flavor".format(
vm_id)
if '-' in nodelist:
nodelist = nodelist.split(sep='-')
nodelist_len = int(nodelist[1]) - int(nodelist[0]) + 1
else:
nodelist_len = 1 if nodelist else 0
assert numa_nodes == nodelist_len, \
"nodelist for vm {} in libvirt view does not match number of numa nodes set in flavor".format(vm_id)
if system_helper.is_avs():
# TC5069
LOG.tc_step(
"Check via vshell that all vNICs are associated with the host NUMA node that guest numa0 maps to"
)
host = vm_helper.get_vm_host(vm_id)
actual_ports = network_helper.get_ports(vm_id)
with host_helper.ssh_to_host(host) as compute_ssh:
for port_id in actual_ports:
ports_tab = table_parser.table(
compute_ssh.exec_cmd("vshell port-show {}".format(port_id),
fail_ok=False)[1])
socket_id = int(
table_parser.get_value_two_col_table(ports_tab,
field='socket-id'))
assert socket_id == numa_node0, "NIC is not associated with numa-node0"
def teardown():
LOG.fixture_step('Back to Hypervisors page')
hypervisors_pg.go_to_target_page()
def test_schedule_vm_mempage_config(self, flavor_memconf, mem_page_size,
reset_host_app_mems):
"""
Test memory used by vm is taken from the expected memory pool and the vm was scheduled
on the correct host/processor
Args:
flavor_memconf (tuple): flavor id of a flavor with ram set to 10G, hosts,
storage_backing
mem_page_size (str): mem page size setting in flavor
Setup:
- Create host aggregate
- Add two hypervisors to the host aggregate
- Host-0 configuration:
- Processor-0:
- Insufficient 1g pages to boot vm that requires 10g
- Insufficient 4k pages to boot vm that requires 10g
- Processor-1:
- Sufficient 1g pages to boot vm that requires 10g
- Insufficient 4k pages to boot vm that requires 10g
- Host-1 configuration:
- Processor-0:
- Insufficient 1g pages to boot vm that requires 10g
- Insufficient 4k pages to boot vm that requires 10g
- Processor-1:
- Insufficient 1g pages to boot vm that requires 10g
- Sufficient 4k pages to boot vm that requires 10g
- Configure a compute to have 10 1G hugepages (module)
- Create a flavor with 10G RAM (module)
- Create a volume with default values (module)
Test Steps:
- Set memory page size flavor spec to given value
- Boot a vm with above flavor and a basic volume
- Calculate the available/used memory change on the vm host
- Verify the memory is taken from 1G hugepage memory pool
- Verify the vm was booted on a supporting host
Teardown:
- Delete created vm
- Delete created volume and flavor (module)
- Re-Configure the compute to have 0 hugepages (module)
- Revert host mem pages back to original
"""
skip_4k_for_ovs(mem_page_size)
flavor_id, hosts_configured, storage_backing = flavor_memconf
LOG.tc_step("Set memory page size extra spec in flavor")
nova_helper.set_flavor(
flavor_id, **{
FlavorSpec.CPU_POLICY: 'dedicated',
FlavorSpec.MEM_PAGE_SIZE: mem_page_size
})
host_helper.wait_for_hypervisors_up(hosts_configured)
prev_computes_mems = {}
for host in hosts_configured:
prev_computes_mems[host] = host_helper.get_host_memories(
host=host, headers=MEMPAGE_HEADERS)
LOG.tc_step(
"Boot a vm with mem page size spec - {}".format(mem_page_size))
host_1g, host_4k = hosts_configured
code, vm_id, msg = vm_helper.boot_vm('mempool_configured',
flavor_id,
fail_ok=True,
avail_zone='cgcsauto',
cleanup='function')
assert 0 == code, "VM is not successfully booted."
instance_name, vm_host = vm_helper.get_vm_values(
vm_id, fields=[":instance_name", ":host"], strict=False)
vm_node = vm_helper.get_vm_numa_nodes_via_ps(
vm_id=vm_id, instance_name=instance_name, host=vm_host)
if mem_page_size == '1048576':
assert host_1g == vm_host, "VM is not created on the configured host {}".format(
hosts_configured[0])
assert vm_node == [
1
], "VM (huge) did not boot on the correct processor"
elif mem_page_size == 'small':
assert host_4k == vm_host, "VM is not created on the configured host {}".format(
hosts_configured[1])
assert vm_node == [
1
], "VM (small) did not boot on the correct processor"
else:
assert vm_host in hosts_configured
LOG.tc_step("Calculate memory change on vm host - {}".format(vm_host))
check_mempage_change(vm_id,
vm_host,
prev_host_mems=prev_computes_mems[vm_host],
mempage_size=mem_page_size,
mem_gib=VM_MEM_GIB,
numa_node=vm_node[0])
LOG.tc_step("Ensure vm is pingable from NatBox")
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
def wait_for_running_pods_ready(pod_names=None,
namespace=None,
all_namespaces=False,
labels=None,
timeout=300,
fail_ok=False,
con_ssh=None,
exclude=False,
strict=False,
**kwargs):
"""
Wait for Running pods to be Ready, such as 1/1, 3/3
Args:
pod_names:
namespace:
all_namespaces:
labels:
timeout:
fail_ok:
con_ssh:
exclude:
strict:
**kwargs:
Returns (bool):
"""
unready_pods = get_unready_running_pods(namespace=namespace,
all_namespaces=all_namespaces,
pod_names=pod_names,
labels=labels,
exclude=exclude,
strict=strict,
con_ssh=con_ssh,
**kwargs)
if not unready_pods:
return True
end_time = time.time() + timeout
while time.time() < end_time:
pods_info = get_pods(field=('NAME', 'READY'),
namespace=namespace,
all_namespaces=all_namespaces,
pod_names=unready_pods,
con_ssh=con_ssh)
for pod_info in pods_info:
pod_name, pod_ready = pod_info
ready_count, total_count = pod_ready.split('/')
if ready_count == total_count:
unready_pods.remove(pod_name)
if not unready_pods:
return True
time.sleep(20)
msg = "Some pods are not ready within {}s: {}".format(
timeout, unready_pods)
LOG.warning(msg)
if fail_ok:
return False
raise exceptions.KubeError(msg)
def test_boot_vm_mem_page_size(self, flavor_memconf, flavor_mem_page_size,
image_mempage, image_mem_page_size,
reset_host_app_mems):
"""
Test boot vm with various memory page size setting in flavor and image.
Args:
flavor_memconf (tuple): flavor id of a flavor with ram set to 10G, hosts configured
and storage_backing
flavor_mem_page_size (str): memory page size extra spec value to set in flavor
image_mempage (str): image id for tis image
image_mem_page_size (str): memory page metadata value to set in image
Setup:
- Create a flavor with 10G RAM (module)
- Get image id of tis image (module)
Test Steps:
- Set/Unset flavor memory page size extra spec with given value (unset if None is given)
- Set/Unset image memory page size metadata with given value (unset if None if given)
- Attempt to boot a vm with above flavor and image
- Verify boot result based on the mem page size values in the flavor and image
Teardown:
- Delete vm if booted
- Delete created flavor (module)
"""
skip_4k_for_ovs(image_mem_page_size)
flavor_id, hosts, storage_backing = flavor_memconf
if image_mem_page_size is None:
glance_helper.unset_image(image_mempage,
properties=ImageMetadata.MEM_PAGE_SIZE)
expt_code = 0
else:
glance_helper.set_image(
image=image_mempage,
properties={ImageMetadata.MEM_PAGE_SIZE: image_mem_page_size})
if flavor_mem_page_size is None:
expt_code = 4
elif flavor_mem_page_size.lower() in ['any', 'large']:
expt_code = 0
else:
expt_code = 0 if flavor_mem_page_size.lower(
) == image_mem_page_size.lower() else 4
LOG.tc_step(
"Attempt to boot a vm with flavor_mem_page_size: {}, and image_mem_page_size: "
"{}. And check return "
"code is {}.".format(flavor_mem_page_size, image_mem_page_size,
expt_code))
actual_code, vm_id, msg = vm_helper.boot_vm(name='mem_page_size',
flavor=flavor_id,
source='image',
source_id=image_mempage,
fail_ok=True,
avail_zone='cgcsauto',
cleanup='function')
assert expt_code == actual_code, "Expect boot vm to return {}; Actual result: {} " \
"with msg: {}".format(expt_code, actual_code, msg)
if expt_code != 0:
assert re.search(
NovaCLIOutput.VM_BOOT_REJECT_MEM_PAGE_SIZE_FORBIDDEN, msg)
else:
assert vm_helper.get_vm_host(vm_id) in hosts, \
"VM is not booted on hosts in cgcsauto zone"
LOG.tc_step("Ensure VM is pingable from NatBox")
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
def image_mempage(self):
LOG.fixture_step("(class) Create a glance image for mempage testcases")
image_id = glance_helper.create_image(name='mempage',
cleanup='class')[1]
return image_id
def remove_host_from_zone():
LOG.fixture_step(
'(class) Remove hosts from cgcsauto aggregate: {}'.format(
hosts_selected))
nova_helper.remove_hosts_from_aggregate(aggregate='cgcsauto',
check_first=False)
def _test_vtpm(vm_operation, extra_specs):
global g_vms, g_reusable
LOG.tc_step(
'Verify vTPM is functioning on VMs right after they are: {}'.format(
vm_operation))
LOG.info('Will perform action:{}'.format(vm_operation))
number_hypervisor = len(host_helper.get_up_hypervisors())
if number_hypervisor < 2 and 'migrate' in vm_operation:
skip('Not enough hypervisors available for testing')
vm_types = [
vm_type for vm_type in extra_specs.split(',') if vm_type in g_vms
]
for vm_type in vm_types:
reuse = reuse_existing_vms(vm_operation, extra_specs)
vm_id = get_vm_id(vm_type, reuse=reuse)
LOG.info('-check vTPM supports on hosting node for VM:' + vm_id +
', vm-type:' + vm_type)
verify_vtpm_on_host(vm_id, host=None)
LOG.info('-OK, passed checking on hosting node for VM:' + vm_id +
', vm-type:' + vm_type)
if vm_operation == 'create':
with vm_helper.ssh_to_vm_from_natbox(vm_id) as ssh_to_vm:
LOG.info(
'Create all types of contents: volatile, non_volatile and persistent'
)
create_values(ssh_to_vm, vm_type)
values = g_vms[vm_type]['values']
LOG.info('Running test on VM:{}, type:{}, values:{}'.format(
vm_id, vm_type, values))
# fail_ok = (vm_operation == 'resize_to_non_vtpm')
fail_ok = True
perform_vm_operation(vm_type,
vm_id,
op=vm_operation,
extra_specs=extra_specs)
ping_timeout = VMTimeout.DHCP_RETRY if vm_operation in (
'reboot_host', 'evacuate') else VMTimeout.PING_VM
vm_helper.wait_for_vm_pingable_from_natbox(vm_id, timeout=ping_timeout)
with vm_helper.ssh_to_vm_from_natbox(vm_id,
timeout=(int(VMTimeout.SSH_LOGIN *
1.3))) as ssh_to_vm:
LOG.info(
'After VM operation:{}, check all types of contents'.format(
vm_operation))
passed = []
if 'persistent' in values:
if check_persistent_values(ssh_to_vm,
values['persistent'],
expecting=vm_op_policy(
vm_type, vm_operation,
'persistent'),
fail_ok=fail_ok)[0]:
passed.append('persistent')
if 'non_volatile' in values:
if check_nv_values(ssh_to_vm,
values['non_volatile'],
expecting=vm_op_policy(
vm_type, vm_operation, 'non_volatile'),
fail_ok=fail_ok)[0]:
passed.append('non_volatile')
if 'transient' in values:
if check_transient_values(ssh_to_vm,
handles=values['transient'],
expecting=vm_op_policy(
vm_type, vm_operation,
'transient'),
fail_ok=fail_ok)[0]:
passed.append('transient')
if len(passed) < 3:
all_checks = ['persistent', 'non_volatile', 'transient']
message = 'Failed in verify {} contents'.format(
', '.join(set(all_checks) - set(passed)))
assert False, message
if 'reboot_host' == vm_operation \
or 'evacuate' in vm_operation\
or 'resize_to_non_vtpm' in vm_operation\
or 'non_autorc' in extra_specs \
or 'non_vtpm' in extra_specs:
g_reusable = False
else:
g_reusable = True
def delete_handle(ssh_con, handle):
LOG.info('Deleting handle:' + str(handle))
cli = 'flushcontext -ha ' + str(handle)
run_cmd(ssh_con, cli, output_handle=False)
LOG.info('-OK, successfully deleted handle:' + str(handle))
def _test_token_expiry(service_params):
"""
Verify that token expiry time can be changed using service parameters
Test Steps:
- Verify that the token expiration is set by default.
- Set token expiry length to set values
- Verify that the length is rejected if it is not between 1 and 4 hours
- Create a token and ensure it expires after the expected expiry time
"""
expire_times = [6000, 7200, 3000, 15500, 3600]
service, section, name = service_params
LOG.tc_step("Verify that token_expiration parameter is defined")
default_exp_time = system_helper.get_service_parameter_values(
field='value', service=service, section=section, name=name)[0]
assert int(
default_exp_time
) == 3600, "Default token_expiration value not 3600, actually {}".format(
default_exp_time)
LOG.tc_step("Verify that tokens now expire after expected time")
token_expire_time = html_helper.get_user_token(field='expires')
expt_time = time.time() + int(default_exp_time)
expt_datetime = datetime.datetime.utcfromtimestamp(expt_time).isoformat()
time_diff = common.get_timedelta_for_isotimes(
expt_datetime, token_expire_time).total_seconds()
LOG.info(
"Expect expiry time to be {}. Token expiry time is {}. Difference is {}."
.format(token_expire_time, expt_datetime, time_diff))
assert -150 < time_diff < 150, "Token expiry time is {}, but token expired {} seconds after expected time.".\
format(expt_time, time_diff)
for expire_time in expire_times:
if expire_time > 14400 or expire_time < 3600:
res, out = system_helper.modify_service_parameter(service,
section,
name,
str(expire_time),
fail_ok=True)
assert 1 == res, "Modifying the expiry time to {} was not rejected".format(
expire_time)
assert "must be between 3600 and 14400 seconds" in out, "Unexpected rejection string"
value = system_helper.get_service_parameter_values(service=service,
section=section,
name=name)
assert expire_time != value, "Expiry time was changed to rejected value"
else:
LOG.tc_step("Set token expiration service parameter to {}".format(
expire_time))
system_helper.modify_service_parameter(service,
section,
name,
str(expire_time),
apply=True)
LOG.tc_step("Verify that tokens now expire after expected time")
token_expire_time = html_helper.get_user_token(field='expires')
expt_time = time.time() + expire_time
expt_datetime = datetime.datetime.utcfromtimestamp(
expt_time).isoformat()
time_diff = common.get_timedelta_for_isotimes(
expt_datetime, token_expire_time).total_seconds()
LOG.info(
"Expect expiry time to be {}. Token expiry time is {}. Difference is {}."
.format(token_expire_time, expt_datetime, time_diff))
assert -150 < time_diff < 150, "Token is not set to expire after {} seconds".format(
expire_time)
def check_nv_values(ssh_con, handle, size=32, expecting=True, fail_ok=False):
LOG.info('check if nv content exists')
output = ''
try:
rc, output = run_cmd(ssh_con,
'nvread -ha {} -sz {}'.format(handle, size),
output_handle=False,
fail_ok=fail_ok)
except Exception as e:
LOG.info('Caught exception:{} when run cmd: "nvread -ha..."'.format(e))
rc = 1
if rc == 0:
if expecting is True:
LOG.info('Found the non_volatile contents:' + output +
' as epxected')
return True, output, handle
else:
LOG.info(
'Not expecting but find non-volatile contents for:{}'.format(
handle))
if fail_ok:
return False, output, handle
else:
assert False, 'Not expecting but find non-volatile contents for:{}'.format(
handle)
else:
if expecting is False:
LOG.info('OK, did not find the NV content, this is expected.')
return True, output, handle
else:
LOG.warn('Did not find the NV content but expecting.')
if fail_ok:
return False, output, handle
else:
assert False, 'Expecting but failed to find non_volatile contents for handle:' + str(
handle)
def get_openstack_configs(conf_file,
configs=None,
node=None,
pods=None,
label_component=None,
label_app=None,
fail_ok=False,
con_ssh=None):
"""
Get config values for openstack pods with given chart
Args:
pods (str|list|tuple): openstack pod name(s)
label_app (str|None): e.g., nova, neutron, panko, ...
label_component (str|None): e.g., api, compute, etc.
conf_file (str): config file path inside the filtered openstack
container, e.g., /etc/nova/nova.conf
configs (dict): {<section1>(str): <field(s)>(str|list|tuple),
<section2>: ..}
e.g., {'database': 'event_time_to_live'}
node (str|None)
fail_ok:
con_ssh:
Returns (dict): {<pod1>(str): <settings>(dict), ... }
"""
if not pods and not (label_component and label_app):
raise ValueError('Either pods, or label_component and label_app '
'have to be specified to locate the containers')
if not pods:
pods = get_openstack_pods(component=label_component,
application=label_app,
fail_ok=fail_ok,
node=node,
con_ssh=con_ssh)
elif isinstance(pods, str):
pods = (pods, )
LOG.info('Getting {} {} values from openstack pods: {}'.format(
conf_file, configs, pods))
cmd = 'cat {}'.format(conf_file)
if configs:
all_fields = []
section_filter = r'$1 ~ /^\[.*\]/'
for fields in configs.values():
if isinstance(fields, str):
all_fields.append(fields)
elif isinstance(fields, (tuple, list)):
all_fields += list(fields)
fields_filter = '|| '.join(
['$1 ~ /^{}/'.format(field) for field in set(all_fields)])
cmd += r" | awk '{{ if ( {} || {}) print }}' | grep --color=never " \
r"--group-separator='' -B 1 -v '\[.*\]'". \
format(section_filter, fields_filter)
config_values = {}
for pod in pods:
code, output = exec_cmd_in_container(cmd,
pod=pod,
namespace='openstack',
con_ssh=con_ssh,
fail_ok=fail_ok)
if code > 0:
config_values[pod] = {}
continue
# Remove irrelevant string at beginning of the output
output = "[{}".format(
re.split(r'\n\[', r'\n{}'.format(output), maxsplit=1)[-1])
settings = configparser.ConfigParser()
settings.read_string(output)
config_values[pod] = settings
return config_values
def check_persistent_values(ssh_con, handle, expecting=True, fail_ok=False):
LOG.info('Check if the value still existing for handle:{}'.format(handle))
cli = 'getcapability -cap 1 -pr ' + str(handle)
output = ''
try:
output = run_cmd(ssh_con, cli, output_handle=False, fail_ok=fail_ok)[1]
except Exception as e:
LOG.warn('Caught exception: {}'.format(e))
if str(handle) in output or str(handle)[2:] in output:
if expecting is True:
LOG.info('OK, found the value in persistent memory, handle:' +
handle + ' as expected')
return True, handle, output
else:
LOG.warn(
'Not expecting but got value for handle:{}, results:{}'.format(
handle, output))
if fail_ok:
return False, output, handle
else:
assert False, 'Not-expecting but find the persistent contents:' + str(
handle)
else:
if expecting is True:
LOG.warn('Expecting but could not find the persistent values')
if fail_ok:
return False, output, handle
assert False, 'Could not find the persistent values, while expecting them'
else:
LOG.info(
'OK, as expected, no value in persistent memory, handle:' +
handle)
return True, output, handle
def wait_for_pods_status(pod_names=None,
partial_names=None,
labels=None,
namespace=None,
status=PodStatus.RUNNING,
timeout=120,
check_interval=3,
con_ssh=None,
fail_ok=False,
strict=False,
**kwargs):
"""
Wait for pod(s) to reach given status via kubectl get pod
Args:
pod_names (str|list|tuple): full name of the pods
partial_names (str|list|tuple): Used only if pod_names are not provided
labels (str|list|tuple|dict|None): Used only if pod_names are not
provided
namespace (None|str):
status (str|None|list): None means any state as long as pod exists.
timeout:
check_interval:
con_ssh:
fail_ok:
strict (bool):
Returns (tuple):
(True, <actual_pods_info>) # actual_pods_info is a dict with
pod_name as key, and pod_info(dict) as value
(False, <actual_pods_info>)
"""
pods_to_check = []
if pod_names:
if isinstance(pod_names, str):
pod_names = [pod_names]
else:
pod_names = list(pod_names)
labels = partial_names = None
pods_to_check = list(pod_names)
elif partial_names:
if isinstance(partial_names, str):
partial_names = [partial_names]
else:
partial_names = list(partial_names)
kwargs['NAME'] = partial_names
pods_to_check = list(partial_names)
actual_status = {}
end_time = time.time() + timeout
while time.time() < end_time:
pod_full_names = pods_to_check if pod_names else None
pods_values = get_pods(pod_names=pod_full_names,
field=('NAME', 'status'),
namespace=namespace,
labels=labels,
strict=strict,
fail_ok=True,
con_ssh=con_ssh,
**kwargs)
if not pods_values:
# No pods returned, continue to check.
time.sleep(check_interval)
continue
continue_check = False # This is used when only labels are provided
for pod_info in pods_values:
pod_name, pod_status = pod_info
actual_status[pod_name] = pod_status
if status and pod_status not in status:
# Status not as expected, continue to wait
continue_check = True
if partial_names:
# In this case, there might be multiple pods that matches
# 1 partial name, so the partial name that
# matches current pod could have been removed if there
# was one other pod that also matched the name
# had reached the desired state. In this case, we will
# add the partial name back to check list
for partial_name in partial_names:
if partial_name in pod_name and partial_name not in \
pods_to_check:
pods_to_check.append(partial_name)
break
else:
# Criteria met for current pod, remove it from check_list
if pod_names:
pods_to_check.remove(pod_name)
elif partial_names:
for partial_name in partial_names:
if partial_name in pod_name and partial_name in \
pods_to_check:
pods_to_check.remove(partial_name)
break
if not pods_to_check and not continue_check:
return True, actual_status
time.sleep(check_interval)
name_str = 'Names: {}'.format(pods_to_check) if pods_to_check else ''
label_str = 'Labels: {}'.format(labels) if labels else ''
criteria = '{} {}'.format(name_str, label_str).strip()
msg = "Pods did not reach expected status within {}s. Criteria not met: " \
"{}. Actual info: {}".format(timeout, criteria, actual_status)
if fail_ok:
LOG.info(msg)
return False, actual_status
raise exceptions.KubeError(msg)
def check_transient_values(ssh_con,
handles=None,
expecting=True,
fail_ok=False):
LOG.info(
'Check if the values stored in volatile memory existing or not, expecting: {}'
.format(expecting))
if handles:
if isinstance(handles, list) or isinstance(handles, tuple):
to_check = [h for h in handles if h]
else:
to_check = [handles]
else:
LOG.info('check if any values in volatile memory')
to_check = []
rc, values = get_volatile_content(ssh_con, fail_ok=fail_ok)
if rc == 0 and values == to_check:
if expecting is True:
LOG.info('OK, found transient contents as expected')
return True, values, handles
else:
LOG.warn('Not expecting but find values for handles:{}, vaules:{}'.
format(handles, values))
if fail_ok:
return False, values, handles
assert False, 'Failed, expecting nothing in transient memory, but got {}'.format(
values)
else:
if expecting is False:
LOG.info('OK, as expected, no transient contents found')
return True, values, handles
else:
LOG.warn(
'Expecting but failed to find values for handles:{}'.format(
handles))
if fail_ok:
return True, values, handles
else:
assert False, 'Failed to find expected contents:{}'.format(
to_check)
def wait_for_pods_healthy(pod_names=None,
namespace=None,
all_namespaces=True,
labels=None,
timeout=300,
check_interval=5,
con_ssh=None,
fail_ok=False,
exclude=False,
strict=False,
**kwargs):
"""
Wait for pods ready
Args:
pod_names (list|tuple|str|None): full name of pod(s)
namespace (str|None):
all_namespaces (bool|None)
labels (str|dict|list|tuple|None):
timeout:
check_interval:
con_ssh:
fail_ok:
exclude (bool)
strict (bool): strict applies to node and name matching if given
**kwargs
Returns (tuple):
"""
LOG.info("Wait for pods ready..")
if not pod_names:
pod_names = None
elif isinstance(pod_names, str):
pod_names = [pod_names]
bad_pods = None
end_time = time.time() + timeout
while time.time() < end_time:
bad_pods_info = get_unhealthy_pods(labels=labels,
field=('NAME', 'STATUS'),
namespace=namespace,
all_namespaces=all_namespaces,
con_ssh=con_ssh,
exclude=exclude,
strict=strict,
**kwargs)
bad_pods = {
pod_info[0]: pod_info[1]
for pod_info in bad_pods_info
if (not pod_names or pod_info[0] in pod_names)
}
if not bad_pods:
LOG.info("Pods are Completed or Running.")
if pod_names:
pod_names = [
pod for pod in pod_names
if not re.search('audit-|init-', pod)
]
if not pod_names:
return True
is_ready = wait_for_running_pods_ready(
pod_names=pod_names,
namespace=namespace,
all_namespaces=all_namespaces,
labels=labels,
timeout=int(end_time - time.time()),
strict=strict,
con_ssh=con_ssh,
fail_ok=fail_ok,
**kwargs)
return is_ready
time.sleep(check_interval)
msg = 'Some pods are not Running or Completed: {}'.format(bad_pods)
LOG.warning(msg)
if fail_ok:
return False
dump_pods_info(con_ssh=con_ssh)
raise exceptions.KubeError(msg)
def create_value(ssh_con, value_type='transient'):
global g_vms, g_flavors
LOG.info('Create values for types:{}'.format(value_type))
try:
if g_vms[value_type] and g_vms[value_type]['values']:
LOG.info(
'The values for the vm-type are already created, vm_type:{}, values:{}'
.format(value_type, g_vms[value_type]['values']))
return g_vms[value_type]['values']
except (TypeError, KeyError):
pass
key = ''
if 'transient' in value_type:
LOG.info('Creating values for types:{}'.format(value_type))
LOG.info('-Creating transient values')
key = create_testing_key(ssh_con)
LOG.info('-OK, successfully created transient values:{}'.format(key))
return key
if 'non_volatile' in value_type:
LOG.info('-Creating non_volatile values')
nv = create_nv_values(ssh_con)[0]
LOG.info('-OK, successfully created non_volatile value:{}'.format(nv))
return nv
if 'persistent' in value_type:
if not key:
LOG.info('-Creating transient values')
key = create_testing_key(ssh_con)
LOG.info(
'-OK, successfully created transient values:{}'.format(key))
LOG.info('-Create persistent values')
persistent = create_persistent_values(ssh_con, key)
LOG.info(
'-OK, successfully created persistent value:{}'.format(persistent))
return persistent
return ''
def test_swact_controllers(wait_for_con_drbd_sync_complete):
"""
Verify swact active controller
Test Steps:
- Boot a vm on system and check ping works
- Swact active controller
- Verify standby controller and active controller are swapped
- Verify vm is still pingable
"""
if system_helper.is_aio_simplex():
skip("Simplex system detected")
if not wait_for_con_drbd_sync_complete:
skip(SkipSysType.LESS_THAN_TWO_CONTROLLERS)
LOG.tc_step('retrieve active and available controllers')
pre_active_controller, pre_standby_controller = \
system_helper.get_active_standby_controllers()
assert pre_standby_controller, "No standby controller available"
pre_res_sys, pre_msg_sys = system_helper.wait_for_services_enable(
timeout=20, fail_ok=True)
up_hypervisors = host_helper.get_up_hypervisors()
pre_res_neutron, pre_msg_neutron = network_helper.wait_for_agents_healthy(
up_hypervisors, timeout=20, fail_ok=True)
LOG.tc_step("Boot a vm from image and ping it")
vm_id_img = vm_helper.boot_vm(name='swact_img',
source='image',
cleanup='function')[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_img)
LOG.tc_step("Boot a vm from volume and ping it")
vm_id_vol = vm_helper.boot_vm(name='swact', cleanup='function')[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_vol)
LOG.tc_step("Swact active controller and ensure active controller is "
"changed")
host_helper.swact_host(hostname=pre_active_controller)
LOG.tc_step("Verify standby controller and active controller are swapped")
post_active_controller = system_helper.get_active_controller_name()
post_standby_controller = system_helper.get_standby_controller_name()
assert pre_standby_controller == post_active_controller, \
"Prev standby: {}; Post active: {}".format(
pre_standby_controller, post_active_controller)
assert pre_active_controller == post_standby_controller, \
"Prev active: {}; Post standby: {}".format(
pre_active_controller, post_standby_controller)
LOG.tc_step("Check boot-from-image vm still pingable after swact")
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_img, timeout=30)
LOG.tc_step("Check boot-from-volume vm still pingable after swact")
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_vol, timeout=30)
LOG.tc_step("Check system services and neutron agents after swact "
"from {}".format(pre_active_controller))
post_res_sys, post_msg_sys = \
system_helper.wait_for_services_enable(fail_ok=True)
post_res_neutron, post_msg_neutron = \
network_helper.wait_for_agents_healthy(hosts=up_hypervisors,
fail_ok=True)
assert post_res_sys, "\nPost-evac system services stats: {}" \
"\nPre-evac system services stats: {}". \
format(post_msg_sys, pre_msg_sys)
assert post_res_neutron, "\nPost evac neutron agents stats: {}" \
"\nPre-evac neutron agents stats: {}". \
format(pre_msg_neutron, post_msg_neutron)
LOG.tc_step("Check hosts are Ready in kubectl get nodes after swact")
kube_helper.wait_for_nodes_ready(hosts=(pre_active_controller,
pre_standby_controller),
timeout=30)
def perform_vm_operation(vm_type,
vm_id,
op='live_migration',
extra_specs='vtpm'):
LOG.info('Perform action:{} to the VM, extra specs:{}'.format(
op, extra_specs))
op_table = {
'live_migration':
lambda x, y: vm_helper.live_migrate_vm(y),
'cold_migration':
lambda x, y: vm_helper.cold_migrate_vm(y),
'stop_start':
lambda x, y: (vm_helper.stop_vms(y), vm_helper.start_vms(y)),
'suspend_resume':
lambda x, y: (vm_helper.suspend_vm(y), vm_helper.resume_vm(y)),
'pause_unpause':
lambda x, y: (vm_helper.pause_vm(y), vm_helper.unpause_vm(y)),
'reboot_host':
lambda x, y: reboot_hosting_node(x, y, force_reboot=False),
'soft_reboot':
lambda x, y: vm_helper.reboot_vm(y, hard=False),
'hard_reboot':
lambda x, y: vm_helper.reboot_vm(y, hard=True),
'lock_unlock':
lambda x, y: lock_unlock_hosting_node(x, y, force_lock=False),
'evacuate':
lambda x, y: reboot_hosting_node(x, y, force_reboot=True),
}
if op in op_table:
LOG.info('Perform action: {}'.format(op))
op_table[op](vm_type, vm_id)
return True
elif op == 'resize_to_autorc':
if vm_type == 'autorc':
LOG.info(
'resize from AUTO-RECOVERY to another AUTO-RECOVER flavor')
to_flavor_id = get_flavor_id(vm_type, 'autorc2')
LOG.info('TODO: {}, m_type={}, to_flavor_id={}'.format(
to_flavor_id, vm_type, to_flavor_id))
vm_helper.resize_vm(vm_id, to_flavor_id)
elif op == 'resize_to_non_autorc':
LOG.info('perform {} on type:{}, id:{}'.format(op, vm_type, vm_id))
if vm_type == 'non_autorc2':
LOG.warn(
'resize from AUTO-RECOVERY to another AUTO-RECOVER flavor')
to_flavor_id = get_flavor_id(vm_type, 'non_autorc2')
vm_helper.resize_vm(vm_id, to_flavor_id)
elif op == 'resize_to_non_vtpm':
LOG.info('perform {} on type:{}, id:{}'.format(op, vm_type, vm_id))
to_flavor_id = get_flavor_id(vm_type, 'non_vtpm')
vm_helper.resize_vm(vm_id, to_flavor_id)
else:
LOG.fatal('Unsupported action: {}'.format(op))
return False
def test_good_authentication(sysinv_rest, operation, resource):
if operation == "GET":
LOG.info("getting... {}".format(resource))
rest_test_helper.get(sysinv_rest, resource=resource)
def check_host_file_for_vm(vm_id, expecting=True, host=None, fail_ok=True):
LOG.info('Verify the file for vTPM exists on the hosting node for VM:' +
vm_id)
if host is None:
host = vm_helper.get_vm_host(vm_id)
active_controller_name = system_helper.get_active_controller_name()
instance_name = vm_helper.get_vm_instance_name(vm_id)
vtpm_file = vtpm_base_dir.format(
vm_id=vm_id, instance_name=instance_name) + '/' + vtpm_file_name
if host != active_controller_name:
hosting_node = host
else:
hosting_node = active_controller_name
with host_helper.ssh_to_host(hosting_node) as ssh_client:
if ssh_client.file_exists(vtpm_file):
LOG.info('OK, found the file for vTPM:{} on host:{}'.format(
vtpm_file, host))
assert expecting is True or fail_ok is True, \
'FAIL, the files supporting vTPM are NOT found on the {} as expected'.format(host)
if expecting is True:
LOG.info('-this is expected')
else:
LOG.info('-this is NOT expected')
return True, expecting
else:
LOG.info('Cannot find the file for vTPM:{} on host:{}'.format(
vtpm_file, host))
assert expecting is False or fail_ok is True, \
'FAIL, the files should be cleared as expected'
if expecting is False:
LOG.info('-this is expected')
else:
LOG.info('-this is NOT expected')
return False, expecting
def post_restore_test(con_ssh):
LOG.info(
'Perform system testing and checking after the system is restored')
check_volumes_spaces(con_ssh)
def test_vm_mem_pool_default_config(prepare_resource, mem_page_size):
"""
Test memory used by vm is taken from the expected memory pool
Args:
prepare_resource (tuple): test fixture
mem_page_size (str): mem page size setting in flavor
Setup:
- Create a flavor with 1G RAM (module)
- Create a volume with default values (module)
- Select a hypervisor to launch vm on
Test Steps:
- Set memory page size flavor spec to given value
- Attempt to boot a vm with above flavor and a basic volume
- Verify the system is taking memory from the expected memory pool:
- If boot vm succeeded:
- Calculate the available/used memory change on the vm host
- Verify the memory is taken from memory pool specified via mem_page_size
- If boot vm failed:
- Verify system attempted to take memory from expected pool, but insufficient
memory is available
Teardown:
- Delete created vm
- Delete created volume and flavor (module)
"""
hypervisor, flavor_1g, volume_ = prepare_resource
_wait_for_all_app_hp_avail(host=hypervisor)
LOG.tc_step("Set memory page size extra spec in flavor")
nova_helper.set_flavor(
flavor_1g, **{
FlavorSpec.CPU_POLICY: 'dedicated',
FlavorSpec.MEM_PAGE_SIZE: mem_page_size
})
LOG.tc_step("Check system host-memory-list before launch vm")
is_sufficient, prev_host_mems = is_host_mem_sufficient(
host=hypervisor, mempage_size=mem_page_size)
LOG.tc_step("Boot a vm with mem page size spec - {}".format(mem_page_size))
code, vm_id, msg = vm_helper.boot_vm('mempool_' + mem_page_size,
flavor_1g,
source='volume',
fail_ok=True,
vm_host=hypervisor,
source_id=volume_,
cleanup='function')
if not is_sufficient:
LOG.tc_step(
"Check boot vm rejected due to insufficient memory from {} pool".
format(mem_page_size))
assert 1 == code, "{} vm launched successfully when insufficient mempage configured on {}".\
format(mem_page_size, hypervisor)
else:
LOG.tc_step(
"Check vm launches successfully and {} available mempages change accordingly"
.format(hypervisor))
assert 0 == code, "VM failed to launch with '{}' mempages".format(
mem_page_size)
check_mempage_change(vm_id,
host=hypervisor,
prev_host_mems=prev_host_mems,
mempage_size=mem_page_size,
mem_gib=1)
