def test_migrate_stress(check_hypervisors, boot_source, count):
LOG.tc_step("Launch a VM from {}".format(boot_source))
vm = vm_helper.boot_vm(name='{}-stress'.format(boot_source), cleanup='function',
source=boot_source)[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm)
block_mig = True if boot_source == 'image' else False
if not block_mig:
LOG.tc_step("Attempt to block migration on boot-from-volume VM and ensure if fails")
code = vm_helper.live_migrate_vm(vm_id=vm, block_migrate=True)[0]
assert code > 0, "Block migration passed unexpectedly for boot-from-volume vm"
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm)
LOG.tc_step("Live migrate and ping vm 1000 times")
for i in range(count):
LOG.info('Live migration iter{}'.format(i+1))
vm_helper.live_migrate_vm(vm)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm)
LOG.tc_step("Cold migrate vm followed by live migrate {} times".format(count))
for i in range(count):
LOG.info('Cold+live migration iter{}'.format(i + 1))
vm_helper.cold_migrate_vm(vm_id=vm)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm)
vm_helper.live_migrate_vm(vm)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm)
def test_migrate_4k_vm_positive(ephemeral, swap, cpu_pol, vcpus, vm_type,
ensure_sufficient_4k_pages):
"""
Test live and cold migrate 4k vm with various vm storage configurations
Args:
ephemeral (int):
swap (int):
cpu_pol (str):
vcpus (int):
vm_type (str): boot-from image or volume vm
ensure_sufficient_4k_pages (tuple): module test fixture to configure 4k pages
Setups:
- Select at least 2 hosts with specified storage backing. e.g., local_image, or remote
- Ensure 2 hosts are in nova zone (move rest to cgcsauto zone if more than 2)
- Configure the 2 hosts with large amount of 4k pages
Test Steps:
- Create flavor with specified ephemeral, swap,
"""
storage_backing, hosts = ensure_sufficient_4k_pages
vm_id = _boot_vm_under_test(storage_backing, ephemeral, swap, cpu_pol,
vcpus, vm_type)
LOG.tc_step("Cold migrate VM and ensure it succeeded")
vm_helper.cold_migrate_vm(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
__check_pagesize(vm_id)
LOG.tc_step("Attempt to live migrate VM")
vm_helper.live_migrate_vm(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
__check_pagesize(vm_id)
def _test_migrate_anti_affinity_vms_in_parallel():
"""
cold-migrate and live-migrate vms from anti-affinity group
It will check if the heat stack is launched already if not it will launch the stack
find the vms in anti-affinity group and will do cold and live migration
"""
# First make sure heat stack is there:
system_test_helper.launch_heat_stack()
srv_grps_info = nova_helper.get_server_groups_info(headers=('Policies',
'Metadata',
'Members'))
vms = []
for group in srv_grps_info:
policies, metadata, members = srv_grps_info[group]
if members and 'anti-affinity' in policies and metadata[
'wrs-sg:best_effort'] == 'false':
if len(members) >= 10:
vms = members[range(0, 9)]
break
else:
skip("There are no VMs in anti-affinity server group")
check_vm_hosts(vms=vms, policy='anti_affinity')
for vm_id in vms:
vm_helper.wait_for_vm_status(vm_id=vm_id, check_interval=10)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)
vm_helper.live_migrate_vm(vm_id=vm_id)
vm_helper.cold_migrate_vm(vm_id=vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)
check_vm_hosts(vms=vms, policy='anti_affinity')
def test_heat_vm_scale_after_actions(vm_scaling_stack, actions):
"""
Test VM auto scaling with swact:
Create heat stack for auto scaling using NestedAutoScale.yaml, swact and perform vm scale up and down.
Test Steps:
- Create a heat stack for auto scaling vm ()
- Verify heat stack is created successfully
- Verify heat resources are created
- live migrate the vm if not sx
- cold migrate the vm if not sx
- swact if not sx
- reboot -f vm host
- trigger auto scale by boosting cpu usage in the vm (using dd)
- verify it scale up to the max number of vms (3)
- trigger scale down by killing dd in the vm
- verify the vm scale down to min number (1)
- Delete Heat stack and verify resource deletion
"""
stack_name, vm_id = vm_scaling_stack
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)
if not system_helper.is_aio_simplex():
actions = actions.split('-')
if "swact" in actions:
LOG.tc_step("Swact before scale in/out")
host_helper.swact_host()
if "live_migrate" in actions:
LOG.tc_step("live migrate vm before scale in/out")
vm_helper.live_migrate_vm(vm_id)
if "cold_migrate" in actions:
LOG.tc_step("cold migrate vm before scale in/out")
vm_helper.cold_migrate_vm(vm_id)
if "host_reboot" in actions:
if system_helper.is_aio_simplex():
host_helper.reboot_hosts('controller-0')
vm_helper.wait_for_vm_status(vm_id,
status=VMStatus.ACTIVE,
timeout=600,
check_interval=10,
fail_ok=False)
vm_helper.wait_for_vm_pingable_from_natbox(
vm_id, timeout=VMTimeout.DHCP_RETRY)
else:
LOG.tc_step("evacuate vm before scale in/out")
vm_host = vm_helper.get_vm_host(vm_id=vm_id)
vm_helper.evacuate_vms(host=vm_host, vms_to_check=vm_id)
LOG.tc_step(
"Wait for {} vms to auto scale out to {} after running dd in vm(s)".
format(stack_name, 3))
vm_helper.wait_for_auto_vm_scale_out(stack_name, expt_max=3)
LOG.tc_step(
"Wait for {} vms to auto scale in to {} after killing dd processes in vms"
.format(stack_name, 1))
vm_helper.wait_for_auto_vm_scale_in(stack_name, expt_min=1)
def test_evacuate_vms(self, vms_):
"""
Test evacuated vms
Args:
vms_: (fixture to create vms)
Pre-requisites:
- At least two up hypervisors on system
Test Steps:
- Create vms with various options:
- vm booted from cinder volume,
- vm booted from glance image,
- vm booted from glance image, and have an extra cinder
volume attached after launch,
- vm booed from cinder volume with ephemeral and swap disks
- Move vms onto same hypervisor
- sudo reboot -f on the host
- Ensure vms are successfully evacuated to other host
- Live migrate vms back to original host
- Check vms can move back, and vms are still reachable from natbox
- Check system services are enabled and neutron agents are alive
"""
vms, target_host = vms_
pre_res_sys, pre_msg_sys = system_helper.wait_for_services_enable(
timeout=20, fail_ok=True)
up_hypervisors = host_helper.get_up_hypervisors()
pre_res_neutron, pre_msg_neutron = \
network_helper.wait_for_agents_healthy(
up_hypervisors, timeout=20, fail_ok=True)
LOG.tc_step(
"reboot -f on vms host, ensure vms are successfully evacuated and "
"host is recovered after reboot")
vm_helper.evacuate_vms(host=target_host,
vms_to_check=vms,
wait_for_host_up=True,
ping_vms=True)
LOG.tc_step("Check rebooted host can still host vm")
vm_helper.live_migrate_vm(vms[0], destination_host=target_host)
vm_helper.wait_for_vm_pingable_from_natbox(vms[0])
LOG.tc_step("Check system services and neutron agents after {} "
"reboot".format(target_host))
post_res_sys, post_msg_sys = system_helper.wait_for_services_enable(
fail_ok=True)
post_res_neutron, post_msg_neutron = \
network_helper.wait_for_agents_healthy(hosts=up_hypervisors,
fail_ok=True)
assert post_res_sys, "\nPost-evac system services stats: {}" \
"\nPre-evac system services stats: {}". \
format(post_msg_sys, pre_msg_sys)
assert post_res_neutron, "\nPost evac neutron agents stats: {}" \
"\nPre-evac neutron agents stats: {}". \
format(pre_msg_neutron, post_msg_neutron)
def obsolete_test_fip(fip_setups):
"""
Test VM Floating IP over VM launch, live-migration, cold-migration, pause/unpause, etc
Args:
fip_setups: test fixture
Test Setups (module):
- Create a floating ip
- boot a vm
- Attach floating ip to vm
Test Steps:
- Ping VM FIP
- Live-migrate the VM and verify ping from VM
- Cold-migrate the VM and verify ping from VM
- Pause and un-pause the VM and verify ping from VM
- Suspend and resume the VM and verify ping from VM
- Stop and start the VM and verify ping from VM
- Reboot the VM and verify ping from VM
- Ping VM FIP
Test Teardown:
- Delete created FIP and vm (module)
"""
vm_id, fip = fip_setups
LOG.tc_step("Ping VM with Floating IP ")
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Live-migrate the VM and verify ping from VM")
vm_helper.live_migrate_vm(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Cold-migrate the VM and verify ping from VM")
vm_helper.cold_migrate_vm(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Pause and un-pause the VM and verify ping from VM")
vm_helper.pause_vm(vm_id)
vm_helper.unpause_vm(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Suspend and resume the VM and verify ping from VM")
vm_helper.suspend_vm(vm_id)
vm_helper.resume_vm(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Stop and start the VM and verify ping from VM")
vm_helper.stop_vms(vm_id)
vm_helper.start_vms(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Reboot the VM and verify ping from VM")
vm_helper.reboot_vm(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Ping VM with Floating IP Ensure FIP reachable ")
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
def test_system_persist_over_host_reboot(host_type, stx_openstack_required):
"""
Validate Inventory summary over reboot of one of the controller see if data persists over reboot
Test Steps:
- capture Inventory summary for list of hosts on system service-list and neutron agent-list
- reboot the current Controller-Active
- Wait for reboot to complete
- Validate key items from inventory persist over reboot
"""
if host_type == 'controller':
host = system_helper.get_active_controller_name()
elif host_type == 'compute':
if system_helper.is_aio_system():
skip("No compute host for AIO system")
host = None
else:
hosts = system_helper.get_hosts(personality='storage')
if not hosts:
skip(msg="Lab has no storage nodes. Skip rebooting storage node.")
host = hosts[0]
LOG.tc_step("Pre-check for system status")
system_helper.wait_for_services_enable()
up_hypervisors = host_helper.get_up_hypervisors()
network_helper.wait_for_agents_healthy(hosts=up_hypervisors)
LOG.tc_step("Launch a vm")
vm_id = vm_helper.boot_vm(cleanup='function')[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
if host is None:
host = vm_helper.get_vm_host(vm_id)
LOG.tc_step("Reboot a {} node and wait for reboot completes: {}".format(host_type, host))
HostsToRecover.add(host)
host_helper.reboot_hosts(host)
host_helper.wait_for_hosts_ready(host)
LOG.tc_step("Check vm is still active and pingable after {} reboot".format(host))
vm_helper.wait_for_vm_status(vm_id, status=VMStatus.ACTIVE, fail_ok=False)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id, timeout=VMTimeout.DHCP_RETRY)
LOG.tc_step("Check neutron agents and system services are in good state after {} reboot".format(host))
network_helper.wait_for_agents_healthy(up_hypervisors)
system_helper.wait_for_services_enable()
if host in up_hypervisors:
LOG.tc_step("Check {} can still host vm after reboot".format(host))
if not vm_helper.get_vm_host(vm_id) == host:
time.sleep(30)
vm_helper.live_migrate_vm(vm_id, destination_host=host)
def pb_migrate_test(backup_info, con_ssh, vm_ids=None):
"""
Run migration test before doing system backup.
Args:
backup_info:
- options for doing backup
con_ssh:
- current ssh connection
vm_ids
Return:
None
"""
hyporvisors = host_helper.get_up_hypervisors(con_ssh=con_ssh)
if len(hyporvisors) < 2:
LOG.info(
'Only {} hyporvisors, it is not enougth to test migration'.format(
len(hyporvisors)))
LOG.info('Skip migration test')
return 0
else:
LOG.debug('There {} hyporvisors'.format(len(hyporvisors)))
LOG.info('Randomly choose some VMs and do migrate:')
target = random.choice(vm_ids)
LOG.info('-OK, test migration of VM:{}'.format(target))
original_host = vm_helper.get_vm_host(target)
LOG.info('Original host:{}'.format(original_host))
vm_helper.live_migrate_vm(target)
current_host = vm_helper.get_vm_host(target)
LOG.info('After live-migration, host:{}'.format(original_host))
if original_host == current_host:
LOG.info('backup_info:{}'.format(backup_info))
LOG.warn(
'VM is still on its original host, live-migration failed? original host:{}'
.format(original_host))
original_host = current_host
vm_helper.cold_migrate_vm(target)
current_host = vm_helper.get_vm_host(target)
LOG.info('After code-migration, host:{}'.format(current_host))
if original_host == current_host:
LOG.warn(
'VM is still on its original host, code-migration failed? original host:{}'
.format(original_host))
def live_migrate_vm(end_time, end_event):
ded_flv = nova_helper.create_flavor(name='dedicated', vcpus=2)[1]
nova_helper.set_flavor(ded_flv, **{FlavorSpec.CPU_POLICY: 'dedicated'})
vm_id = vm_helper.boot_vm(name='live-mig', flavor=ded_flv)[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
while time.time() < end_time:
if end_event.is_set():
assert 0, "Other thread failed. Terminate live-mgiration thread."
time.sleep(15)
LOG.tc_step("Live migrate live-mig vm")
vm_helper.live_migrate_vm(vm_id=vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
def _perform_nova_actions(vms_dict, flavors, vfs=None):
for vm_name, vm_id in vms_dict.items():
LOG.tc_step("Cold migrate VM {} ....".format(vm_name))
vm_helper.cold_migrate_vm(vm_id=vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
LOG.tc_step("Live migrate VM {} ....".format(vm_name))
expt_codes = [0] if 'vm_no_crypto' in vm_name else [1, 6]
code, msg = vm_helper.live_migrate_vm(vm_id=vm_id, fail_ok=True)
assert code in expt_codes, "Expect live migrate to fail for vm with pci device attached. Actual: {}".format(msg)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
LOG.tc_step("Suspend/Resume VM {} ....".format(vm_name))
vm_helper.suspend_vm(vm_id)
vm_helper.resume_vm(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
if vfs is None:
resize_flavor_id = flavors["flavor_resize_qat_vf_1"] if "no_crypto" not in vm_name else \
flavors["flavor_resize_none"]
else:
resize_flavor_id = flavors['flavor_resize_qat_vf_{}'.format(vfs)]
LOG.info("Resizing VM {} to new flavor {} ...".format(vm_name, resize_flavor_id))
vm_helper.resize_vm(vm_id, resize_flavor_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)
def _check_anti_affinity_vms():
storage_backing, hosts = keywords.host_helper.get_storage_backing_with_max_hosts()
best_effort = True if len(hosts) < 3 else False
anti_affinity_vms = nova_helper.get_server_group_info(group_name='grp_anti_affinity', headers='Members')[0]
check_vm_hosts(vms=anti_affinity_vms, policy='anti_affinity', best_effort=best_effort)
vm_hosts = []
for vm_id in anti_affinity_vms:
vm_helper.wait_for_vm_status(vm_id=vm_id, check_interval=10)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)
vm_helper.live_migrate_vm(vm_id=vm_id)
vm_helper.cold_migrate_vm(vm_id=vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)
vm_hosts.append(vm_helper.get_vm_host(vm_id))
return vm_hosts, anti_affinity_vms
def test_live_migrate_vm_positive(hosts_per_stor_backing, storage_backing,
ephemeral, swap, cpu_pol, vcpus, vm_type,
block_mig):
"""
Skip Condition:
- Less than two hosts have specified storage backing
Test Steps:
- create flavor with specified vcpus, cpu_policy, ephemeral, swap,
storage_backing
- boot vm from specified boot source with above flavor
- (attach volume to vm if 'image_with_vol', specified in vm_type)
- Live migrate the vm with specified block_migration flag
- Verify VM is successfully live migrated to different host
Teardown:
- Delete created vm, volume, flavor
"""
if len(hosts_per_stor_backing.get(storage_backing, [])) < 2:
skip("Less than two hosts have {} storage backing".format(
storage_backing))
vm_id = _boot_vm_under_test(storage_backing, ephemeral, swap, cpu_pol,
vcpus, vm_type)
prev_vm_host = vm_helper.get_vm_host(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
vm_disks = vm_helper.get_vm_devices_via_virsh(vm_id)
file_paths, content = touch_files_under_vm_disks(vm_id=vm_id,
ephemeral=ephemeral,
swap=swap, vm_type=vm_type,
disks=vm_disks)
LOG.tc_step("Live migrate VM and ensure it succeeded")
# block_mig = True if boot_source == 'image' else False
code, output = vm_helper.live_migrate_vm(vm_id, block_migrate=block_mig)
assert 0 == code, "Live migrate is not successful. Details: {}".format(
output)
post_vm_host = vm_helper.get_vm_host(vm_id)
assert prev_vm_host != post_vm_host
LOG.tc_step("Ensure vm is pingable from NatBox after live migration")
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
LOG.tc_step("Check files after live migrate")
check_helper.check_vm_files(vm_id=vm_id, storage_backing=storage_backing,
ephemeral=ephemeral, swap=swap,
vm_type=vm_type, vm_action='live_migrate',
file_paths=file_paths, content=content,
disks=vm_disks, prev_host=prev_vm_host,
post_host=post_vm_host)
def _check_affinity_vms():
affinity_vms = nova_helper.get_server_group_info(group_name='grp_affinity', headers='Members')[0]
vm_host = check_vm_hosts(vms=affinity_vms, policy='affinity')[0]
for vm_id in affinity_vms:
vm_helper.wait_for_vm_status(vm_id=vm_id, check_interval=10)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)
res, out = vm_helper.live_migrate_vm(vm_id=vm_id, fail_ok=True)
assert res in (1, 2, 6), out
res, out = vm_helper.cold_migrate_vm(vm_id=vm_id, fail_ok=True)
assert res in (1, 2), out
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id)
return vm_host, affinity_vms
def test_shared_cpu_migrate(self, config_host_cpus):
"""
Test vm with shared cpus enabled can successful live migrate to a node with shared vcpus enabled and fails when
it tries to migrate to a node with shared vcpus disabled
Setup:
- Skip if there are less than 3 hosts
- Configure at least one compute to disable shared vcpus
- Configure at least two computes to have shared cpus via
'system host-cpu-modify -f shared p0=1,p1=1 <hostname>' (module)
Test Steps:
- Create flavor with given number of vcpus
- Add specific cpu_policy, shared_vcpu values to flavor extra specs
- Boot a vm with the flavor
- Ensure vm is booted successfully
- Perform a non-forced live migration on vm. Ensure that vm is on a shared cpu host.
- Perform a non-forced cold migration on vm. Ensure that vm is on a shared cpu host.
- Force live-migrate vm to host with shared vcpus enabled. The migration should succeed
- Ensure that the vm is on a different host
- Force live-migrate vm to the host with disabled shared vcpus. The migration should fail
- Verify error by ensuring that vm is still on same host and grep nova-scheduler logs for
'CANNOT SCHEDULE'
Teardown:
- Delete created vm if any (function)
- Revert any hosts that were changed for this test
"""
storage_backing, disable_shared_cpu_host, enabled_shared_hosts = config_host_cpus
LOG.tc_step("Create a flavor with given number of vcpus")
flavor = create_shared_flavor(vcpus=2, storage_backing=storage_backing, shared_vcpu=1)
nova_helper.set_flavor(flavor, **{FlavorSpec.MEM_PAGE_SIZE: 2048})
LOG.tc_step("Boot a vm with above flavor, and ensure vm is booted successfully")
vm_id = vm_helper.boot_vm(name='shared_cpu', flavor=flavor, fail_ok=False, cleanup='function')[1]
origin_host = vm_helper.get_vm_host(vm_id)
assert origin_host in enabled_shared_hosts, "VM not booted on shared cpu host"
LOG.tc_step("Perform a non-forced live migration onto an enabled shared cpu host, expect success")
vm_helper.live_migrate_vm(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
new_host = vm_helper.get_vm_host(vm_id)
assert new_host in enabled_shared_hosts, "VM not migrated on shared cpu host"
LOG.tc_step("Perform a non-forced cold migration onto an enabled shared cpu host, expect success")
vm_helper.cold_migrate_vm(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
new_host = vm_helper.get_vm_host(vm_id)
assert new_host in enabled_shared_hosts, "VM not migrated on shared cpu host"
if new_host != enabled_shared_hosts[0]:
dest_host = enabled_shared_hosts[0]
else:
dest_host = enabled_shared_hosts[1]
LOG.tc_step("Perform second live migration onto an enabled shared cpu host, expect success")
vm_helper.live_migrate_vm(vm_id, destination_host=dest_host)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
LOG.tc_step("Perform third live migration onto a disabled shared cpu host, expect failure")
code = vm_helper.live_migrate_vm(vm_id, destination_host=disable_shared_cpu_host, fail_ok=True)[0]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
assert code > 0, "Migrate not rejected as expected"
assert vm_helper.get_vm_host(vm_id) == dest_host, "VM not on same compute node"
LOG.tc_step("Verify second live migration failed via nova-scheduler.log")
req_id = get_failed_live_migrate_action_id(vm_id)
grepcmd = "grep 'CANNOT SCHEDULE' /var/log/nova/nova-scheduler.log | grep {}".format(req_id)
control_ssh = ControllerClient.get_active_controller()
control_ssh.exec_cmd(grepcmd, fail_ok=False)
def test_launch_vm_with_shared_cpu(self, vcpus, shared_vcpu, error, add_shared_cpu, origin_total_vcpus):
"""
Test boot vm cli returns error when system does not meet the shared cpu requirement(s) in given flavor
Args:
vcpus (int): number of vcpus to set when creating flavor
shared_vcpu (int):
error
add_shared_cpu
origin_total_vcpus
Setup:
- Configure one compute to have shared cpus via 'system host-cpu-modify -f shared p0=1,p1=1 <hostname>'
Test Steps:
- Create flavor with given number of vcpus
- Add specific cpu_policy, number of numa nodes, nume_node.0 , shared_vcpu values to flavor extra specs
- Boot a vm with the flavor
- Ensure vm is booted successfully
- Validate the shared cpu
- Live migrate the vm
- Re-validate the shared cpu
- Cold migrate the vm
- Re-validate the shared cpu
Teardown:
- Delete created vm if any (function)
- Delete created volume if any (module)
- Set shared cpus to 0 (default setting) on the compute node under test (module)
"""
storage_backing, shared_cpu_hosts, max_vcpus_per_proc = add_shared_cpu
LOG.tc_step("Create a flavor with given number of vcpus")
flavor = create_shared_flavor(vcpus, storage_backing=storage_backing, shared_vcpu=shared_vcpu)
LOG.tc_step("Boot a vm with above flavor")
code, vm_id, output = vm_helper.boot_vm(name='shared_cpu', flavor=flavor, fail_ok=True, cleanup='function')
if error:
LOG.tc_step("Check vm boot fail")
assert 1 == code, "Expect error vm. Actual result: {}".format(output)
LOG.tc_step("Ensure vm is in error state with expected fault message in nova show")
vm_helper.wait_for_vm_values(vm_id, 10, status='ERROR', fail_ok=False)
actual_fault = vm_helper.get_vm_fault_message(vm_id)
expt_fault = 'shared vcpu with 0 requested dedicated vcpus is not allowed'
assert expt_fault in actual_fault, "Expected fault message mismatch"
return
LOG.tc_step("Check vm booted successfully and shared cpu indicated in vm-topology")
assert 0 == code, "Boot vm failed. Details: {}".format(output)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
check_shared_vcpu(vm=vm_id, shared_vcpu=shared_vcpu, vcpus=vcpus, prev_total_vcpus=origin_total_vcpus)
# live migrate
LOG.tc_step("Live migrate vm and then ping vm from NatBox")
vm_helper.live_migrate_vm(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
check_shared_vcpu(vm=vm_id, shared_vcpu=shared_vcpu, vcpus=vcpus, prev_total_vcpus=origin_total_vcpus)
# cold migrate
LOG.tc_step("Cold migrate vm and then ping vm from NatBox")
vm_helper.cold_migrate_vm(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
check_shared_vcpu(vm=vm_id, shared_vcpu=shared_vcpu, vcpus=vcpus, prev_total_vcpus=origin_total_vcpus)
def _test_ea_max_vms_with_crypto_vfs(_flavors, hosts_pci_device_info):
"""
Verify maximum number of guests with Crypto VFs can be launched and
stabilized
Args:
_flavors:
hosts_pci_device_info:
Returns:
"""
LOG.info("Pci device {}".format(hosts_pci_device_info))
flavor_id = _flavors['flavor_qat_vf_4']
# Assume we only have 1 coleto creek pci device on system
crypto_hosts = list(hosts_pci_device_info.keys())
host = crypto_hosts[0]
vf_info = hosts_pci_device_info[host][0]
vf_device_id = vf_info['vf_device_id']
vf_count = vf_info['vf_count']
LOG.info("Vf_device_id {}, count: {}".format(vf_device_id, vf_count))
# number of vms to launch to max out the total configured device VFs. Each VM is launched with 4 Vfs. 4 Vfs in each
# compute are reserved for resize nova action.
number_of_vms = int((vf_count - 4 * len(crypto_hosts)) / 4)
mgmt_net_id = network_helper.get_mgmt_net_id()
tenant_net_id = network_helper.get_tenant_net_id()
vif_type = get_vif_type()
nics = [{'net-id': mgmt_net_id},
{'net-id': tenant_net_id, 'vif-model': vif_type}]
vm_helper.ensure_vms_quotas(number_of_vms + 10)
vms = {}
LOG.tc_step("Launch {} vms using flavor flavor_qat_vf_4 and nics {}".format(number_of_vms, nics))
for i in range(1, number_of_vms + 1):
vm_name = 'vm_crypto_{}'.format(i)
vm_id = vm_helper.boot_vm(cleanup='function', name='vm_crypto_{}'.format(i), nics=nics, flavor=flavor_id)[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
vms[vm_name] = vm_id
for vm_name_, vm_id_ in vms.items():
vm_host = vm_helper.get_vm_host(vm_id_)
host_dev_name = hosts_pci_device_info[vm_host][0]['device_name']
expt_qat_devs = {host_dev_name: 4}
check_helper.check_qat_service(vm_id=vm_id_, qat_devs=expt_qat_devs)
LOG.info("Checking if other host has room for cold migrate vm {}".format(vm_name_))
for host_ in crypto_hosts:
if host_ != vm_host:
total_vfs, used_vfs = network_helper.get_pci_device_vfs_counts_for_host(
host_, device_id=vf_device_id, fields=('pci_vfs_configured', 'pci_vfs_used'))
if int(total_vfs) - int(used_vfs) >= 4:
LOG.info("Migrate to other host is possible")
expt_res = 0
break
else:
LOG.info("Migrate to other host is not possible")
expt_res = 2
LOG.tc_step("Attempt to cold migrate {} and ensure it {}".format(vm_name_,
'succeeds' if expt_res == '0' else 'fails'))
rc, msg = vm_helper.cold_migrate_vm(vm_id=vm_id_, fail_ok=True)
assert expt_res == rc, "Expected: {}. Actual: {}".format(expt_res, msg)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_)
LOG.tc_step("Suspend/resume VM {} ....".format(vm_name_))
vm_helper.suspend_vm(vm_id_)
vm_helper.resume_vm(vm_id_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_)
# vm_host = nova_helper.get_vm_host(vm_id_)
# total, used = network_helper.get_pci_device_vfs_counts_for_host(vm_host, vf_device_id)[0]
# if (total - int(used)) >= 4:
# expt_res = 0
flavor_resize_id = _flavors['flavor_resize_qat_vf_4']
LOG.tc_step("Resize VM {} to new flavor {} with increased memory...".format(vm_name_, flavor_resize_id))
vm_helper.resize_vm(vm_id_, flavor_resize_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_)
# else:
# expt_res = 1
# LOG.info("Resizing of vm {} skipped; host {} max out vfs; used vfs = {}".format(vm_name_, vm_host, used))
LOG.tc_step("Attempt to live migrate {} and ensure it's rejected".format(vm_name_))
rc, msg = vm_helper.live_migrate_vm(vm_id=vm_id_, fail_ok=True)
assert 6 == rc, "Expect live migration to fail on vm with pci alias device. Actual: {}".format(msg)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_)
check_helper.check_qat_service(vm_id=vm_id_, qat_devs=expt_qat_devs)
def test_snat_vm_actions(snat_setups, snat):
"""
Test VM external access over VM launch, live-migration, cold-migration, pause/unpause, etc
Args:
snat_setups (tuple): returns vm id and fip. Enable snat, create vm and attach floating ip.
Test Setups (module):
- Find a tenant router that is dvr or non-dvr based on the parameter
- Enable SNAT on tenant router
- boot a vm and attach a floating ip
- Ping vm from NatBox
Test Steps:
- Enable/Disable SNAT based on snat param
- Ping from VM to 8.8.8.8
- wget <lab_fip> to VM
- scp from NatBox to VM
- Live-migrate the VM and verify ping from VM
- Cold-migrate the VM and verify ping from VM
- Pause and un-pause the VM and verify ping from VM
- Suspend and resume the VM and verify ping from VM
- Stop and start the VM and verify ping from VM
- Reboot the VM and verify ping from VM
Test Teardown:
- Enable snat for next test in the same module (function)
- Delete the created vm (module)
- Disable snat (module)
"""
vm_ = snat_setups[0]
snat = True if snat == 'snat_enabled' else False
LOG.tc_step("Update tenant router external gateway to set SNAT to {}".format(snat))
network_helper.set_router_gateway(enable_snat=snat)
# Allow router update to complete, since we've seen cases where ping vm pass but ssh fail
time.sleep(30)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, timeout=60, use_fip=snat)
LOG.tc_step("Ping from VM {} to 8.8.8.8".format(vm_))
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("wget to VM {}".format(vm_))
with vm_helper.ssh_to_vm_from_natbox(vm_id=vm_, use_fip=True) as vm_ssh:
vm_ssh.exec_cmd('wget google.ca', fail_ok=False)
LOG.tc_step("scp from NatBox to VM {}".format(vm_))
vm_fip = network_helper.get_external_ips_for_vms(vms=vm_)[0]
natbox_ssh = NATBoxClient.get_natbox_client()
natbox_ssh.scp_on_source(source_path='test', dest_user='******', dest_ip=vm_fip, dest_path='/tmp/',
dest_password='******', timeout=30)
LOG.tc_step("Live-migrate the VM and verify ping from VM")
vm_helper.live_migrate_vm(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Cold-migrate the VM and verify ping from VM")
vm_helper.cold_migrate_vm(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Pause and un-pause the VM and verify ping from VM")
vm_helper.pause_vm(vm_)
vm_helper.unpause_vm(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Suspend and resume the VM and verify ping from VM")
vm_helper.suspend_vm(vm_)
vm_helper.resume_vm(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Stop and start the VM and verify ping from VM")
vm_helper.stop_vms(vm_)
vm_helper.start_vms(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Reboot the VM and verify ping from VM")
vm_helper.reboot_vm(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Resize the vm to a flavor with 2 dedicated cpus and verify ping from VM")
new_flv = nova_helper.create_flavor(name='ded', vcpus=2)[1]
ResourceCleanup.add('flavor', new_flv, scope='module')
nova_helper.set_flavor(new_flv, **{FlavorSpec.CPU_POLICY: 'dedicated'})
vm_helper.resize_vm(vm_, new_flv)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
def test_dpdk_live_migrate_latency(ovs_dpdk_1_core, launch_instances,
no_simplex, no_duplex):
con_ssh = ssh.ControllerClient.get_active_controller()
prev_st = None
prev_et = None
res = list()
for i in range(20):
LOG.tc_step("Start of iter {}".format(i))
vm_host = vm_helper.get_vm_host(launch_instances)
cmd_get_pod_name = ("kubectl get pods -n openstack | "
"grep --color=never nova-compute-{} | "
"awk '{{print$1}}'".format(vm_host))
pod_name = con_ssh.exec_cmd(cmd=cmd_get_pod_name)[1].rstrip().lstrip()
cmd_get_start_date = (
"kubectl -n openstack logs {} -c nova-compute | "
"grep --color=never 'instance: {}' | "
"grep --color=never 'pre_live_migration on destination host' | "
"tail -1 | "
"awk '{{print $1 \" \" $2}}'".format(pod_name, launch_instances))
cmd_get_end_date = (
"kubectl -n openstack logs {} -c nova-compute | "
"grep --color=never 'instance: {}' | "
"egrep --color=never "
"'Migrating instance to [a-zA-Z]+-[0-9] finished successfully' | "
"tail -1 | "
"awk '{{print $1 \" \" $2}}'".format(pod_name, launch_instances))
vm_helper.live_migrate_vm(vm_id=launch_instances)
st = con_ssh.exec_cmd(cmd=cmd_get_start_date)[1]
et = con_ssh.exec_cmd(cmd=cmd_get_end_date)[1]
st_date = datetime.datetime.strptime(st, '%Y-%m-%d %H:%M:%S.%f')
et_date = datetime.datetime.strptime(et, '%Y-%m-%d %H:%M:%S.%f')
if i == 0:
prev_st = st_date
prev_et = et_date
elif i > 0:
if st_date <= prev_st or et_date <= prev_et:
msg = ("new start time {} is less "
"or equal than old start time {}\n"
"or new end time {} is less "
"or equal than old end time "
"{}".format(st_date, prev_st, et_date, prev_et))
LOG.error(msg)
raise Exception(msg)
else:
prev_st = st_date
prev_et = et_date
diff = et_date - st_date
LOG.info("\nstart time = {}\nend time = {}".format(st, et))
LOG.info("\ndiff = {}".format(diff))
res.append(diff)
def calc_avg(lst):
rtrn_sum = datetime.timedelta()
for i in lst:
LOG.info("Iter {}: {}".format(lst.index(i), i))
rtrn_sum += i
return rtrn_sum / len(lst)
final_res = calc_avg(res)
LOG.info("Avg time is : {}".format(final_res))
def _prepare_test(vm1, vm2, get_hosts, with_router):
"""
VMs:
VM1: under test (primary tenant)
VM2: traffic observer
"""
vm1_host = vm_helper.get_vm_host(vm1)
vm2_host = vm_helper.get_vm_host(vm2)
vm1_router = network_helper.get_tenant_router(
auth_info=Tenant.get_primary())
vm2_router = network_helper.get_tenant_router(
auth_info=Tenant.get_secondary())
vm1_router_host = network_helper.get_router_host(router=vm1_router)
vm2_router_host = network_helper.get_router_host(router=vm2_router)
targets = list(get_hosts)
if vm1_router_host == vm2_router_host:
end_time = time.time() + 360
while time.time() < end_time:
vm1_router_host = network_helper.get_router_host(
router=vm1_router)
vm2_router_host = network_helper.get_router_host(
router=vm2_router)
if vm1_router_host != vm2_router_host:
break
else:
assert vm1_router_host != vm2_router_host, "two routers are located on the same compute host"
if not with_router:
"""
Setup:
VM1 on COMPUTE-A
VM2 not on COMPUTE-A
ROUTER1 on COMPUTE-B
ROUTER2 on COMPUTE-C
"""
if len(get_hosts) < 3:
skip(
"Lab not suitable for without_router, requires at least three hypervisors"
)
LOG.tc_step(
"Ensure VM2, ROUTER2 not on COMPUTE-A, for simplicity, ensure they are on the same compute"
)
if vm2_host != vm2_router_host:
vm_helper.live_migrate_vm(vm_id=vm2,
destination_host=vm2_router_host)
vm2_host = vm_helper.get_vm_host(vm2)
assert vm2_host == vm2_router_host, "live-migration failed"
host_observer = vm2_host
LOG.tc_step(
"Ensure VM1 and (ROUTER1, VM2, ROUTER2) are on different hosts"
)
if vm1_router_host in targets:
# ensure vm1_router_host is not selected for vm1
# vm1_router_host can be backed by any type of storage
targets.remove(vm1_router_host)
if vm2_host in targets:
targets.remove(vm2_host)
if vm1_host in targets:
host_src_evacuation = vm1_host
else:
assert targets, "no suitable compute for vm1, after excluding ROUTER1, VM2, ROUTER2 's hosts"
host_src_evacuation = targets[0]
vm_helper.live_migrate_vm(vm_id=vm1,
destination_host=host_src_evacuation)
vm1_host = vm_helper.get_vm_host(vm1)
assert vm1_host == host_src_evacuation, "live-migration failed"
# verify setup
vm1_host = vm_helper.get_vm_host(vm1)
vm2_host = vm_helper.get_vm_host(vm2)
vm1_router_host = network_helper.get_router_host(router=vm1_router)
vm2_router_host = network_helper.get_router_host(router=vm2_router)
assert vm1_router_host != vm1_host and vm2_host != vm1_host and vm2_router_host != vm1_host, \
"setup is incorrect"
else:
"""
Setup:
VM1, ROUTER1 on COMPUTE-A
VM2 not on COMPUTE-A
ROUTER2 on COMPUTE-B
"""
LOG.tc_step("Ensure VM1, ROUTER1 on COMPUTE-A")
# VM1 must be sitting on ROUTER1's host, thus vm1_router_host must be backed by local_image
assert vm1_router_host in targets, "vm1_router_host is not backed by local_image"
if vm1_host != vm1_router_host:
vm_helper.live_migrate_vm(vm_id=vm1,
destination_host=vm1_router_host)
vm1_host = vm_helper.get_vm_host(vm1)
assert vm1_host == vm1_router_host, "live-migration failed"
host_src_evacuation = vm1_host
LOG.tc_step(
"Ensure VM2, ROUTER2 not on COMPUTE-A, for simplicity, ensure they are on the same compute"
)
targets.remove(host_src_evacuation)
if vm2_host in targets:
host_observer = vm2_host
else:
assert targets, "no suitable compute for vm2, after excluding COMPUTE-A"
host_observer = targets[0]
vm_helper.live_migrate_vm(vm_id=vm2,
destination_host=host_observer)
vm2_host = vm_helper.get_vm_host(vm2)
assert vm2_host == host_observer, "live-migration failed"
# verify setup
vm1_host = vm_helper.get_vm_host(vm1)
vm2_host = vm_helper.get_vm_host(vm2)
vm1_router_host = network_helper.get_router_host(router=vm1_router)
vm2_router_host = network_helper.get_router_host(router=vm2_router)
assert vm1_host == vm1_router_host and vm2_host != vm1_host and vm2_router_host != vm1_host, \
"setup is incorrect"
assert vm1_host == host_src_evacuation and vm2_host == host_observer, "setup is incorrect"
LOG.info("Evacuate: VM {} on {}, ROUTER on {}".format(
vm1, vm1_host, vm1_router_host))
LOG.info("Observer: VM {} on {}, ROUTER on {}".format(
vm2, vm2_host, vm2_router_host))
return host_src_evacuation, host_observer
def operation_live(vm_id_):
code, msg = vm_helper.live_migrate_vm(vm_id=vm_id_)
assert 0 == code, msg
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm_id_)
# kernel routing
vm_helper.ping_between_routed_vms(vm_id, vm_observer, vshell=False)
def test_vm_vcpu_model(vcpu_model, vcpu_source, boot_source,
cpu_models_supported):
"""
Test vcpu model specified in flavor will be applied to vm. In case host does not support specified vcpu model,
proper error message should be displayed in nova show.
Args:
vcpu_model
vcpu_source
boot_source
Test Steps:
- Set flavor extra spec or image metadata with given vcpu model.
- Boot a vm from volume/image
- Stop and then start vm and ensure that it retains its cpu model
- If vcpu model is supported by host,
- Check vcpu model specified in flavor/image is used by vm via virsh, ps aux (and /proc/cpuinfo)
- Live migrate vm and check vcpu model again
- Cold migrate vm and check vcpu model again
- If vcpu model is not supported by host, check proper error message is included if host does not
support specified vcpu model.
Teardown:
- Delete created vm, volume, image, flavor
"""
cpu_models_multi_host, all_cpu_models_supported = cpu_models_supported
flv_model = vcpu_model if vcpu_source == 'flavor' else None
img_model = vcpu_model if vcpu_source == 'image' else None
code, vm, msg = _boot_vm_vcpu_model(flv_model=flv_model,
img_model=img_model,
boot_source=boot_source)
is_supported = (not vcpu_model) or (vcpu_model == 'Passthrough') or (
vcpu_model in all_cpu_models_supported)
if not is_supported:
LOG.tc_step(
"Check vm in error state due to vcpu model unsupported by hosts.")
assert 1 == code, "boot vm cli exit code is not 1. Actual fail reason: {}".format(
msg)
expt_fault = VCPUSchedulerErr.CPU_MODEL_UNAVAIL
res_bool, vals = vm_helper.wait_for_vm_values(vm,
10,
regex=True,
strict=False,
status='ERROR')
err = vm_helper.get_vm_fault_message(vm)
assert res_bool, "VM did not reach expected error state. Actual: {}".format(
vals)
assert re.search(expt_fault, err), "Incorrect fault reported. Expected: {} Actual: {}" \
.format(expt_fault, err)
return
# System supports specified vcpu, continue to verify
expt_arch = None
if vcpu_model == 'Passthrough':
host = vm_helper.get_vm_host(vm)
expt_arch = host_helper.get_host_cpu_model(host)
LOG.tc_step("Check vm is launched with expected vcpu model")
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm)
check_vm_cpu_model(vm_id=vm, vcpu_model=vcpu_model, expt_arch=expt_arch)
multi_hosts_supported = (not vcpu_model) or (vcpu_model in cpu_models_multi_host) or \
(vcpu_model == 'Passthrough' and cpu_models_multi_host)
# TC5141
LOG.tc_step(
"Stop and then restart vm and check if it retains its vcpu model")
vm_helper.stop_vms(vm)
vm_helper.start_vms(vm)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm)
check_vm_cpu_model(vm_id=vm, vcpu_model=vcpu_model, expt_arch=expt_arch)
if not multi_hosts_supported:
LOG.info(
"Skip migration steps. Less than two hosts in same storage aggregate support {}"
.format(vcpu_model))
return
LOG.tc_step(
"Live (block) migrate vm and check {} vcpu model".format(vcpu_model))
vm_helper.live_migrate_vm(vm_id=vm)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm)
check_vm_cpu_model(vm, vcpu_model, expt_arch=expt_arch)
LOG.tc_step("Cold migrate vm and check {} vcpu model".format(vcpu_model))
vm_helper.cold_migrate_vm(vm_id=vm)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id=vm)
check_vm_cpu_model(vm, vcpu_model, expt_arch=expt_arch)
def test_vm_with_a_large_volume_live_migrate(vms_, pre_alarm_):
"""
Test instantiate a vm with a large volume ( 20 GB and 40 GB) and live
migrate:
Args:
vms_ (dict): vms created by vms_ fixture
pre_alarm_ (list): alarm lists obtained by pre_alarm_ fixture
Test Setups:
- get tenant1 and management networks which are already created for lab
setup
- get or create a "small" flavor
- get the guest image id
- create two large volumes (20 GB and 40 GB) in cinder
- boot two vms ( test_inst1, test_inst2) using volumes 20 GB and 40 GB
respectively
Test Steps:
- Verify VM status is ACTIVE
- Validate that VMs boot, and that no timeouts or error status occur.
- Verify the VM can be pinged from NATBOX
- Verify login to VM and rootfs (dev/vda) filesystem is rw mode
- Attempt to live migrate of VMs
- Validate that the VMs migrated and no errors or alarms are present
- Log into both VMs and validate that file systems are read-write
- Terminate VMs
Skip conditions:
- less than two computes
- no storage node
"""
for vm in vms_:
vm_id = vm['id']
LOG.tc_step(
"Checking VM status; VM Instance id is: {}......".format(vm_id))
vm_state = vm_helper.get_vm_status(vm_id)
assert vm_state == VMStatus.ACTIVE, 'VM {} state is {}; Not in ' \
'ACTIVATE state as expected' \
.format(vm_id, vm_state)
LOG.tc_step("Verify VM can be pinged from NAT box...")
rc, boot_time = check_vm_boot_time(vm_id)
assert rc, "VM is not pingable after {} seconds ".format(boot_time)
LOG.tc_step("Verify Login to VM and check filesystem is rw mode....")
assert is_vm_filesystem_rw(
vm_id), 'rootfs filesystem is not RW as expected for VM {}' \
.format(vm['display_name'])
LOG.tc_step(
"Attempting live migration; vm id = {}; vm_name = {} ....".format(
vm_id, vm['display_name']))
code, msg = vm_helper.live_migrate_vm(vm_id=vm_id, fail_ok=False)
LOG.tc_step("Verify live migration succeeded...")
assert code == 0, "Expected return code 0. Actual return code: {}; " \
"details: {}".format(code, msg)
LOG.tc_step(
"Verifying filesystem is rw mode after live migration....")
assert is_vm_filesystem_rw(
vm_id), 'After live migration rootfs filesystem is not RW as ' \
'expected for VM {}'. \
format(vm['display_name'])
def test_vm_with_large_volume_and_evacuation(vms_, pre_alarm_):
"""
Test instantiate a vm with a large volume ( 20 GB and 40 GB) and evacuate:
Args:
vms_ (dict): vms created by vms_ fixture
pre_alarm_ (list): alarm lists obtained by pre_alarm_ fixture
Test Setups:
- get tenant1 and management networks which are already created for lab
setup
- get or create a "small" flavor
- get the guest image id
- create two large volumes (20 GB and 40 GB) in cinder
- boot two vms ( test_inst1, test_inst2) using volumes 20 GB and 40 GB
respectively
Test Steps:
- Verify VM status is ACTIVE
- Validate that VMs boot, and that no timeouts or error status occur.
- Verify the VM can be pinged from NATBOX
- Verify login to VM and rootfs (dev/vda) filesystem is rw mode
- live migrate, if required, to bring both VMs to the same compute
- Validate migrated VM and no errors or alarms are present
- Reboot compute host to initiate evacuation
- Verify VMs are evacuated
- Check for any system alarms
- Verify login to VM and rootfs (dev/vda) filesystem is still rw mode
after evacuation
- Terminate VMs
Skip conditions:
- less that two computes
- no storage node
"""
vm_ids = []
for vm in vms_:
vm_id = vm['id']
vm_ids.append(vm_id)
LOG.tc_step(
"Checking VM status; VM Instance id is: {}......".format(vm_id))
vm_state = vm_helper.get_vm_status(vm_id)
assert vm_state == VMStatus.ACTIVE, 'VM {} state is {}; Not in ' \
'ACTIVATE state as expected' \
.format(vm_id, vm_state)
LOG.tc_step("Verify VM can be pinged from NAT box...")
rc, boot_time = check_vm_boot_time(vm_id)
assert rc, "VM is not pingable after {} seconds ".format(boot_time)
LOG.tc_step("Verify Login to VM and check filesystem is rw mode....")
assert is_vm_filesystem_rw(
vm_id), 'rootfs filesystem is not RW as expected for VM {}' \
.format(vm['display_name'])
LOG.tc_step(
"Checking if live migration is required to put the vms to a single "
"compute....")
host_0 = vm_helper.get_vm_host(vm_ids[0])
host_1 = vm_helper.get_vm_host(vm_ids[1])
if host_0 != host_1:
LOG.tc_step("Attempting to live migrate vm {} to host {} ....".format(
(vms_[1])['display_name'], host_0))
code, msg = vm_helper.live_migrate_vm(vm_ids[1],
destination_host=host_0)
LOG.tc_step("Verify live migration succeeded...")
assert code == 0, "Live migration of vm {} to host {} did not " \
"success".format((vms_[1])['display_name'], host_0)
LOG.tc_step("Verify both VMs are in same host....")
assert host_0 == vm_helper.get_vm_host(
vm_ids[1]), "VMs are not in the same compute host"
LOG.tc_step(
"Rebooting compute {} to initiate vm evacuation .....".format(host_0))
vm_helper.evacuate_vms(host=host_0, vms_to_check=vm_ids, ping_vms=True)
LOG.tc_step("Login to VM and to check filesystem is rw mode....")
assert is_vm_filesystem_rw((vms_[0])[
'id']), 'After evacuation the rootfs ' \
'filesystem is not RW as expected ' \
'for VM {}'.format(
(vms_[0])['display_name'])
LOG.tc_step("Login to VM and to check filesystem is rw mode....")
assert is_vm_filesystem_rw((vms_[1])['id']), \
'After evacuation the rootfs filesystem is not RW as expected ' \
'for VM {}'.format((vms_[1])['display_name'])
def test_cpu_pol_vm_actions(flv_vcpus, cpu_pol, pol_source, boot_source):
LOG.tc_step("Create flavor with {} vcpus".format(flv_vcpus))
flavor_id = nova_helper.create_flavor(name='cpu_pol', vcpus=flv_vcpus)[1]
ResourceCleanup.add('flavor', flavor_id)
image_id = glance_helper.get_image_id_from_name(
GuestImages.DEFAULT['guest'], strict=True)
if cpu_pol is not None:
if pol_source == 'flavor':
specs = {FlavorSpec.CPU_POLICY: cpu_pol}
LOG.tc_step("Set following extra specs: {}".format(specs))
nova_helper.set_flavor(flavor_id, **specs)
else:
image_meta = {ImageMetadata.CPU_POLICY: cpu_pol}
LOG.tc_step(
"Create image with following metadata: {}".format(image_meta))
image_id = glance_helper.create_image(
name='cpu_pol_{}'.format(cpu_pol),
cleanup='function',
**image_meta)[1]
if boot_source == 'volume':
LOG.tc_step("Create a volume from image")
source_id = cinder_helper.create_volume(name='cpu_pol'.format(cpu_pol),
source_id=image_id)[1]
ResourceCleanup.add('volume', source_id)
else:
source_id = image_id
prev_cpus = host_helper.get_vcpus_for_computes(field='used_now')
LOG.tc_step(
"Boot a vm from {} with above flavor and check vm topology is as "
"expected".format(boot_source))
vm_id = vm_helper.boot_vm(name='cpu_pol_{}_{}'.format(cpu_pol, flv_vcpus),
flavor=flavor_id,
source=boot_source,
source_id=source_id,
cleanup='function')[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
vm_host = vm_helper.get_vm_host(vm_id)
check_helper.check_topology_of_vm(vm_id,
vcpus=flv_vcpus,
cpu_pol=cpu_pol,
vm_host=vm_host,
prev_total_cpus=prev_cpus[vm_host])
LOG.tc_step("Suspend/Resume vm and check vm topology stays the same")
vm_helper.suspend_vm(vm_id)
vm_helper.resume_vm(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
check_helper.check_topology_of_vm(vm_id,
vcpus=flv_vcpus,
cpu_pol=cpu_pol,
vm_host=vm_host,
prev_total_cpus=prev_cpus[vm_host])
LOG.tc_step("Stop/Start vm and check vm topology stays the same")
vm_helper.stop_vms(vm_id)
vm_helper.start_vms(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
prev_siblings = check_helper.check_topology_of_vm(
vm_id,
vcpus=flv_vcpus,
cpu_pol=cpu_pol,
vm_host=vm_host,
prev_total_cpus=prev_cpus[vm_host])[1]
LOG.tc_step("Live migrate vm and check vm topology stays the same")
vm_helper.live_migrate_vm(vm_id=vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
vm_host = vm_helper.get_vm_host(vm_id)
prev_siblings = prev_siblings if cpu_pol == 'dedicated' else None
check_helper.check_topology_of_vm(vm_id,
vcpus=flv_vcpus,
cpu_pol=cpu_pol,
vm_host=vm_host,
prev_total_cpus=prev_cpus[vm_host],
prev_siblings=prev_siblings)
LOG.tc_step("Cold migrate vm and check vm topology stays the same")
vm_helper.cold_migrate_vm(vm_id=vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
vm_host = vm_helper.get_vm_host(vm_id)
check_helper.check_topology_of_vm(vm_id,
vcpus=flv_vcpus,
cpu_pol=cpu_pol,
vm_host=vm_host,
prev_total_cpus=prev_cpus[vm_host])
def test_instantiate_a_vm_with_multiple_volumes_and_migrate():
"""
Test a vm with a multiple volumes live, cold migration and evacuation:
Test Setups:
- get guest image_id
- get or create 'small' flavor_id
- get tenenat and managment network ids
Test Steps:
- create volume for boot and another extra size 8GB
- boot vms from the created volume
- Validate that VMs boot, and that no timeouts or error status occur.
- Verify VM status is ACTIVE
- Attach the second volume to VM
- Attempt live migrate VM
- Login to VM and verify the filesystem is rw mode on both volumes
- Attempt cold migrate VM
- Login to VM and verify the filesystem is rw mode on both volumes
- Reboot the compute host to initiate evacuation
- Login to VM and verify the filesystem is rw mode on both volumes
- Terminate VMs
Skip conditions:
- less than two computes
- less than one storage
"""
# skip("Currently not working. Centos image doesn't see both volumes")
LOG.tc_step("Creating a volume size=8GB.....")
vol_id_0 = cinder_helper.create_volume(size=8)[1]
ResourceCleanup.add('volume', vol_id_0, scope='function')
LOG.tc_step("Creating a second volume size=8GB.....")
vol_id_1 = cinder_helper.create_volume(size=8, bootable=False)[1]
LOG.tc_step("Volume id is: {}".format(vol_id_1))
ResourceCleanup.add('volume', vol_id_1, scope='function')
LOG.tc_step("Booting instance vm_0...")
vm_id = vm_helper.boot_vm(name='vm_0',
source='volume',
source_id=vol_id_0,
cleanup='function')[1]
time.sleep(5)
LOG.tc_step("Verify VM can be pinged from NAT box...")
rc, boot_time = check_vm_boot_time(vm_id)
assert rc, "VM is not pingable after {} seconds ".format(boot_time)
LOG.tc_step("Login to VM and to check filesystem is rw mode....")
assert is_vm_filesystem_rw(
vm_id), 'vol_0 rootfs filesystem is not RW as expected.'
LOG.tc_step("Attemping to attach a second volume to VM...")
vm_helper.attach_vol_to_vm(vm_id, vol_id_1)
LOG.tc_step(
"Login to VM and to check filesystem is rw mode for both volumes....")
assert is_vm_filesystem_rw(vm_id, rootfs=['vda',
'vdb']), 'volumes rootfs ' \
'filesystem is not RW ' \
'as expected.'
LOG.tc_step("Attemping live migrate VM...")
vm_helper.live_migrate_vm(vm_id=vm_id)
LOG.tc_step("Login to VM and to check filesystem is rw mode after live "
"migration....")
assert is_vm_filesystem_rw(vm_id, rootfs=['vda',
'vdb']), 'After live migration ' \
'rootfs filesystem is ' \
'not RW'
LOG.tc_step("Attempting cold migrate VM...")
vm_helper.cold_migrate_vm(vm_id)
LOG.tc_step("Login to VM and to check filesystem is rw mode after live "
"migration....")
assert is_vm_filesystem_rw(vm_id, rootfs=['vda',
'vdb']), 'After cold migration ' \
'rootfs filesystem is ' \
'not RW'
LOG.tc_step("Testing VM evacuation.....")
before_host_0 = vm_helper.get_vm_host(vm_id)
LOG.tc_step("Rebooting compute {} to initiate vm evacuation .....".format(
before_host_0))
vm_helper.evacuate_vms(host=before_host_0,
vms_to_check=vm_id,
ping_vms=True)
LOG.tc_step("Login to VM and to check filesystem is rw mode after live "
"migration....")
assert is_vm_filesystem_rw(vm_id, rootfs=['vda',
'vdb']), 'After evacuation ' \
'filesystem is not RW'
def test_live_migrate_v1(vm_boot_type, vm_storage, vm_interface, block_migrate,
specify_host):
"""
Live migrate VM with:
various vm storage type,
various vm interface types,
with/without block migration,
with/without specify host when sending live-migration request
Expected results can be successful or rejected depending on the VM storage details and hosts storage backing.
Args:
vm_boot_type (str): e.g, image, volume
vm_storage (str): VM storage. e.g., local_image, local_volume, remote
vm_interface (str): VM interface, e.g., virio, avp
block_migrate (bool): Whether to live-migrate with block migration
specify_host (bool): Whether to specify host in live-migration request
=====
Prerequisites (requirement for the system):
- system is preconfigured to test scenario under test
Skip conditions:
- at least two hypervisor hosts on the system
Test Steps:
- Find/boot a VM that satisfy the given criteria
- Find a suitable host to migrate to if specify_host is True
- Attempt to live-migrate
- Verify the results based on the return code
"""
# Make skip decision based on the value(s) of test param(s) and the system condition
if vm_boot_type == 'image' and vm_tenant2_image_unavailable():
skip("VM named 'tenant2-image' doesn't exist on the system")
# Mark test start
# LOG.tc_start()
# Mark test steps when applicable
LOG.tc_step("Boot vm if not already booted.")
if vm_boot_type == 'image':
vm_name = 'tenant2-image'
vm_id = vm_helper.get_vm_id_from_name(vm_name)
else:
# boot from volume using launch script from lab_setup
vm_id = vm_helper.launch_vms_via_script(vm_type=vm_interface,
num_vms=1)[0]
dest_host = ''
if specify_host:
# This step only applicable when test param specify_host=True
LOG.tc_step("Getting specific destination host")
dest_host = vm_helper.get_dest_host_for_live_migrate(vm_id)
# Another test step
LOG.tc_step("Attempt to live migrate VM")
return_code, message = vm_helper.live_migrate_vm(
vm_id,
fail_ok=True,
block_migrate=block_migrate,
destination_host=dest_host)
# Verify test results using assert
LOG.tc_step("Verify test result")
assert return_code in [0, 2], message
def test_migrate_vm(check_system, guest_os, mig_type, cpu_pol):
"""
Test migrate vms for given guest type
Args:
check_system:
guest_os:
mig_type:
cpu_pol:
Test Steps:
- Create a glance image from given guest type
- Create a vm from cinder volume using above image
- Live/cold migrate the vm
- Ensure vm moved to other host and in good state (active and
reachabe from NatBox)
"""
LOG.tc_step("Create a flavor with 1 vcpu")
flavor_id = \
nova_helper.create_flavor(name='{}-mig'.format(mig_type), vcpus=1,
root_disk=9, cleanup='function')[1]
if cpu_pol is not None:
specs = {FlavorSpec.CPU_POLICY: cpu_pol}
LOG.tc_step("Add following extra specs: {}".format(specs))
nova_helper.set_flavor(flavor=flavor_id, **specs)
LOG.tc_step("Create a volume from {} image".format(guest_os))
image_id = glance_helper.get_guest_image(guest_os=guest_os)
vol_id = cinder_helper.create_volume(source_id=image_id, size=9,
guest_image=guest_os)[1]
ResourceCleanup.add('volume', vol_id)
LOG.tc_step("Boot a vm from above flavor and volume")
vm_id = vm_helper.boot_vm(guest_os, flavor=flavor_id, source='volume',
source_id=vol_id, cleanup='function')[1]
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
if guest_os == 'ubuntu_14':
system_helper.wait_for_alarm_gone(alarm_id=EventLogID.CINDER_IO_CONGEST,
entity_id='cinder_io_monitor',
strict=False, timeout=300,
fail_ok=False)
LOG.tc_step("{} migrate vm and check vm is moved to different host".format(
mig_type))
prev_vm_host = vm_helper.get_vm_host(vm_id)
if mig_type == 'live':
code, output = vm_helper.live_migrate_vm(vm_id)
if code == 1:
assert False, "No host to live migrate to. System may not be in " \
"good state."
else:
vm_helper.cold_migrate_vm(vm_id)
vm_host = vm_helper.get_vm_host(vm_id)
assert prev_vm_host != vm_host, "vm host did not change after {} " \
"migration".format(mig_type)
LOG.tc_step("Ping vm from NatBox after {} migration".format(mig_type))
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
def test_live_migrate_vm_negative(storage_backing, ephemeral, swap, vm_type,
block_mig, expt_err,
hosts_per_stor_backing, no_simplex):
"""
Skip Condition:
- Less than two hosts have specified storage backing
Test Steps:
- create flavor with specified vcpus, cpu_policy, ephemeral, swap,
storage_backing
- boot vm from specified boot source with above flavor
- (attach volume to vm if 'image_with_vol', specified in vm_type)
- Live migrate the vm with specified block_migration flag
- Verify VM is successfully live migrated to different host
Teardown:
- Delete created vm, volume, flavor
"""
if len(hosts_per_stor_backing.get(storage_backing, [])) < 2:
skip("Less than two hosts have {} storage backing".format(
storage_backing))
vm_id = _boot_vm_under_test(storage_backing, ephemeral, swap, None, 1,
vm_type)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
prev_vm_host = vm_helper.get_vm_host(vm_id)
vm_disks = vm_helper.get_vm_devices_via_virsh(vm_id)
file_paths, content = touch_files_under_vm_disks(vm_id=vm_id,
ephemeral=ephemeral,
swap=swap, vm_type=vm_type,
disks=vm_disks)
LOG.tc_step(
"Live migrate VM and ensure it's rejected with proper error message")
# block_mig = True if boot_source == 'image' else False
code, output = vm_helper.live_migrate_vm(vm_id, block_migrate=block_mig)
assert 2 == code, "Expect live migration to have expected fail. Actual: " \
"{}".format(output)
# Remove below code due to live-migration is async in newton
# assert 'Unexpected API Error'.lower() not in output.lower(),
# "'Unexpected API Error' returned."
#
# # remove extra spaces in error message
# output = re.sub(r'\s\s+', " ", output)
# assert eval(expt_err) in output, "Expected error message {} is not in
# actual error message: {}".\
# format(eval(expt_err), output)
post_vm_host = vm_helper.get_vm_host(vm_id)
assert prev_vm_host == post_vm_host, "VM host changed even though live " \
"migration request rejected."
LOG.tc_step(
"Ensure vm is pingable from NatBox after live migration rejected")
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
LOG.tc_step("Check files after live migrate attempt")
check_helper.check_vm_files(vm_id=vm_id, storage_backing=storage_backing,
ephemeral=ephemeral, swap=swap,
vm_type=vm_type, vm_action='live_migrate',
file_paths=file_paths, content=content,
disks=vm_disks, prev_host=prev_vm_host,
post_host=post_vm_host)
def test_vm_with_max_vnics_attached_during_boot(base_vm, guest_os, nic_arg,
boot_source):
"""
Setups:
- Boot a base vm with mgmt net and tenant_port_id (module)
Test Steps:
- Boot a vm with 1 mgmt and 15 avp/virtio Interfaces
- Perform nova action (live migrate --force, live migrate, rebuild, reboot hard/soft, resize revert, resize)
- ping between base_vm and vm_under_test over mgmt & tenant network
Teardown:
- Delete created vm, volume, port (if any) (func)
- Delete base vm, volume (module)
"""
base_vm_id, mgmt_nic, tenant_nic, internal_net_id, tenant_net_id, mgmt_net_id = base_vm
vif_type = 'avp' if system_helper.is_avs() else None
LOG.tc_step("Get/Create {} glance image".format(guest_os))
cleanup = None if re.search(GuestImages.TIS_GUEST_PATTERN,
guest_os) else 'function'
image_id = glance_helper.get_guest_image(guest_os=guest_os,
cleanup=cleanup)
# TODO Update vif model config. Right now vif model avp still under implementation
nics = [mgmt_nic]
for i in range(15):
if nic_arg == 'port_id':
port_id = network_helper.create_port(tenant_net_id,
'tenant_port-{}'.format(i),
wrs_vif=vif_type,
cleanup='function')[1]
nics.append({'port-id': port_id})
else:
nics.append({'net-id': tenant_net_id, 'vif-model': vif_type})
LOG.tc_step(
"Boot a {} vm and flavor from {} with 1 mgmt and 15 data interfaces".
format(guest_os, boot_source))
vm_under_test = vm_helper.boot_vm('max_vifs-{}-{}'.format(
guest_os, boot_source),
nics=nics,
source=boot_source,
image_id=image_id,
guest_os=guest_os,
cleanup='function')[1]
vm_ports_count = len(network_helper.get_ports(server=vm_under_test))
expt_vnics = 16
LOG.info("vnics attached to VM: {}".format(vm_ports_count))
assert vm_ports_count == expt_vnics, "vnics attached is not equal to max number."
_ping_vm_data(vm_under_test, vm_under_test, action='boot')
vm_helper.configure_vm_vifs_on_same_net(vm_id=vm_under_test)
_ping_vm_data(vm_under_test, base_vm_id, action='configure routes')
destination_host = vm_helper.get_dest_host_for_live_migrate(
vm_id=vm_under_test)
if destination_host:
# LOG.tc_step("Perform following action(s) on vm {}: {}".format(vm_under_test, 'live-migrate --force'))
# vm_helper.live_migrate_vm(vm_id=vm_under_test, destination_host=destination_host, force=True)
# _ping_vm_data(vm_under_test, base_vm_id, action='live migrate --force')
LOG.tc_step("Perform following action(s) on vm {}: {}".format(
vm_under_test, 'live-migrate'))
vm_helper.live_migrate_vm(vm_id=vm_under_test)
_ping_vm_data(vm_under_test, base_vm_id, action='live-migrate')
LOG.tc_step("Perform following action(s) on vm {}: {}".format(
vm_under_test, 'hard reboot'))
vm_helper.reboot_vm(vm_id=vm_under_test, hard=True)
_ping_vm_data(vm_under_test, base_vm_id, action='hard reboot')
LOG.tc_step("Perform following action(s) on vm {}: {}".format(
vm_under_test, 'soft reboot'))
vm_helper.reboot_vm(vm_id=vm_under_test)
_ping_vm_data(vm_under_test, base_vm_id, action='soft rebuild')
LOG.tc_step('Create destination flavor')
dest_flavor_id = nova_helper.create_flavor(name='dest_flavor',
vcpus=2,
guest_os=guest_os)[1]
LOG.tc_step('Resize vm to dest flavor and revert')
vm_helper.resize_vm(vm_under_test,
dest_flavor_id,
revert=True,
fail_ok=False)
_ping_vm_data(vm_under_test, base_vm_id, action='resize revert')
LOG.tc_step('Resize vm to dest flavor and revert False')
vm_helper.resize_vm(vm_under_test, dest_flavor_id, fail_ok=False)
vm_helper.configure_vm_vifs_on_same_net(vm_id=vm_under_test)
_ping_vm_data(vm_under_test, base_vm_id, action='resize')
LOG.tc_step("Perform following action(s) on vm {}: {}".format(
vm_under_test, 'rebuild'))
vm_helper.rebuild_vm(vm_id=vm_under_test)
_ping_vm_data(vm_under_test, vm_under_test, action='rebuild')
vm_helper.configure_vm_vifs_on_same_net(vm_id=vm_under_test)
_ping_vm_data(vm_under_test, base_vm_id, action='rebuild')
def _test_basic_swift_provisioning(pool_size, pre_swift_check):
"""
Verifies basic swift provisioning works as expected
Args:
pool_size:
pre_swift_check:
Returns:
"""
ceph_backend_info = get_ceph_backend_info()
if pool_size == 'default' and pre_swift_check[0]:
skip("Swift is already provisioned")
if pool_size == 'fixed_size' and pre_swift_check[0]:
skip("Swift is already provisioned and set to non-default pool value")
object_pool_gib = None
cinder_pool_gib = ceph_backend_info['cinder_pool_gib']
if pool_size == 'default':
if not ceph_backend_info['object_gateway']:
LOG.tc_step("Enabling SWIFT object store .....")
else:
if not ceph_backend_info['object_gateway']:
skip("Swift is not provisioned")
total_gib = ceph_backend_info['ceph_total_space_gib']
unallocated_gib = (total_gib - cinder_pool_gib -
ceph_backend_info['glance_pool_gib'] -
ceph_backend_info['ephemeral_pool_gib'])
if unallocated_gib == 0:
unallocated_gib = int(int(cinder_pool_gib) / 4)
cinder_pool_gib = str(int(cinder_pool_gib) - unallocated_gib)
elif unallocated_gib < 0:
skip("Unallocated gib < 0. System is in unknown state.")
object_pool_gib = str(unallocated_gib)
LOG.tc_step(
"Enabling SWIFT object store and setting object pool size to {}....."
.format(object_pool_gib))
rc, updated_backend_info = storage_helper.modify_storage_backend(
'ceph',
object_gateway=False,
cinder=cinder_pool_gib,
object_gib=object_pool_gib,
services='cinder,glance,nova,swift')
LOG.info("Verifying if swift object gateway is enabled...")
assert str(updated_backend_info['object_gateway']).lower() == 'true', "Fail to enable Swift object gateway: {}"\
.format(updated_backend_info)
LOG.info("Swift object gateway is enabled.")
LOG.info("Verifying ceph task ...")
state = storage_helper.get_storage_backends(backend='ceph',
field='state')[0]
if system_helper.wait_for_alarm(alarm_id=EventLogID.CONFIG_OUT_OF_DATE,
timeout=10,
fail_ok=True,
entity_id='controller-')[0]:
LOG.info("Verifying ceph task is set to 'add-object-gateway'...")
assert BackendState.CONFIGURING == state, \
"Unexpected ceph state '{}' after swift object gateway update ".format(state)
LOG.info("Lock/Unlock controllers...")
active_controller, standby_controller = system_helper.get_active_standby_controllers(
)
LOG.info("Active Controller is {}; Standby Controller is {}...".format(
active_controller, standby_controller))
for controller in [standby_controller, active_controller]:
if not controller:
continue
HostsToRecover.add(controller)
host_helper.lock_host(controller, swact=True)
storage_helper.wait_for_storage_backend_vals(
backend='ceph-store',
**{
'task': BackendTask.RECONFIG_CONTROLLER,
'state': BackendState.CONFIGURING
})
host_helper.unlock_host(controller)
system_helper.wait_for_alarm_gone(
alarm_id=EventLogID.CONFIG_OUT_OF_DATE, fail_ok=False)
else:
assert BackendState.CONFIGURED == state, \
"Unexpected ceph state '{}' after swift object gateway update ".format(state)
LOG.info("Verifying Swift provisioning setups...")
assert verify_swift_object_setup(), "Failure in swift setups"
for i in range(3):
vm_name = 'vm_swift_api_{}'.format(i)
LOG.tc_step(
"Boot vm {} and perform nova actions on it".format(vm_name))
vm_id = vm_helper.boot_vm(name=vm_name, cleanup='function')[1]
vm_helper.wait_for_vm_pingable_from_natbox(
vm_id, timeout=VMTimeout.DHCP_RETRY)
LOG.info("Cold migrate VM {} ....".format(vm_name))
rc = vm_helper.cold_migrate_vm(vm_id=vm_id)[0]
assert rc == 0, "VM {} failed to cold migrate".format(vm_name)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
LOG.info("Live migrate VM {} ....".format(vm_name))
rc = vm_helper.live_migrate_vm(vm_id=vm_id)[0]
assert rc == 0, "VM {} failed to live migrate".format(vm_name)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
LOG.info("Suspend/Resume VM {} ....".format(vm_name))
vm_helper.suspend_vm(vm_id)
vm_helper.resume_vm(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
LOG.info("Checking overall system health...")
assert system_helper.get_system_health_query(
), "System health not OK after VMs"
LOG.tc_step("Create Swift container using swift post cli command ...")
container_names = [
"test_container_1", "test_container_2", "test_container_3"
]
for container in container_names:
LOG.info("Creating swift object container {}".format(container))
rc, out = swift_helper.create_swift_container(container)
assert rc == 0, "Fail to create swift container {}".format(container)
LOG.info(
"Create swift object container {} successfully".format(container))
LOG.tc_step("Verify swift list to list containers ...")
container_list = swift_helper.get_swift_containers()[1]
assert set(container_names) <= set(container_list), "Swift containers {} not listed in {}"\
.format(container_names, container_list)
LOG.tc_step("Verify swift delete a container...")
container_to_delete = container_names[2]
rc, out = swift_helper.delete_swift_container(container_to_delete)
assert rc == 0, "Swift delete container rejected: {}".format(out)
assert container_to_delete not in swift_helper.get_swift_containers()[1], "Unable to delete swift container {}"\
.format(container_to_delete)
LOG.tc_step("Verify swift stat to show info of a single container...")
container_to_stat = container_names[0]
out = swift_helper.get_swift_container_stat_info(container_to_stat)
assert out["Container"] == container_to_stat, "Unable to stat swift container {}"\
.format(container_to_stat)
assert out["Objects"] == '0', "Incorrect number of objects container {}. Expected O objects, but has {} objects"\
.format(container_to_stat, out["Objects"])
