def obsolete_test_fip(fip_setups):
"""
Test VM Floating IP over VM launch, live-migration, cold-migration, pause/unpause, etc
Args:
fip_setups: test fixture
Test Setups (module):
- Create a floating ip
- boot a vm
- Attach floating ip to vm
Test Steps:
- Ping VM FIP
- Live-migrate the VM and verify ping from VM
- Cold-migrate the VM and verify ping from VM
- Pause and un-pause the VM and verify ping from VM
- Suspend and resume the VM and verify ping from VM
- Stop and start the VM and verify ping from VM
- Reboot the VM and verify ping from VM
- Ping VM FIP
Test Teardown:
- Delete created FIP and vm (module)
"""
vm_id, fip = fip_setups
LOG.tc_step("Ping VM with Floating IP ")
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Live-migrate the VM and verify ping from VM")
vm_helper.live_migrate_vm(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Cold-migrate the VM and verify ping from VM")
vm_helper.cold_migrate_vm(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Pause and un-pause the VM and verify ping from VM")
vm_helper.pause_vm(vm_id)
vm_helper.unpause_vm(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Suspend and resume the VM and verify ping from VM")
vm_helper.suspend_vm(vm_id)
vm_helper.resume_vm(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Stop and start the VM and verify ping from VM")
vm_helper.stop_vms(vm_id)
vm_helper.start_vms(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Reboot the VM and verify ping from VM")
vm_helper.reboot_vm(vm_id)
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
LOG.tc_step("Ping VM with Floating IP Ensure FIP reachable ")
vm_helper.ping_ext_from_vm(vm_id, use_fip=True)
def check_server_group_messaging_enabled(vms, action):
vms = list(set(vms))
vm_sender = random.choice(vms)
vms.remove(vm_sender)
if action == 'message':
msg = MSG
timeout = 180
elif action == 'pause':
msg = '{}.*paused'.format(vm_sender)
timeout = 240
else:
raise ValueError("Unknown action - '{}' provided".format(action))
res_events = []
sent_event = Events("srv msg/event triggered")
listener_event = Events("VM started listening to server group messages")
vm_threads = []
sender_thread = None
try:
for vm in vms:
listener_event.clear()
new_thread = MThread(_wait_for_srv_grp_msg,
vm,
msg,
timeout=timeout,
res_events=res_events,
listener_event=listener_event,
sent_event=sent_event)
new_thread.start_thread(timeout=timeout + 30)
vm_threads.append(new_thread)
listener_event.wait_for_event()
time.sleep(5)
# this 60 seconds timeout is hardcoded for action == 'message' scenario to send
# the message out
sender_thread = MThread(trigger_srv_grp_msg,
vm_sender,
action,
timeout=60,
sent_event=sent_event,
rcv_event=res_events)
sender_thread.start_thread(timeout=timeout)
sent_event.wait_for_event()
for res_event in res_events:
res_event.wait_for_event()
finally:
# wait for server group msg to be received
for vm_thr in vm_threads:
vm_thr.wait_for_thread_end(timeout=30)
if sender_thread:
sender_thread.wait_for_thread_end(timeout=30)
if action == 'pause':
vm_helper.unpause_vm(vm_sender)
def test_patch_orch_reject_with_alarms(patch_orchestration_setup, patch_function_check):
"""
This test verifies the patch orchestration operation can not proceed with presence of alarms that are not normally
ignored by the orchestration. The test generates the alarm ( 700.002 - VM paused) before executing the patch
orchestration.
Args:
patch_orchestration_setup:
patch_function_check
Returns:
"""
vms = patch_function_check
patches, controllers, computes, storages = patch_orchestration_setup
LOG.tc_step("Generate VM paused ( 700.002) critical alarm")
paused_vm, unpaused_vm = vms
vm_helper.pause_vm(paused_vm)
system_helper.wait_for_alarm(alarm_id='700.002')
patch = patching_helper.parse_test_patches(patch_ids=patches, search_str='RR_ALLNODES')[0]
patch_file = patches[patch]
LOG.tc_step("Upload patch file {}".format(patch_file))
uploaded_id = patching_helper.upload_patches(patch_files=patch_file)[1][0]
assert patch == uploaded_id, "Expected patch {} and uploaded patch {} mismatch"\
.format(patch, uploaded_id)
LOG.info("Patch {} uploaded".format(uploaded_id))
LOG.tc_step("Apply patch {}".format(uploaded_id))
applied = patching_helper.apply_patches(patch_ids=[uploaded_id])[1]
LOG.info("Patch {} applied".format(applied))
LOG.tc_step("Attempt to create patch orchestration strategy; expected to fail")
rc, msg = orchestration_helper.create_strategy('patch', fail_ok=True)
assert rc != 0, "Patch orchestration strategy created with presence of critical alarm; expected to fail: {}"\
.format(msg)
LOG.info("Delete the failed patch orchestration strategy")
orchestration_helper.delete_strategy("patch")
LOG.tc_step("Remove test patch {}".format(applied))
patching_helper.remove_patches(patch_ids=applied)
assert 0 == patching_helper.wait_for_patch_states(applied, expected_states=PatchState.AVAILABLE)[0]
LOG.tc_step("Un-pause vm after test patch removal, and check vms are in good state.")
vm_helper.unpause_vm(paused_vm)
vm_helper.wait_for_vm_pingable_from_natbox(paused_vm)
check_vms(vms)
def test_pause_unpause_instances(launch_instances):
vm_helper.pause_vm(vm_id=launch_instances)
vm_helper.unpause_vm(vm_id=launch_instances)
def test_pci_vm_nova_actions(self, pci_numa_affinity,
pci_irq_affinity_mask, pci_alias,
vif_model_check, pci_dev_numa_nodes):
"""
Test vm actions on vm with multiple ports with given vif models on the same tenant network
Args:
Setups:
- create a flavor with dedicated cpu policy (module)
- choose one tenant network and one internal network to be used by test (module)
- boot a base vm - vm1 with above flavor and networks, and ping it from NatBox (module)
- Boot a vm under test - vm2 with above flavor and with multiple ports on same tenant network with base vm,
and ping it from NatBox (class)
- Ping vm2's own data network ips (class)
- Ping vm2 from vm1 to verify management and data networks connection (class)
Test Steps:
- Perform given actions on vm2 (migrate, start/stop, etc)
- Verify ping from vm1 to vm2 over management and data networks still works
- Verify the correct number of PCI devices are created, in correct types,
the numa node of the PCI devices aligns with that of CPUs, and affined CPUs for PCI devices
are same as specified by 'pci_alias' (if applicable)
Teardown:
- Delete created vms and flavor
"""
pci_irq_affinity_mask, pci_alias = _convert_irqmask_pcialias(
pci_irq_affinity_mask, pci_alias)
boot_forbidden = False
migrate_forbidden = False
if pci_numa_affinity == 'required' and pci_alias is not None:
host_count = pci_dev_numa_nodes
if host_count == 0:
boot_forbidden = True
elif host_count == 1:
migrate_forbidden = True
LOG.tc_step(
"Expected result - Disallow boot: {}; Disallow migrate: {}".format(
boot_forbidden, migrate_forbidden))
self.pci_numa_affinity = pci_numa_affinity
self.pci_alias = pci_alias
self.pci_irq_affinity_mask = pci_irq_affinity_mask
if pci_alias is not None:
LOG.info('Check if PCI-Alias devices existing')
self.is_pci_device_supported(pci_alias)
self.vif_model, self.base_vm, self.base_flavor_id, self.nics_to_test, self.seg_id, \
self.pnet_name, self.extra_pcipt_net = vif_model_check
LOG.tc_step(
"Create a flavor with specified extra-specs and dedicated cpu policy"
)
flavor_id = self.create_flavor_for_pci()
LOG.tc_step("Boot a vm with {} vif model on internal net".format(
self.vif_model))
# TODO: feature unavailable atm. Update required
# resource_param = 'pci_vfs_used' if 'sriov' in self.vif_model else 'pci_pfs_used'
# LOG.tc_step("Get resource usage for {} interface before booting VM(s)".format(self.vif_model))
# pre_resource_value = nova_helper.get_provider_net_info(self.pnet_name, field=resource_param)
res, vm_id, err = vm_helper.boot_vm(name=self.vif_model,
flavor=flavor_id,
cleanup='function',
nics=self.nics_to_test,
fail_ok=boot_forbidden)
if boot_forbidden:
assert res > 0, "VM booted successfully while it numa node for pcipt/sriov and pci alias mismatch"
return
self.vm_id = vm_id
if 'pci-passthrough' == self.vif_model:
LOG.tc_step("Add vlan to pci-passthrough interface for VM.")
vm_helper.add_vlan_for_vm_pcipt_interfaces(vm_id=vm_id,
net_seg_id=self.seg_id,
init_conf=True)
LOG.tc_step("Ping vm over mgmt and internal nets from base vm")
vm_helper.ping_vms_from_vm(to_vms=self.vm_id,
from_vm=self.vm_id,
net_types=['mgmt', 'internal'])
vm_helper.ping_vms_from_vm(to_vms=self.vm_id,
from_vm=self.base_vm,
net_types=['mgmt', 'internal'])
self.vm_topology = vm_helper.get_vm_values(
vm_id=self.vm_id, fields='wrs-res:topology')[0]
vnic_type = 'direct' if self.vif_model == 'pci-sriov' else 'direct-physical'
self.pci_nics = vm_helper.get_vm_nics_info(vm_id=self.vm_id,
vnic_type=vnic_type)
assert self.pci_nics
self.wait_check_vm_states(step='boot')
# TODO: feature unavailable atm. Update required
# LOG.tc_step("Check {} usage is incremented by 1".format(resource_param))
# post_resource_value = nova_helper.get_provider_net_info(self.pnet_name, field=resource_param)
# expt_change = 2 if self.vif_model == 'pci-passthrough' and self.extra_pcipt_net else 1
# assert pre_resource_value + expt_change == post_resource_value, "{} usage is not incremented by {} as " \
# "expected".format(resource_param, expt_change)
LOG.tc_step('Pause/Unpause {} vm'.format(self.vif_model))
vm_helper.pause_vm(self.vm_id)
vm_helper.unpause_vm(self.vm_id)
LOG.tc_step(
"Check vm still pingable over mgmt and internal nets after pause/unpause"
)
self.wait_check_vm_states(step='pause/unpause')
LOG.tc_step('Suspend/Resume {} vm'.format(self.vif_model))
vm_helper.suspend_vm(self.vm_id)
vm_helper.resume_vm(self.vm_id)
LOG.tc_step(
"Check vm still pingable over mgmt and internal nets after suspend/resume"
)
self.wait_check_vm_states(step='suspend/resume')
if 'pci-passthrough' == self.vif_model:
LOG.tc_step(
"Add/Check vlan interface is added to pci-passthrough device for vm {}."
.format(self.vm_id))
vm_helper.add_vlan_for_vm_pcipt_interfaces(vm_id=self.vm_id,
net_seg_id=self.seg_id)
vm_helper.ping_vms_from_vm(to_vms=self.vm_id,
from_vm=self.base_vm,
net_types=['mgmt', 'internal'])
LOG.tc_step('Cold migrate {} vm'.format(self.vif_model))
code, msg = vm_helper.cold_migrate_vm(self.vm_id,
fail_ok=migrate_forbidden)
if migrate_forbidden:
assert code > 0, "Expect migrate fail due to no other host has pcipt/sriov and pci-alias on same numa. " \
"Actual: {}".format(msg)
self.wait_check_vm_states(step='cold-migrate')
if 'pci-passthrough' == self.vif_model:
LOG.tc_step(
"Add/Check vlan interface is added to pci-passthrough device for vm {}."
.format(self.vm_id))
vm_helper.add_vlan_for_vm_pcipt_interfaces(vm_id=self.vm_id,
net_seg_id=self.seg_id)
LOG.tc_step(
"Check vm still pingable over mgmt and internal nets after cold migration"
)
vm_helper.ping_vms_from_vm(to_vms=self.vm_id,
from_vm=self.base_vm,
net_types=['mgmt', 'internal'])
LOG.tc_step('Set vm to error and wait for it to be auto recovered')
vm_helper.set_vm_state(vm_id=self.vm_id,
error_state=True,
fail_ok=False)
vm_helper.wait_for_vm_values(vm_id=self.vm_id,
status=VMStatus.ACTIVE,
fail_ok=False,
timeout=600)
LOG.tc_step(
"Check vm still pingable over mgmt and internal nets after auto recovery"
)
self.wait_check_vm_states(step='set-error-state-recover')
if 'pci-passthrough' == self.vif_model:
LOG.tc_step(
"Add/Check vlan interface is added to pci-passthrough device for vm {}."
.format(self.vm_id))
vm_helper.add_vlan_for_vm_pcipt_interfaces(vm_id=self.vm_id,
net_seg_id=self.seg_id)
vm_helper.ping_vms_from_vm(to_vms=self.vm_id,
from_vm=self.base_vm,
net_types=['mgmt', 'internal'])
LOG.tc_step("Hard reboot {} vm".format(self.vif_model))
vm_helper.reboot_vm(self.vm_id, hard=True)
LOG.tc_step(
"Check vm still pingable over mgmt and internal nets after nova reboot hard"
)
self.wait_check_vm_states(step='hard-reboot')
if 'pci-passthrough' == self.vif_model:
LOG.tc_step(
"Add/Check vlan interface is added to pci-passthrough device for vm {}."
.format(self.vm_id))
vm_helper.add_vlan_for_vm_pcipt_interfaces(vm_id=self.vm_id,
net_seg_id=self.seg_id)
vm_helper.ping_vms_from_vm(to_vms=self.vm_id,
from_vm=self.base_vm,
net_types=['mgmt', 'internal'])
LOG.tc_step(
"Create a flavor with dedicated cpu policy and resize vm to new flavor"
)
resize_flavor = nova_helper.create_flavor(name='dedicated',
ram=2048,
cleanup='function')[1]
extra_specs = {FlavorSpec.CPU_POLICY: 'dedicated'}
nova_helper.set_flavor(flavor=resize_flavor, **extra_specs)
vm_helper.resize_vm(self.vm_id, resize_flavor)
LOG.tc_step("Check vm still reachable after resize")
self.wait_check_vm_states(step='resize')
if 'pci-passthrough' == self.vif_model:
LOG.tc_step(
"Add/Check vlan interface is added to pci-passthrough device for vm {}."
.format(self.vm_id))
vm_helper.add_vlan_for_vm_pcipt_interfaces(vm_id=self.vm_id,
net_seg_id=self.seg_id)
vm_helper.ping_vms_from_vm(to_vms=self.vm_id,
from_vm=self.base_vm,
net_types=['mgmt', 'internal'])
def test_snat_vm_actions(snat_setups, snat):
"""
Test VM external access over VM launch, live-migration, cold-migration, pause/unpause, etc
Args:
snat_setups (tuple): returns vm id and fip. Enable snat, create vm and attach floating ip.
Test Setups (module):
- Find a tenant router that is dvr or non-dvr based on the parameter
- Enable SNAT on tenant router
- boot a vm and attach a floating ip
- Ping vm from NatBox
Test Steps:
- Enable/Disable SNAT based on snat param
- Ping from VM to 8.8.8.8
- wget <lab_fip> to VM
- scp from NatBox to VM
- Live-migrate the VM and verify ping from VM
- Cold-migrate the VM and verify ping from VM
- Pause and un-pause the VM and verify ping from VM
- Suspend and resume the VM and verify ping from VM
- Stop and start the VM and verify ping from VM
- Reboot the VM and verify ping from VM
Test Teardown:
- Enable snat for next test in the same module (function)
- Delete the created vm (module)
- Disable snat (module)
"""
vm_ = snat_setups[0]
snat = True if snat == 'snat_enabled' else False
LOG.tc_step("Update tenant router external gateway to set SNAT to {}".format(snat))
network_helper.set_router_gateway(enable_snat=snat)
# Allow router update to complete, since we've seen cases where ping vm pass but ssh fail
time.sleep(30)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, timeout=60, use_fip=snat)
LOG.tc_step("Ping from VM {} to 8.8.8.8".format(vm_))
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("wget to VM {}".format(vm_))
with vm_helper.ssh_to_vm_from_natbox(vm_id=vm_, use_fip=True) as vm_ssh:
vm_ssh.exec_cmd('wget google.ca', fail_ok=False)
LOG.tc_step("scp from NatBox to VM {}".format(vm_))
vm_fip = network_helper.get_external_ips_for_vms(vms=vm_)[0]
natbox_ssh = NATBoxClient.get_natbox_client()
natbox_ssh.scp_on_source(source_path='test', dest_user='******', dest_ip=vm_fip, dest_path='/tmp/',
dest_password='******', timeout=30)
LOG.tc_step("Live-migrate the VM and verify ping from VM")
vm_helper.live_migrate_vm(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Cold-migrate the VM and verify ping from VM")
vm_helper.cold_migrate_vm(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Pause and un-pause the VM and verify ping from VM")
vm_helper.pause_vm(vm_)
vm_helper.unpause_vm(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Suspend and resume the VM and verify ping from VM")
vm_helper.suspend_vm(vm_)
vm_helper.resume_vm(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Stop and start the VM and verify ping from VM")
vm_helper.stop_vms(vm_)
vm_helper.start_vms(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Reboot the VM and verify ping from VM")
vm_helper.reboot_vm(vm_)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
LOG.tc_step("Resize the vm to a flavor with 2 dedicated cpus and verify ping from VM")
new_flv = nova_helper.create_flavor(name='ded', vcpus=2)[1]
ResourceCleanup.add('flavor', new_flv, scope='module')
nova_helper.set_flavor(new_flv, **{FlavorSpec.CPU_POLICY: 'dedicated'})
vm_helper.resize_vm(vm_, new_flv)
vm_helper.wait_for_vm_pingable_from_natbox(vm_, use_fip=snat)
vm_helper.ping_ext_from_vm(vm_, use_fip=True)
def perform_vm_operation(vm_type,
vm_id,
op='live_migration',
extra_specs='vtpm'):
LOG.info('Perform action:{} to the VM, extra specs:{}'.format(
op, extra_specs))
op_table = {
'live_migration':
lambda x, y: vm_helper.live_migrate_vm(y),
'cold_migration':
lambda x, y: vm_helper.cold_migrate_vm(y),
'stop_start':
lambda x, y: (vm_helper.stop_vms(y), vm_helper.start_vms(y)),
'suspend_resume':
lambda x, y: (vm_helper.suspend_vm(y), vm_helper.resume_vm(y)),
'pause_unpause':
lambda x, y: (vm_helper.pause_vm(y), vm_helper.unpause_vm(y)),
'reboot_host':
lambda x, y: reboot_hosting_node(x, y, force_reboot=False),
'soft_reboot':
lambda x, y: vm_helper.reboot_vm(y, hard=False),
'hard_reboot':
lambda x, y: vm_helper.reboot_vm(y, hard=True),
'lock_unlock':
lambda x, y: lock_unlock_hosting_node(x, y, force_lock=False),
'evacuate':
lambda x, y: reboot_hosting_node(x, y, force_reboot=True),
}
if op in op_table:
LOG.info('Perform action: {}'.format(op))
op_table[op](vm_type, vm_id)
return True
elif op == 'resize_to_autorc':
if vm_type == 'autorc':
LOG.info(
'resize from AUTO-RECOVERY to another AUTO-RECOVER flavor')
to_flavor_id = get_flavor_id(vm_type, 'autorc2')
LOG.info('TODO: {}, m_type={}, to_flavor_id={}'.format(
to_flavor_id, vm_type, to_flavor_id))
vm_helper.resize_vm(vm_id, to_flavor_id)
elif op == 'resize_to_non_autorc':
LOG.info('perform {} on type:{}, id:{}'.format(op, vm_type, vm_id))
if vm_type == 'non_autorc2':
LOG.warn(
'resize from AUTO-RECOVERY to another AUTO-RECOVER flavor')
to_flavor_id = get_flavor_id(vm_type, 'non_autorc2')
vm_helper.resize_vm(vm_id, to_flavor_id)
elif op == 'resize_to_non_vtpm':
LOG.info('perform {} on type:{}, id:{}'.format(op, vm_type, vm_id))
to_flavor_id = get_flavor_id(vm_type, 'non_vtpm')
vm_helper.resize_vm(vm_id, to_flavor_id)
else:
LOG.fatal('Unsupported action: {}'.format(op))
return False