def run(test, params, env):
"""
:param test: kvm test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
guest_stress = params.get("guest_stress", "no") == "yes"
host_stress = params.get("host_stress", "no") == "yes"
stress_events = params.get("stress_events", "reboot")
vms = env.get_all_vms()
vms_uptime_init = {}
if "reboot" not in stress_events:
for vm in vms:
vms_uptime_init[vm.name] = vm.uptime()
stress_event = utils_stress.VMStressEvents(params, env)
if guest_stress:
try:
utils_test.load_stress("stress_in_vms", params=params, vms=vms)
except Exception as err:
test.fail("Error running stress in vms: %s" % err)
if host_stress:
if params.get("host_stress_args", ""):
params["stress_args"] = params.get("host_stress_args")
try:
utils_test.load_stress("stress_on_host", params=params)
except Exception as err:
test.fail("Error running stress in host: %s" % err)
try:
stress_event.run_threads()
finally:
stress_event.wait_for_threads()
if guest_stress:
utils_test.unload_stress("stress_in_vms", params=params, vms=vms)
if host_stress:
utils_test.unload_stress("stress_on_host", params=params)
if "reboot" not in stress_events:
fail = False
for vm in vms:
if vm.uptime() < vms_uptime_init[vm.name]:
logging.error("Unexpected reboot of VM: %s between test", vm.name)
fail = True
if fail:
test.fail("Unexpected VM reboot detected")
def do_stress_migration(vms, srcuri, desturi, stress_type,
migration_type, params, thread_timeout=60):
"""
Migrate vms with stress.
:param vms: migrated vms.
:param srcuri: connect uri for source machine
:param desturi: connect uri for destination machine
:param stress_type: type of stress test in VM
:param migration_type: type of migration to be performed
:param params: Test dict params
:param thread_timeout: default timeout for migration thread
:raise: exceptions.TestFail if migration fails
"""
fail_info = utils_test.load_stress(stress_type, vms, params)
migtest = utlv.MigrationTest()
options = ''
if migration_type == "compressed":
options = "--compressed"
migration_type = "orderly"
shared_dir = os.path.dirname(data_dir.get_data_dir())
src_file = os.path.join(shared_dir, "scripts", "duplicate_pages.py")
dest_dir = "/tmp"
for vm in vms:
session = vm.wait_for_login()
vm.copy_files_to(src_file, dest_dir)
status = session.cmd_status("cd /tmp;python duplicate_pages.py")
if status:
fail_info.append("Set duplicated pages for vm failed.")
if len(fail_info):
logging.warning("Add stress for migration failed:%s", fail_info)
logging.debug("Starting migration...")
migrate_options = ("--live --unsafe %s --timeout %s"
% (options, params.get("virsh_migrate_timeout", 60)))
migtest.do_migration(vms, srcuri, desturi, migration_type,
options=migrate_options,
thread_timeout=thread_timeout)
# vms will be shutdown, so no need to do this cleanup
# And migrated vms may be not login if the network is local lan
if stress_type == "stress_on_host":
utils_test.unload_stress(stress_type, vms)
if not migtest.RET_MIGRATION:
raise exceptions.TestFail()
def do_stress_migration(vms, srcuri, desturi, stress_type,
migration_type, params, thread_timeout=60):
"""
Migrate vms with stress.
:param vms: migrated vms.
"""
fail_info = utils_test.load_stress(stress_type, vms, params)
if len(fail_info):
logging.warning("Add stress for migration failed:%s", fail_info)
migtest = utlv.MigrationTest()
migtest.do_migration(vms, srcuri, desturi, migration_type, thread_timeout)
utils_test.unload_stress(stress_type, vms)
if not migtest.RET_MIGRATION:
raise error.TestFail()
def load_stress(vms, stress_type, params={}):
"""
Load different stress in vm: unixbench, stress, running vms and so on
"""
fail_info = []
# Special operations for test
if stress_type in ["stress_in_vms", "stress_on_host"]:
logging.debug("Run stress %s", stress_type)
fail_info = utils_test.load_stress(stress_type, vms, params)
elif stress_type == "inject_nmi":
inject_times = int(params.get("inject_times", 10))
for vm in vms:
while inject_times > 0:
try:
inject_times -= 1
virsh.inject_nmi(vm.name, debug=True, ignore_status=False)
except error.CmdError, detail:
fail_info.append("Inject operations failed:%s", detail)
def do_stress_migration(vms, srcuri, desturi, stress_type,
migration_type, params, thread_timeout=60):
"""
Migrate vms with stress.
:param vms: migrated vms.
"""
fail_info = utils_test.load_stress(stress_type, vms, params)
if len(fail_info):
logging.warning("Add stress for migration failed:%s", fail_info)
migtest = utlv.MigrationTest()
migtest.do_migration(vms, srcuri, desturi, migration_type, options=None,
thread_timeout=thread_timeout)
# vms will be shutdown, so no need to do this cleanup
# And migrated vms may be not login if the network is local lan
if stress_type == "stress_on_host":
utils_test.unload_stress(stress_type, vms)
if not migtest.RET_MIGRATION:
raise error.TestFail()
def manipulate_vm(vm, operation, params=None):
"""
Manipulate the VM.
:param vm: VM instance
:param operation: stress_in_vms, inject_nmi, dump, suspend_resume
or save_restore
:param params: Test parameters
"""
err_msg = ''
# Special operations for test
if operation == "stress":
logging.debug("Load stress in VM")
err_msg = utils_test.load_stress(operation, [vm], params)[0]
elif operation == "inject_nmi":
inject_times = int(params.get("inject_times", 10))
logging.info("Trying to inject nmi %s times", inject_times)
while inject_times > 0:
try:
inject_times -= 1
virsh.inject_nmi(vm.name, debug=True, ignore_status=False)
except error.CmdError, detail:
err_msg = "Inject nmi failed: %s" % detail
def manipulate_vm(vm, operation, params=None):
"""
Manipulate the VM.
:param vm: VM instance
:param operation: stress_in_vms, inject_nmi, dump, suspend_resume
or save_restore
:param params: Test parameters
"""
err_msg = ''
# Special operations for test
if operation == "stress":
logging.debug("Load stress in VM")
err_msg = utils_test.load_stress(operation, [vm], params)[0]
elif operation == "inject_nmi":
inject_times = int(params.get("inject_times", 10))
logging.info("Trying to inject nmi %s times", inject_times)
while inject_times > 0:
try:
inject_times -= 1
virsh.inject_nmi(vm.name, debug=True, ignore_status=False)
except process.CmdError as detail:
err_msg = "Inject nmi failed: %s" % detail
elif operation == "dump":
dump_times = int(params.get("dump_times", 10))
logging.info("Trying to dump vm %s times", dump_times)
while dump_times > 0:
dump_times -= 1
dump_path = os.path.join(data_dir.get_tmp_dir(), "dump.file")
try:
virsh.dump(vm.name, dump_path, debug=True, ignore_status=False)
except (process.CmdError, OSError) as detail:
err_msg = "Dump %s failed: %s" % (vm.name, detail)
try:
os.remove(dump_path)
except OSError:
pass
elif operation == "suspend_resume":
paused_times = int(params.get("paused_times", 10))
logging.info("Trying to suspend/resume vm %s times", paused_times)
while paused_times > 0:
paused_times -= 1
try:
virsh.suspend(vm.name, debug=True, ignore_status=False)
virsh.resume(vm.name, debug=True, ignore_status=False)
except process.CmdError as detail:
err_msg = "Suspend-Resume %s failed: %s" % (vm.name, detail)
elif operation == "save_restore":
save_times = int(params.get("save_times", 10))
logging.info("Trying to save/restore vm %s times", save_times)
while save_times > 0:
save_times -= 1
save_path = os.path.join(data_dir.get_tmp_dir(), "save.file")
try:
virsh.save(vm.name, save_path, debug=True, ignore_status=False)
virsh.restore(save_path, debug=True, ignore_status=False)
except process.CmdError as detail:
err_msg = "Save-Restore %s failed: %s" % (vm.name, detail)
try:
os.remove(save_path)
except OSError:
pass
else:
err_msg = "Unsupport operation in this function: %s" % operation
return err_msg
def manipulate_vm(vm, operation, params=None):
"""
Manipulate the VM.
:param vm: VM instance
:param operation: stress_in_vms, inject_nmi, dump, suspend_resume
or save_restore
:param params: Test parameters
"""
err_msg = ''
# Special operations for test
if operation == "stress":
logging.debug("Load stress in VM")
err_msg = utils_test.load_stress(operation, params=params, vms=[vm])[0]
elif operation == "inject_nmi":
inject_times = int(params.get("inject_times", 10))
logging.info("Trying to inject nmi %s times", inject_times)
while inject_times > 0:
try:
inject_times -= 1
virsh.inject_nmi(vm.name, debug=True, ignore_status=False)
except process.CmdError as detail:
err_msg = "Inject nmi failed: %s" % detail
elif operation == "dump":
dump_times = int(params.get("dump_times", 10))
logging.info("Trying to dump vm %s times", dump_times)
while dump_times > 0:
dump_times -= 1
dump_path = os.path.join(data_dir.get_tmp_dir(), "dump.file")
try:
virsh.dump(vm.name, dump_path, debug=True, ignore_status=False)
except (process.CmdError, OSError) as detail:
err_msg = "Dump %s failed: %s" % (vm.name, detail)
try:
os.remove(dump_path)
except OSError:
pass
elif operation == "suspend_resume":
paused_times = int(params.get("paused_times", 10))
logging.info("Trying to suspend/resume vm %s times", paused_times)
while paused_times > 0:
paused_times -= 1
try:
virsh.suspend(vm.name, debug=True, ignore_status=False)
virsh.resume(vm.name, debug=True, ignore_status=False)
except process.CmdError as detail:
err_msg = "Suspend-Resume %s failed: %s" % (vm.name, detail)
elif operation == "save_restore":
save_times = int(params.get("save_times", 10))
logging.info("Trying to save/restore vm %s times", save_times)
while save_times > 0:
save_times -= 1
save_path = os.path.join(data_dir.get_tmp_dir(), "save.file")
try:
virsh.save(vm.name, save_path, debug=True,
ignore_status=False)
virsh.restore(save_path, debug=True, ignore_status=False)
except process.CmdError as detail:
err_msg = "Save-Restore %s failed: %s" % (vm.name, detail)
try:
os.remove(save_path)
except OSError:
pass
else:
err_msg = "Unsupport operation in this function: %s" % operation
return err_msg
def copied_migration(vm, params, blockjob_type=None, block_target="vda"):
"""
Migrate vms with storage copied under some stress.
And during it, some qemu-monitor-command will be sent.
"""
dest_uri = params.get("migrate_dest_uri")
remote_host = params.get("migrate_dest_host")
copy_option = params.get("copy_storage_option", "")
username = params.get("remote_user")
password = params.get("migrate_dest_pwd")
timeout = int(params.get("thread_timeout", 1200))
options = "--live %s --unsafe" % copy_option
# Get vm ip for remote checking
if vm.is_dead():
vm.start()
vm.wait_for_login()
vms_ip = {}
vms_ip[vm.name] = vm.get_address()
logging.debug("VM %s IP: %s", vm.name, vms_ip[vm.name])
# Start to load stress
stress_type = params.get("migrate_stress_type")
if stress_type == "cpu":
params['stress_args'] = "--cpu 2 --quiet --timeout 60"
elif stress_type == "memory":
params['stress_args'] = "--vm 2 --vm-bytes 256M --vm-keep --timeout 60"
if stress_type is not None:
utils_test.load_stress("stress_in_vms", [vm], params)
cp_mig = utlv.MigrationTest()
migration_thread = threading.Thread(target=cp_mig.thread_func_migration,
args=(vm, dest_uri, options))
migration_thread.start()
# Wait for migration launched
time.sleep(5)
job_ret = virsh.domjobinfo(vm.name, debug=True)
if job_ret.exit_status:
raise error.TestError("Prepare migration for blockjob failed.")
# Execute some qemu monitor commands
pause_cmd = "block-job-pause %s" % block_target
resume_cmd = "block-job-resume %s" % block_target
cancel_cmd = "block-job-cancel %s" % block_target
complete_cmd = "block-job-complete %s" % block_target
blockjob_failures = []
try:
if blockjob_type == "cancel":
virsh.qemu_monitor_command(vm.name,
cancel_cmd,
debug=True,
ignore_status=False)
elif blockjob_type == "pause_resume":
virsh.qemu_monitor_command(vm.name,
pause_cmd,
debug=True,
ignore_status=False)
# TODO: Check whether it is paused.
virsh.qemu_monitor_command(vm.name,
resume_cmd,
debug=True,
ignore_status=False)
elif blockjob_type == "complete":
virsh.qemu_monitor_command(vm.name,
complete_cmd,
debug=True,
ignore_status=False)
except process.CmdError, detail:
blockjob_failures.append(str(detail))
def copied_migration(vm, params, blockjob_type=None, block_target="vda"):
"""
Migrate vms with storage copied under some stress.
And during it, some qemu-monitor-command will be sent.
"""
dest_uri = params.get("migrate_dest_uri")
remote_host = params.get("migrate_dest_host")
copy_option = params.get("copy_storage_option", "")
username = params.get("remote_user")
password = params.get("migrate_dest_pwd")
timeout = int(params.get("thread_timeout", 1200))
options = "--live %s --unsafe" % copy_option
# Get vm ip for remote checking
if vm.is_dead():
vm.start()
vm.wait_for_login()
vms_ip = {}
vms_ip[vm.name] = vm.get_address()
logging.debug("VM %s IP: %s", vm.name, vms_ip[vm.name])
# Start to load stress
stress_type = params.get("migrate_stress_type")
if stress_type == "cpu":
params['stress_args'] = "--cpu 2 --quiet --timeout 60"
elif stress_type == "memory":
params['stress_args'] = "--vm 2 --vm-bytes 256M --vm-keep --timeout 60"
if stress_type is not None:
utils_test.load_stress("stress_in_vms", [vm], params)
cp_mig = utlv.MigrationTest()
migration_thread = threading.Thread(target=cp_mig.thread_func_migration,
args=(vm, dest_uri, options))
migration_thread.start()
# Wait for migration launched
time.sleep(5)
job_ret = virsh.domjobinfo(vm.name, debug=True)
if job_ret.exit_status:
raise error.TestError("Prepare migration for blockjob failed.")
# Execute some qemu monitor commands
pause_cmd = "block-job-pause %s" % block_target
resume_cmd = "block-job-resume %s" % block_target
cancel_cmd = "block-job-cancel %s" % block_target
complete_cmd = "block-job-complete %s" % block_target
blockjob_failures = []
try:
if blockjob_type == "cancel":
virsh.qemu_monitor_command(vm.name, cancel_cmd, debug=True,
ignore_status=False)
elif blockjob_type == "pause_resume":
virsh.qemu_monitor_command(vm.name, pause_cmd, debug=True,
ignore_status=False)
# TODO: Check whether it is paused.
virsh.qemu_monitor_command(vm.name, resume_cmd, debug=True,
ignore_status=False)
elif blockjob_type == "complete":
virsh.qemu_monitor_command(vm.name, complete_cmd, debug=True,
ignore_status=False)
except error.CmdError, detail:
blockjob_failures.append(str(detail))
def copied_migration(test, vm, params, blockjob_type=None, block_target="vda"):
"""
Migrate vms with storage copied under some stress.
And during it, some qemu-monitor-command will be sent.
"""
dest_uri = params.get("migrate_dest_uri")
remote_host = params.get("migrate_dest_host")
copy_option = params.get("copy_storage_option", "")
username = params.get("remote_user")
password = params.get("migrate_dest_pwd")
timeout = int(params.get("thread_timeout", 1200))
options = "--live %s --unsafe" % copy_option
# Get vm ip for remote checking
if vm.is_dead():
vm.start()
vm.wait_for_login()
vms_ip = {}
vms_ip[vm.name] = vm.get_address()
logging.debug("VM %s IP: %s", vm.name, vms_ip[vm.name])
# Start to load stress
stress_type = params.get("migrate_stress_type")
if stress_type == "cpu":
params['stress_args'] = "--cpu 2 --quiet --timeout 60"
elif stress_type == "memory":
params['stress_args'] = "--vm 2 --vm-bytes 256M --vm-keep --timeout 60"
if stress_type is not None:
utils_test.load_stress("stress_in_vms", [vm], params)
cp_mig = utlv.MigrationTest()
migration_thread = threading.Thread(target=cp_mig.thread_func_migration,
args=(vm, dest_uri, options))
migration_thread.start()
# Wait for migration launched
time.sleep(5)
job_ret = virsh.domjobinfo(vm.name, debug=True)
if job_ret.exit_status:
test.error("Prepare migration for blockjob failed.")
# Execute some qemu monitor commands
pause_cmd = "block-job-pause %s" % block_target
resume_cmd = "block-job-resume %s" % block_target
cancel_cmd = "block-job-cancel %s" % block_target
complete_cmd = "block-job-complete %s" % block_target
blockjob_failures = []
try:
if blockjob_type == "cancel":
virsh.qemu_monitor_command(vm.name,
cancel_cmd,
debug=True,
ignore_status=False)
elif blockjob_type == "pause_resume":
virsh.qemu_monitor_command(vm.name,
pause_cmd,
debug=True,
ignore_status=False)
# TODO: Check whether it is paused.
virsh.qemu_monitor_command(vm.name,
resume_cmd,
debug=True,
ignore_status=False)
elif blockjob_type == "complete":
virsh.qemu_monitor_command(vm.name,
complete_cmd,
debug=True,
ignore_status=False)
except process.CmdError as detail:
blockjob_failures.append(str(detail))
# Job info FYI
virsh.domjobinfo(vm.name, debug=True)
if len(blockjob_failures):
timeout = 30
migration_thread.join(timeout)
if migration_thread.isAlive():
logging.error("Migrate %s timeout.", migration_thread)
cp_mig.RET_LOCK.acquire()
cp_mig.RET_MIGRATION = False
cp_mig.RET_LOCK.release()
if len(blockjob_failures):
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
test.fail("Run qemu monitor command failed %s" % blockjob_failures)
check_ip_failures = []
if cp_mig.RET_MIGRATION:
try:
utils_test.check_dest_vm_network(vm, vms_ip[vm.name], remote_host,
username, password)
except exceptions.TestFail as detail:
check_ip_failures.append(str(detail))
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
test.fail("Storage migration passed even after " "cancellation.")
else:
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
logging.error("Expected Migration Error for %s", blockjob_type)
return
else:
test.fail("Command blockjob does not work well under "
"storage copied migration.")
if len(check_ip_failures):
test.fail("Check IP failed:%s" % check_ip_failures)
def run(test, params, env):
"""
Test: vcpu hotplug.
The command can change the number of virtual CPUs for VM.
1.Prepare test environment,destroy or suspend a VM.
2.Perform virsh setvcpus operation.
3.Recover test environment.
4.Confirm the test result.
"""
vm_name = params.get("main_vm")
min_count = int(params.get("setvcpus_min_count", "1"))
max_count = int(params.get("setvcpus_max_count", "2"))
test_times = int(params.get("setvcpus_test_times", "1"))
stress_type = params.get("stress_type", "")
stress_param = params.get("stress_param", "")
add_by_virsh = ("yes" == params.get("add_by_virsh"))
del_by_virsh = ("yes" == params.get("del_by_virsh"))
hotplug_timeout = int(params.get("hotplug_timeout", 30))
test_set_max = max_count * 2
# Save original configuration
orig_config_xml = libvirt_xml.VMXML.new_from_inactive_dumpxml(vm_name)
# Set min/max of vcpu
libvirt_xml.VMXML.set_vm_vcpus(vm_name,
test_set_max,
min_count,
topology_correction=True)
# prepare VM instance
vm = libvirt_vm.VM(vm_name, params, test.bindir, env.get("address_cache"))
# prepare guest-agent service
vm.prepare_guest_agent()
# Increase the workload
load_vms = []
if stress_type in ['cpu', 'memory', 'io']:
params["stress_args"] = stress_param
load_vms.append(vm)
if stress_type in ['cpu', 'memory']:
utils_test.load_stress("stress_in_vms", params, vms=load_vms)
else:
utils_test.load_stress("iozone_in_vms", params, vms=load_vms)
session = vm.wait_for_login()
try:
# Clear dmesg before set vcpu
session.cmd("dmesg -c")
for i in range(test_times):
# 1. Add vcpu
add_result = cpu.hotplug_domain_vcpu(vm, max_count, add_by_virsh)
add_status = add_result.exit_status
# 1.1 check add status
if add_status:
if add_result.stderr.count("support"):
test.cancel("vcpu hotplug not supported, "
"no need to test any more:\n %s" %
add_result.stderr.strip())
test.fail("Test failed for:\n %s" % add_result.stderr.strip())
if not utils_misc.wait_for(
lambda: cpu.check_if_vm_vcpu_match(max_count, vm),
hotplug_timeout,
text="wait for vcpu online"):
test.fail("vcpu hotplug failed")
if 'ppc' not in platform.machine():
# 1.2 check dmesg
domain_add_dmesg = session.cmd_output("dmesg -c")
dmesg1 = "CPU%d has been hot-added" % (max_count - 1)
dmesg2 = "CPU %d got hotplugged" % (max_count - 1)
if (not domain_add_dmesg.count(dmesg1)
and not domain_add_dmesg.count(dmesg2)):
test.fail("Cannot find hotplug info in dmesg: %s" %
domain_add_dmesg)
# 1.3 check cpu related file
online_cmd = "cat /sys/devices/system/cpu/cpu%d/online" \
% (max_count - 1)
st, ot = session.cmd_status_output(online_cmd)
if st:
test.fail("Cannot find CPU%d after hotplug" % (max_count - 1))
# 1.4 check online
if not ot.strip().count("1"):
test.fail("CPU%d is not online after hotplug: %s" %
((max_count - 1), ot))
# 1.5 check online interrupts info
inter_on_output = session.cmd_output("cat /proc/interrupts")
if not inter_on_output.count("CPU%d" % (int(max_count) - 1)):
test.fail("CPU%d can not be found in "
"/proc/interrupts when it's online:%s" %
((int(max_count) - 1), inter_on_output))
# 1.6 offline vcpu
off_st = session.cmd_status(
"echo 0 > "
"/sys/devices/system/cpu/cpu%d/online" % (max_count - 1))
if off_st:
test.fail("Set cpu%d offline failed!" % (max_count - 1))
# 1.7 check offline interrupts info
inter_off_output = session.cmd_output("cat /proc/interrupts")
if inter_off_output.count("CPU%d" % (int(max_count) - 1)):
test.fail("CPU%d can be found in /proc/interrupts"
" when it's offline" % (int(max_count) - 1))
# 1.8 online vcpu
on_st = session.cmd_status("echo 1 > "
"/sys/devices/system/cpu/cpu%d/online" %
(max_count - 1))
if on_st:
test.fail("Set cpu%d online failed!" % (max_count - 1))
# 2. Del vcpu
del_result = cpu.hotplug_domain_vcpu(vm,
min_count,
del_by_virsh,
hotplug=False)
del_status = del_result.exit_status
if del_status:
logging.info("del_result: %s" % del_result.stderr.strip())
# A qemu older than 1.5 or an unplug for 1.6 will result in
# the following failure.
# TODO: when CPU-hotplug feature becomes stable and strong,
# remove these codes used to handle kinds of exceptions
if re.search("The command cpu-del has not been found",
del_result.stderr):
test.cancel("vcpu hotunplug not supported")
if re.search("cannot change vcpu count", del_result.stderr):
test.cancel("unhotplug failed")
if re.search("got wrong number of vCPU pids from QEMU monitor",
del_result.stderr):
test.cancel("unhotplug failed")
# process all tips that contains keyword 'support'
# for example, "unsupported"/"hasn't been support" and so on
if re.search("support", del_result.stderr):
test.cancel("vcpu hotunplug not supported")
# besides above, regard it failed
test.fail("Test fail for:\n %s" % del_result.stderr.strip())
if not utils_misc.wait_for(
lambda: cpu.check_if_vm_vcpu_match(min_count, vm),
hotplug_timeout,
text="wait for vcpu offline"):
test.fail("vcpu hotunplug failed")
if 'ppc' not in platform.machine():
domain_del_dmesg = session.cmd_output("dmesg -c")
if not domain_del_dmesg.count("CPU %d is now offline" %
(max_count - 1)):
test.fail("Cannot find hot-unplug info in dmesg: %s" %
domain_del_dmesg)
except exceptions.TestCancel:
# So far, QEMU doesn't support unplug vcpu,
# unplug operation will encounter kind of errors.
pass
finally:
utils_test.unload_stress("stress_in_vms", params, load_vms)
if session:
session.close()
# Cleanup
orig_config_xml.sync()
def run(test, params, env):
"""
Test different hmi injections with guest
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
def set_condn(action, recover=False):
"""
Set/reset guest state/action
:param action: Guest state change/action
:param recover: whether to recover given state default: False
"""
if not recover:
if action == "pin_vcpu":
for i in range(cur_vcpu):
virsh.vcpupin(vm_name,
i,
hmi_cpu,
"--live",
ignore_status=False,
debug=True)
virsh.emulatorpin(vm_name,
hmi_cpu,
"live",
ignore_status=False,
debug=True)
elif action == "filetrans":
utils_test.run_file_transfer(test, params, env)
elif action == "save":
save_file = os.path.join(data_dir.get_tmp_dir(),
vm_name + ".save")
result = virsh.save(vm_name,
save_file,
ignore_status=True,
debug=True)
utils_test.libvirt.check_exit_status(result)
time.sleep(10)
if os.path.exists(save_file):
result = virsh.restore(save_file,
ignore_status=True,
debug=True)
utils_test.libvirt.check_exit_status(result)
os.remove(save_file)
elif action == "suspend":
result = virsh.suspend(vm_name, ignore_status=True, debug=True)
utils_test.libvirt.check_exit_status(result)
time.sleep(10)
result = virsh.resume(vm_name, ignore_status=True, debug=True)
utils_test.libvirt.check_exit_status(result)
return
host_version = params.get("host_version")
guest_version = params.get("guest_version", "")
max_vcpu = int(params.get("ppchmi_vcpu_max", '1'))
cur_vcpu = int(params.get("ppchmi_vcpu_cur", "1"))
cores = int(params.get("ppchmi_cores", '1'))
sockets = int(params.get("ppchmi_sockets", '1'))
threads = int(params.get("ppchmi_threads", '1'))
status_error = "yes" == params.get("status_error", "no")
condition = params.get("condn", "")
inject_code = params.get("inject_code", "")
scom_base = params.get("scom_base", "")
hmi_name = params.get("hmi_name", "")
hmi_iterations = int(params.get("hmi_iterations", 1))
if host_version not in cpu.get_cpu_arch():
test.cancel("Unsupported Host cpu version")
vm_name = params.get("main_vm")
vm = env.get_vm(vm_name)
sm = SoftwareManager()
if not sm.check_installed("opal-utils") and not sm.install("opal-utils"):
test.cancel("opal-utils package install failed")
cpus_list = cpu.cpu_online_list()
cpu_idle_state = cpu.get_cpuidle_state()
cpu.set_cpuidle_state()
# Lets use second available host cpu
hmi_cpu = cpus_list[1]
pir = int(
open('/sys/devices/system/cpu/cpu%s/pir' % hmi_cpu).read().strip(), 16)
if host_version == 'power9':
coreid = (((pir) >> 2) & 0x3f)
nodeid = (((pir) >> 8) & 0x7f) & 0xf
hmi_scom_addr = hex(((coreid & 0x1f + 0x20) << 24)
| int(scom_base, 16))
if host_version == 'power8':
coreid = (((pir) >> 3) & 0xf)
nodeid = (((pir) >> 7) & 0x3f)
hmi_scom_addr = hex(((coreid & 0xf) << 24) | int(scom_base, 16))
hmi_cmd = "putscom -c %s %s %s" % (nodeid, hmi_scom_addr, inject_code)
vmxml = libvirt_xml.VMXML.new_from_inactive_dumpxml(vm_name)
org_xml = vmxml.copy()
# Destroy the vm
vm.destroy()
try:
session = None
bgt = None
libvirt_xml.VMXML.set_vm_vcpus(vm_name,
max_vcpu,
cur_vcpu,
sockets=sockets,
cores=cores,
threads=threads,
add_topology=True)
if guest_version:
libvirt_xml.VMXML.set_cpu_mode(vm_name, model=guest_version)
vm.start()
# Lets clear host and guest dmesg
process.system("dmesg -C", verbose=False)
session = vm.wait_for_login()
session.cmd("dmesg -C")
# Set condn
if "vcpupin" in condition:
set_condn("pin_vcpu")
if "stress" in condition:
utils_test.load_stress("stress_in_vms", params=params, vms=[vm])
if "save" in condition:
set_condn("save")
if "suspend" in condition:
set_condn("suspend")
# hmi inject
logging.debug("Injecting %s HMI on cpu %s", hmi_name, hmi_cpu)
logging.debug("HMI Command: %s", hmi_cmd)
process.run(hmi_cmd)
# Check host and guest dmesg
host_dmesg = process.system_output("dmesg -c", verbose=False)
guest_dmesg = session.cmd_output("dmesg")
if "Unrecovered" in host_dmesg:
test.fail("Unrecovered host hmi\n%s", host_dmesg)
else:
logging.debug("Host dmesg: %s", host_dmesg)
logging.debug("Guest dmesg: %s", guest_dmesg)
if "save" in condition:
set_condn("save")
if "suspend" in condition:
set_condn("suspend")
finally:
if "stress" in condition:
utils_test.unload_stress("stress_in_vms", params=params, vms=[vm])
if session:
session.close()
org_xml.sync()
cpu.set_cpuidle_state(setstate=cpu_idle_state)
def set_condition(vm_name, condn, reset=False, guestbt=None):
"""
Set domain to given state or reset it.
"""
bt = None
if not reset:
if condn == "avocadotest":
bt = utils_test.run_avocado_bg(vm, params, test)
if not bt:
test.cancel("guest stress failed to start")
# Allow stress to start
time.sleep(condn_sleep_sec)
return bt
elif condn == "stress":
utils_test.load_stress("stress_in_vms", params=params, vms=[vm])
elif condn in ["save", "managedsave"]:
# No action
pass
elif condn == "suspend":
result = virsh.suspend(vm_name, ignore_status=True, debug=True)
libvirt.check_exit_status(result)
elif condn == "hotplug":
result = virsh.setvcpus(vm_name, max_vcpu, "--live",
ignore_status=True, debug=True)
libvirt.check_exit_status(result)
exp_vcpu = {'max_config': max_vcpu, 'max_live': max_vcpu,
'cur_config': current_vcpu, 'cur_live': max_vcpu,
'guest_live': max_vcpu}
result = cpu.check_vcpu_value(vm, exp_vcpu,
option="--live")
elif condn == "host_smt":
if cpuutil.get_cpu_vendor_name() == 'power9':
result = process.run("ppc64_cpu --smt=4", shell=True)
else:
test.cancel("Host SMT changes not allowed during guest live")
else:
logging.debug("No operation for the domain")
else:
if condn == "save":
save_file = os.path.join(data_dir.get_tmp_dir(), vm_name + ".save")
result = virsh.save(vm_name, save_file,
ignore_status=True, debug=True)
libvirt.check_exit_status(result)
time.sleep(condn_sleep_sec)
if os.path.exists(save_file):
result = virsh.restore(save_file, ignore_status=True,
debug=True)
libvirt.check_exit_status(result)
os.remove(save_file)
else:
test.error("No save file for domain restore")
elif condn == "managedsave":
result = virsh.managedsave(vm_name,
ignore_status=True, debug=True)
libvirt.check_exit_status(result)
time.sleep(condn_sleep_sec)
result = virsh.start(vm_name, ignore_status=True, debug=True)
libvirt.check_exit_status(result)
elif condn == "suspend":
result = virsh.resume(vm_name, ignore_status=True, debug=True)
libvirt.check_exit_status(result)
elif condn == "avocadotest":
guestbt.join(ignore_status=True)
elif condn == "stress":
utils_test.unload_stress("stress_in_vms", params=params, vms=[vm])
elif condn == "hotplug":
result = virsh.setvcpus(vm_name, current_vcpu, "--live",
ignore_status=True, debug=True)
libvirt.check_exit_status(result)
exp_vcpu = {'max_config': max_vcpu, 'max_live': current_vcpu,
'cur_config': current_vcpu, 'cur_live': current_vcpu,
'guest_live': current_vcpu}
result = cpu.check_vcpu_value(vm, exp_vcpu,
option="--live")
elif condn == "host_smt":
result = process.run("ppc64_cpu --smt=2", shell=True)
# Change back the host smt
result = process.run("ppc64_cpu --smt=4", shell=True)
# Work around due to known cgroup issue after cpu hot(un)plug
# sequence
root_cpuset_path = utils_cgroup.get_cgroup_mountpoint("cpuset")
machine_cpuset_paths = []
if os.path.isdir(os.path.join(root_cpuset_path,
"machine.slice")):
machine_cpuset_paths.append(os.path.join(root_cpuset_path,
"machine.slice"))
if os.path.isdir(os.path.join(root_cpuset_path, "machine")):
machine_cpuset_paths.append(os.path.join(root_cpuset_path,
"machine"))
if not machine_cpuset_paths:
logging.warning("cgroup cpuset might not recover properly "
"for guests after host smt changes, "
"restore it manually")
root_cpuset_cpus = os.path.join(root_cpuset_path, "cpuset.cpus")
for path in machine_cpuset_paths:
machine_cpuset_cpus = os.path.join(path, "cpuset.cpus")
# check if file content differs
cmd = "diff %s %s" % (root_cpuset_cpus,
machine_cpuset_cpus)
if process.system(cmd, verbose=True, ignore_status=True):
cmd = "cp %s %s" % (root_cpuset_cpus,
machine_cpuset_cpus)
process.system(cmd, verbose=True)
else:
logging.debug("No need recover the domain")
return bt
def copied_migration(test, vm, params, blockjob_type=None, block_target="vda"):
"""
Migrate vms with storage copied under some stress.
And during it, some qemu-monitor-command will be sent.
"""
dest_uri = params.get("migrate_dest_uri")
remote_host = params.get("migrate_dest_host")
copy_option = params.get("copy_storage_option", "")
username = params.get("remote_user")
password = params.get("migrate_dest_pwd")
timeout = int(params.get("thread_timeout", 1200))
options = "--live %s --unsafe" % copy_option
# Get vm ip for remote checking
if vm.is_dead():
vm.start()
vm.wait_for_login()
vms_ip = {}
vms_ip[vm.name] = vm.get_address()
logging.debug("VM %s IP: %s", vm.name, vms_ip[vm.name])
# Start to load stress
stress_type = params.get("migrate_stress_type")
if stress_type == "cpu":
params['stress_args'] = "--cpu 2 --quiet --timeout 60"
elif stress_type == "memory":
params['stress_args'] = "--vm 2 --vm-bytes 256M --vm-keep --timeout 60"
if stress_type is not None:
utils_test.load_stress("stress_in_vms", params=params, vms=[vm])
cp_mig = utlv.MigrationTest()
migration_thread = threading.Thread(target=cp_mig.thread_func_migration,
args=(vm, dest_uri, options))
migration_thread.start()
# Wait for migration launched
time.sleep(5)
job_ret = virsh.domjobinfo(vm.name, debug=True)
if job_ret.exit_status:
test.error("Prepare migration for blockjob failed.")
# Execute some qemu monitor commands
pause_cmd = "block-job-pause %s" % block_target
resume_cmd = "block-job-resume %s" % block_target
cancel_cmd = "block-job-cancel %s" % block_target
complete_cmd = "block-job-complete %s" % block_target
blockjob_failures = []
try:
if blockjob_type == "cancel":
virsh.qemu_monitor_command(vm.name, cancel_cmd, debug=True,
ignore_status=False)
elif blockjob_type == "pause_resume":
virsh.qemu_monitor_command(vm.name, pause_cmd, debug=True,
ignore_status=False)
# TODO: Check whether it is paused.
virsh.qemu_monitor_command(vm.name, resume_cmd, debug=True,
ignore_status=False)
elif blockjob_type == "complete":
virsh.qemu_monitor_command(vm.name, complete_cmd, debug=True,
ignore_status=False)
except process.CmdError as detail:
blockjob_failures.append(str(detail))
# Job info FYI
virsh.domjobinfo(vm.name, debug=True)
if len(blockjob_failures):
timeout = 30
migration_thread.join(timeout)
if migration_thread.isAlive():
logging.error("Migrate %s timeout.", migration_thread)
cp_mig.RET_LOCK.acquire()
cp_mig.RET_MIGRATION = False
cp_mig.RET_LOCK.release()
if len(blockjob_failures):
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
test.fail("Run qemu monitor command failed %s"
% blockjob_failures)
check_ip_failures = []
if cp_mig.RET_MIGRATION:
try:
utils_test.check_dest_vm_network(vm, vms_ip[vm.name],
remote_host, username,
password)
except exceptions.TestFail as detail:
check_ip_failures.append(str(detail))
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
test.fail("Storage migration passed even after "
"cancellation.")
else:
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
logging.error("Expected Migration Error for %s", blockjob_type)
return
else:
test.fail("Command blockjob does not work well under "
"storage copied migration.")
if len(check_ip_failures):
test.fail("Check IP failed:%s" % check_ip_failures)
def run(test, params, env):
"""
Test: vcpu hotplug.
The command can change the number of virtual CPUs for VM.
1.Prepare test environment,destroy or suspend a VM.
2.Perform virsh setvcpus operation.
3.Recover test environment.
4.Confirm the test result.
"""
vm_name = params.get("main_vm")
min_count = int(params.get("setvcpus_min_count", "1"))
max_count = int(params.get("setvcpus_max_count", "2"))
test_times = int(params.get("setvcpus_test_times", "1"))
stress_type = params.get("stress_type", "")
stress_param = params.get("stress_param", "")
add_by_virsh = ("yes" == params.get("add_by_virsh"))
del_by_virsh = ("yes" == params.get("del_by_virsh"))
hotplug_timeout = int(params.get("hotplug_timeout", 30))
test_set_max = max_count * 2
# Save original configuration
orig_config_xml = libvirt_xml.VMXML.new_from_inactive_dumpxml(vm_name)
# Set min/max of vcpu
libvirt_xml.VMXML.set_vm_vcpus(vm_name, test_set_max, min_count,
topology_correction=True)
# prepare VM instance
vm = libvirt_vm.VM(vm_name, params, test.bindir, env.get("address_cache"))
# prepare guest-agent service
vm.prepare_guest_agent()
# Increase the workload
load_vms = []
if stress_type in ['cpu', 'memory', 'io']:
params["stress_args"] = stress_param
load_vms.append(vm)
if stress_type in ['cpu', 'memory']:
utils_test.load_stress("stress_in_vms", params, vms=load_vms)
else:
utils_test.load_stress("iozone_in_vms", params, vms=load_vms)
session = vm.wait_for_login()
try:
# Clear dmesg before set vcpu
session.cmd("dmesg -c")
for i in range(test_times):
# 1. Add vcpu
add_result = libvirt.hotplug_domain_vcpu(vm,
max_count,
add_by_virsh)
add_status = add_result.exit_status
# 1.1 check add status
if add_status:
if add_result.stderr.count("support"):
test.cancel("vcpu hotplug not supported, "
"no need to test any more:\n %s"
% add_result.stderr.strip())
test.fail("Test failed for:\n %s"
% add_result.stderr.strip())
if not utils_misc.wait_for(lambda: utils_misc.check_if_vm_vcpu_match(max_count, vm),
hotplug_timeout,
text="wait for vcpu online"):
test.fail("vcpu hotplug failed")
if 'ppc' not in platform.machine():
# 1.2 check dmesg
domain_add_dmesg = session.cmd_output("dmesg -c")
dmesg1 = "CPU%d has been hot-added" % (max_count - 1)
dmesg2 = "CPU %d got hotplugged" % (max_count - 1)
if (not domain_add_dmesg.count(dmesg1) and
not domain_add_dmesg.count(dmesg2)):
test.fail("Cannot find hotplug info in dmesg: %s"
% domain_add_dmesg)
# 1.3 check cpu related file
online_cmd = "cat /sys/devices/system/cpu/cpu%d/online" \
% (max_count - 1)
st, ot = session.cmd_status_output(online_cmd)
if st:
test.fail("Cannot find CPU%d after hotplug"
% (max_count - 1))
# 1.4 check online
if not ot.strip().count("1"):
test.fail("CPU%d is not online after hotplug: %s"
% ((max_count - 1), ot))
# 1.5 check online interrupts info
inter_on_output = session.cmd_output("cat /proc/interrupts")
if not inter_on_output.count("CPU%d" % (int(max_count) - 1)):
test.fail("CPU%d can not be found in "
"/proc/interrupts when it's online:%s"
% ((int(max_count) - 1), inter_on_output))
# 1.6 offline vcpu
off_st = session.cmd_status("echo 0 > "
"/sys/devices/system/cpu/cpu%d/online"
% (max_count - 1))
if off_st:
test.fail("Set cpu%d offline failed!"
% (max_count - 1))
# 1.7 check offline interrupts info
inter_off_output = session.cmd_output("cat /proc/interrupts")
if inter_off_output.count("CPU%d" % (int(max_count) - 1)):
test.fail("CPU%d can be found in /proc/interrupts"
" when it's offline"
% (int(max_count) - 1))
# 1.8 online vcpu
on_st = session.cmd_status("echo 1 > "
"/sys/devices/system/cpu/cpu%d/online"
% (max_count - 1))
if on_st:
test.fail("Set cpu%d online failed!"
% (max_count - 1))
# 2. Del vcpu
del_result = libvirt.hotplug_domain_vcpu(vm,
min_count,
del_by_virsh,
hotplug=False)
del_status = del_result.exit_status
if del_status:
logging.info("del_result: %s" % del_result.stderr.strip())
# A qemu older than 1.5 or an unplug for 1.6 will result in
# the following failure.
# TODO: when CPU-hotplug feature becomes stable and strong,
# remove these codes used to handle kinds of exceptions
if re.search("The command cpu-del has not been found",
del_result.stderr):
test.cancel("vcpu hotunplug not supported")
if re.search("cannot change vcpu count", del_result.stderr):
test.cancel("unhotplug failed")
if re.search("got wrong number of vCPU pids from QEMU monitor",
del_result.stderr):
test.cancel("unhotplug failed")
# process all tips that contains keyword 'support'
# for example, "unsupported"/"hasn't been support" and so on
if re.search("support", del_result.stderr):
test.cancel("vcpu hotunplug not supported")
# besides above, regard it failed
test.fail("Test fail for:\n %s"
% del_result.stderr.strip())
if not utils_misc.wait_for(lambda: utils_misc.check_if_vm_vcpu_match(min_count, vm),
hotplug_timeout,
text="wait for vcpu offline"):
test.fail("vcpu hotunplug failed")
if 'ppc' not in platform.machine():
domain_del_dmesg = session.cmd_output("dmesg -c")
if not domain_del_dmesg.count("CPU %d is now offline"
% (max_count - 1)):
test.fail("Cannot find hot-unplug info in dmesg: %s"
% domain_del_dmesg)
except exceptions.TestCancel:
# So far, QEMU doesn't support unplug vcpu,
# unplug operation will encounter kind of errors.
pass
finally:
utils_test.unload_stress("stress_in_vms", params, load_vms)
if session:
session.close()
# Cleanup
orig_config_xml.sync()
def set_condition(vm_name, condn, reset=False, guestbt=None):
"""
Set domain to given state or reset it.
"""
bt = None
if not reset:
if condn == "avocadotest":
bt = utils_test.run_avocado_bg(vm, params, test)
if not bt:
test.cancel("guest stress failed to start")
# Allow stress to start
time.sleep(condn_sleep_sec)
return bt
elif condn == "stress":
utils_test.load_stress("stress_in_vms", params=params, vms=[vm])
elif condn in ["save", "managedsave"]:
# No action
pass
elif condn == "suspend":
result = virsh.suspend(vm_name, ignore_status=True, debug=True)
libvirt.check_exit_status(result)
elif condn == "hotplug":
result = virsh.setvcpus(vm_name, max_vcpu, "--live",
ignore_status=True, debug=True)
libvirt.check_exit_status(result)
exp_vcpu = {'max_config': max_vcpu, 'max_live': max_vcpu,
'cur_config': current_vcpu, 'cur_live': max_vcpu,
'guest_live': max_vcpu}
result = utils_hotplug.check_vcpu_value(vm, exp_vcpu,
option="--live")
elif condn == "host_smt":
if cpu.get_cpu_arch() == 'power9':
result = process.run("ppc64_cpu --smt=4", shell=True)
else:
test.cancel("Host SMT changes not allowed during guest live")
else:
logging.debug("No operation for the domain")
else:
if condn == "save":
save_file = os.path.join(data_dir.get_tmp_dir(), vm_name + ".save")
result = virsh.save(vm_name, save_file,
ignore_status=True, debug=True)
libvirt.check_exit_status(result)
time.sleep(condn_sleep_sec)
if os.path.exists(save_file):
result = virsh.restore(save_file, ignore_status=True,
debug=True)
libvirt.check_exit_status(result)
os.remove(save_file)
else:
test.error("No save file for domain restore")
elif condn == "managedsave":
result = virsh.managedsave(vm_name,
ignore_status=True, debug=True)
libvirt.check_exit_status(result)
time.sleep(condn_sleep_sec)
result = virsh.start(vm_name, ignore_status=True, debug=True)
libvirt.check_exit_status(result)
elif condn == "suspend":
result = virsh.resume(vm_name, ignore_status=True, debug=True)
libvirt.check_exit_status(result)
elif condn == "avocadotest":
guestbt.join(ignore_status=True)
elif condn == "stress":
utils_test.unload_stress("stress_in_vms", params=params, vms=[vm])
elif condn == "hotplug":
result = virsh.setvcpus(vm_name, current_vcpu, "--live",
ignore_status=True, debug=True)
libvirt.check_exit_status(result)
exp_vcpu = {'max_config': max_vcpu, 'max_live': current_vcpu,
'cur_config': current_vcpu, 'cur_live': current_vcpu,
'guest_live': current_vcpu}
result = utils_hotplug.check_vcpu_value(vm, exp_vcpu,
option="--live")
elif condn == "host_smt":
result = process.run("ppc64_cpu --smt=2", shell=True)
# Change back the host smt
result = process.run("ppc64_cpu --smt=4", shell=True)
# Work around due to known cgroup issue after cpu hot(un)plug
# sequence
root_cpuset_path = utils_cgroup.get_cgroup_mountpoint("cpuset")
machine_cpuset_paths = []
if os.path.isdir(os.path.join(root_cpuset_path,
"machine.slice")):
machine_cpuset_paths.append(os.path.join(root_cpuset_path,
"machine.slice"))
if os.path.isdir(os.path.join(root_cpuset_path, "machine")):
machine_cpuset_paths.append(os.path.join(root_cpuset_path,
"machine"))
if not machine_cpuset_paths:
logging.warning("cgroup cpuset might not recover properly "
"for guests after host smt changes, "
"restore it manually")
root_cpuset_cpus = os.path.join(root_cpuset_path, "cpuset.cpus")
for path in machine_cpuset_paths:
machine_cpuset_cpus = os.path.join(path, "cpuset.cpus")
# check if file content differs
cmd = "diff %s %s" % (root_cpuset_cpus,
machine_cpuset_cpus)
if process.system(cmd, verbose=True, ignore_status=True):
cmd = "cp %s %s" % (root_cpuset_cpus,
machine_cpuset_cpus)
process.system(cmd, verbose=True)
else:
logging.debug("No need recover the domain")
return bt
def run(test, params, env):
"""
Stress test for the hotplug feature of serial device
"""
vm_name = params.get("main_vm", "vm1")
char_dev = params.get("char_dev", "file")
hotplug_type = params.get("hotplug_type", "qmp")
load_type = params.get("load_type", "")
load_params = params.get("load_params", "")
test_count = int(params.get("test_count", 5))
test_type = params.get("test_type", "multi")
tmp_dir = os.path.join(test.tmpdir, "hotplug_serial_load")
if not os.path.exists(tmp_dir):
os.mkdir(tmp_dir)
os.chmod(tmp_dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
load_vms = []
if load_type in ['cpu', 'memory', 'io']:
params["stress_args"] = load_params
load_vms.append(
libvirt_vm.VM(vm_name, params, test.bindir, env.get("address_cache")))
vm = env.get_vm(vm_name)
session = vm.wait_for_login()
def prepare_channel_xml(to_file, char_type, index=1, id=0):
params = {}
mode = ''
if char_type == "file":
channel_type = char_type
channel_path = ("%s/%s%s" % (tmp_dir, char_type, index))
elif char_type == "socket":
channel_type = 'unix'
channel_path = ("%s/%s%s" % (tmp_dir, char_type, index))
mode = 'bind'
elif char_type == "pty":
channel_type = char_type
channel_path = ("/dev/pts/%s" % id)
params = {
'channel_type_name': channel_type,
'source_path': channel_path,
'source_mode': mode,
'target_type': 'virtio',
'target_name': char_type + str(index)
}
channel_xml = utlv.create_channel_xml(params, alias=True, address=True)
shutil.copyfile(channel_xml.xml, to_file)
def hotplug_device(hotplug_type, char_dev, index=1, id=0):
if hotplug_type == "qmp":
char_add_opt = "chardev-add "
dev_add_opt = "device_add virtserialport,chardev="
if char_dev == "file":
char_add_opt += ("file,path=%s/file%s,id=file%s" %
(tmp_dir, index, index))
dev_add_opt += (
"file%s,name=file%s,bus=virtio-serial0.0,id=file%s" %
(index, index, index))
elif char_dev == "socket":
char_add_opt += (
"socket,path=%s/socket%s,server,nowait,id=socket%s" %
(tmp_dir, index, index))
dev_add_opt += (
"socket%s,name=socket%s,bus=virtio-serial0.0,id=socket%s" %
(index, index, index))
elif char_dev == "pty":
char_add_opt += "pty,path=/dev/pts/%s,id=pty%s" % (id, index)
dev_add_opt += (
"pty%s,name=pty%s,bus=virtio-serial0.0,id=pty%s" %
(index, index, index))
virsh.qemu_monitor_command(vm_name, char_add_opt, "--hmp")
virsh.qemu_monitor_command(vm_name, dev_add_opt, "--hmp")
elif hotplug_type == "attach":
xml_file = "%s/xml_%s%s" % (tmp_dir, char_dev, index)
if char_dev in ["file", "socket"]:
prepare_channel_xml(xml_file, char_dev, index)
elif char_dev == "pty":
prepare_channel_xml(xml_file, char_dev, index, id)
virsh.attach_device(vm_name, xml_file, flagstr="--live")
def confirm_hotplug_result(char_dev, index=1, id=0):
result = virsh.qemu_monitor_command(vm_name, "info qtree", "--hmp")
h_o = result.stdout.strip()
chardev_c = h_o.count("chardev = %s%s" % (char_dev, index))
name_c = h_o.count("name = \"%s%s\"" % (char_dev, index))
if chardev_c == 0 and name_c == 0:
raise error.TestFail("Cannot get serial device info: '%s'" % h_o)
tmp_file = "%s/%s%s" % (tmp_dir, char_dev, index)
serial_file = "/dev/virtio-ports/%s%s" % (char_dev, index)
if char_dev == "file":
session.cmd("echo test > %s" % serial_file)
f = open(tmp_file, "r")
output = f.read()
f.close()
elif char_dev == "socket":
session.cmd("echo test > /tmp/file")
sock = socket.socket(socket.AF_UNIX)
sock.connect(tmp_file)
session.cmd("dd if=/tmp/file of=%s" % serial_file)
output = sock.recv(1024)
sock.close()
elif char_dev == "pty":
session.cmd("echo test > /tmp/file")
session.cmd("dd if=/tmp/file of=%s &" % serial_file)
dev_file = "/dev/pts/%s" % id
if not os.path.exists(dev_file):
raise error.TestFail("%s doesn't exist." % dev_file)
p = subprocess.Popen(["/usr/bin/cat", dev_file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
while True:
output = p.stdout.readline()
if output or p.poll():
break
time.sleep(0.2)
p.kill()
if not output.count("test"):
err_info = "%s device file doesn't match 'test':%s" % (char_dev,
output)
raise error.TestFail(err_info)
def unhotplug_serial_device(hotplug_type, char_dev, index=1):
if hotplug_type == "qmp":
del_dev_opt = "device_del %s%s" % (char_dev, index)
del_char_opt = "chardev-remove %s%s" % (char_dev, index)
virsh.qemu_monitor_command(vm_name, del_dev_opt, "--hmp")
virsh.qemu_monitor_command(vm_name, del_char_opt, "--hmp")
elif hotplug_type == "attach":
xml_file = "%s/xml_%s%s" % (tmp_dir, char_dev, index)
virsh.detach_device(vm_name, xml_file, flagstr="--live")
def confirm_unhotplug_result(char_dev, index=1):
serial_file = "/dev/virtio-ports/%s%s" % (char_dev, index)
result = virsh.qemu_monitor_command(vm_name, "info qtree", "--hmp")
uh_o = result.stdout.strip()
if uh_o.count("chardev = %s%s" % (char_dev, index)):
raise error.TestFail("Still can get serial device info: '%s'" %
uh_o)
if not session.cmd_status("test -e %s" % serial_file):
raise error.TestFail("File '%s' still exists after unhotplug" %
serial_file)
# run test case
try:
# increase workload
if load_type in ['cpu', 'memory']:
utils_test.load_stress("stress_in_vms", load_vms, params)
else:
utils_test.load_stress("iozone_in_vms", load_vms, params)
if test_type == "multi":
for i in range(test_count):
if char_dev == "pty":
ptsid = utils_misc.aton(
utils_misc.get_dev_pts_max_id()) + 1
else:
ptsid = 0
hotplug_device(hotplug_type, char_dev, i + 1, id=ptsid)
confirm_hotplug_result(char_dev, i + 1, id=ptsid)
unhotplug_serial_device(hotplug_type, char_dev, i + 1)
confirm_unhotplug_result(char_dev, i + 1)
elif test_type == "circle":
if char_dev != "all":
for i in range(test_count):
if char_dev == "pty":
ptsid = utils_misc.aton(
utils_misc.get_dev_pts_max_id()) + 1
else:
ptsid = 0
#1.hotplug serial device
hotplug_device(hotplug_type, char_dev, id=ptsid)
#2.confirm hotplug result
confirm_hotplug_result(char_dev, id=ptsid)
#3.unhotplug serial device
unhotplug_serial_device(hotplug_type, char_dev)
#4.confirm unhotplug result
confirm_unhotplug_result(char_dev)
else:
for i in range(test_count):
#1.hotplug serial device
hotplug_device(hotplug_type, "file")
hotplug_device(hotplug_type, "socket")
ptsid = utils_misc.aton(
utils_misc.get_dev_pts_max_id()) + 1
hotplug_device(hotplug_type, "pty", id=ptsid)
#2.confirm hotplug result
confirm_hotplug_result("file")
confirm_hotplug_result("socket")
confirm_hotplug_result("pty", id=ptsid)
#3.unhotplug serial device
unhotplug_serial_device(hotplug_type, "file")
unhotplug_serial_device(hotplug_type, "socket")
unhotplug_serial_device(hotplug_type, "pty")
#4.confirm unhotplug result
confirm_unhotplug_result("file")
confirm_unhotplug_result("socket")
confirm_unhotplug_result("pty")
finally:
session.close()
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def run(test, params, env):
"""
Stress test for the hotplug feature of serial device
"""
vm_name = params.get("main_vm", "vm1")
status_error = "yes" == params.get("status_error", "no")
char_dev = params.get("char_dev", "file")
hotplug_type = params.get("hotplug_type", "qmp")
load_type = params.get("load_type", "")
load_params = params.get("load_params", "")
test_count = int(params.get("test_count", 5))
test_type = params.get("test_type", "multi")
tmp_dir = os.path.join(test.tmpdir, "hotplug_serial_load")
if not os.path.exists(tmp_dir):
os.mkdir(tmp_dir)
os.chmod(tmp_dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
load_vms = []
if load_type in ['cpu', 'memory', 'io']:
params["stress_args"] = load_params
load_vms.append(libvirt_vm.VM(vm_name, params, test.bindir,
env.get("address_cache")))
vm = env.get_vm(vm_name)
session = vm.wait_for_login()
def prepare_channel_xml(to_file, char_type, index=1, id=0):
params = {}
mode = ''
if char_type == "file":
channel_type = char_type
channel_path = ("%s/%s%s" % (tmp_dir, char_type, index))
elif char_type == "socket":
channel_type = 'unix'
channel_path = ("%s/%s%s" % (tmp_dir, char_type, index))
mode = 'bind'
elif char_type == "pty":
channel_type = char_type
channel_path = ("/dev/pts/%s" % id)
params = {'channel_type_name': channel_type,
'source_path': channel_path,
'source_mode': mode,
'target_type': 'virtio',
'target_name': char_type + str(index)}
channel_xml = utlv.create_channel_xml(params, alias=True, address=True)
shutil.copyfile(channel_xml.xml, to_file)
def hotplug_device(hotplug_type, char_dev, index=1, id=0):
if hotplug_type == "qmp":
char_add_opt = "chardev-add "
dev_add_opt = "device_add virtserialport,chardev="
if char_dev == "file":
char_add_opt += ("file,path=%s/file%s,id=file%s"
% (tmp_dir, index, index))
dev_add_opt += ("file%s,name=file%s,bus=virtio-serial0.0,id=file%s"
% (index, index, index))
elif char_dev == "socket":
char_add_opt += ("socket,path=%s/socket%s,server,nowait,id=socket%s"
% (tmp_dir, index, index))
dev_add_opt += ("socket%s,name=socket%s,bus=virtio-serial0.0,id=socket%s"
% (index, index, index))
elif char_dev == "pty":
char_add_opt += "pty,path=/dev/pts/%s,id=pty%s" % (id, index)
dev_add_opt += ("pty%s,name=pty%s,bus=virtio-serial0.0,id=pty%s"
% (index, index, index))
virsh.qemu_monitor_command(vm_name, char_add_opt, "--hmp")
virsh.qemu_monitor_command(vm_name, dev_add_opt, "--hmp")
elif hotplug_type == "attach":
xml_file = "%s/xml_%s%s" % (tmp_dir, char_dev, index)
if char_dev in ["file", "socket"]:
prepare_channel_xml(xml_file, char_dev, index)
elif char_dev == "pty":
prepare_channel_xml(xml_file, char_dev, index, id)
virsh.attach_device(vm_name, xml_file, flagstr="--live")
def confirm_hotplug_result(char_dev, index=1, id=0):
result = virsh.qemu_monitor_command(vm_name, "info qtree", "--hmp")
h_o = result.stdout.strip()
chardev_c = h_o.count("chardev = %s%s" % (char_dev, index))
name_c = h_o.count("name = \"%s%s\"" % (char_dev, index))
if chardev_c == 0 and name_c == 0:
raise error.TestFail("Cannot get serial device info: '%s'" % h_o)
tmp_file = "%s/%s%s" % (tmp_dir, char_dev, index)
serial_file = "/dev/virtio-ports/%s%s" % (char_dev, index)
if char_dev == "file":
session.cmd("echo test > %s" % serial_file)
f = open(tmp_file, "r")
output = f.read()
f.close()
elif char_dev == "socket":
session.cmd("echo test > /tmp/file")
sock = socket.socket(socket.AF_UNIX)
sock.connect(tmp_file)
session.cmd("dd if=/tmp/file of=%s" % serial_file)
output = sock.recv(1024)
sock.close()
elif char_dev == "pty":
session.cmd("echo test > /tmp/file")
session.cmd("dd if=/tmp/file of=%s &" % serial_file)
dev_file = "/dev/pts/%s" % id
if not os.path.exists(dev_file):
raise error.TestFail("%s doesn't exist." % dev_file)
p = subprocess.Popen(["/usr/bin/cat", dev_file],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while True:
output = p.stdout.readline()
if output or p.poll():
break
time.sleep(0.2)
p.kill()
if not output.count("test"):
err_info = "%s device file doesn't match 'test':%s" % (char_dev, output)
raise error.TestFail(err_info)
def unhotplug_serial_device(hotplug_type, char_dev, index=1):
if hotplug_type == "qmp":
del_dev_opt = "device_del %s%s" % (char_dev, index)
del_char_opt = "chardev-remove %s%s" % (char_dev, index)
virsh.qemu_monitor_command(vm_name, del_dev_opt, "--hmp")
virsh.qemu_monitor_command(vm_name, del_char_opt, "--hmp")
elif hotplug_type == "attach":
xml_file = "%s/xml_%s%s" % (tmp_dir, char_dev, index)
virsh.detach_device(vm_name, xml_file, flagstr="--live")
def confirm_unhotplug_result(char_dev, index=1):
serial_file = "/dev/virtio-ports/%s%s" % (char_dev, index)
result = virsh.qemu_monitor_command(vm_name, "info qtree", "--hmp")
uh_o = result.stdout.strip()
if uh_o.count("chardev = %s%s" % (char_dev, index)):
raise error.TestFail("Still can get serial device info: '%s'" % uh_o)
if not session.cmd_status("test -e %s" % serial_file):
raise error.TestFail("File '%s' still exists after unhotplug" % serial_file)
# run test case
try:
# increase workload
if load_type in ['cpu', 'memory']:
utils_test.load_stress("stress_in_vms", load_vms, params)
else:
utils_test.load_stress("iozone_in_vms", load_vms, params)
if test_type == "multi":
for i in range(test_count):
if char_dev == "pty":
ptsid = utils_misc.aton(utils_misc.get_dev_pts_max_id()) + 1
else:
ptsid = 0
hotplug_device(hotplug_type, char_dev, i + 1, id=ptsid)
confirm_hotplug_result(char_dev, i + 1, id=ptsid)
unhotplug_serial_device(hotplug_type, char_dev, i + 1)
confirm_unhotplug_result(char_dev, i + 1)
elif test_type == "circle":
if char_dev != "all":
for i in range(test_count):
if char_dev == "pty":
ptsid = utils_misc.aton(utils_misc.get_dev_pts_max_id()) + 1
else:
ptsid = 0
#1.hotplug serial device
hotplug_device(hotplug_type, char_dev, id=ptsid)
#2.confirm hotplug result
confirm_hotplug_result(char_dev, id=ptsid)
#3.unhotplug serial device
unhotplug_serial_device(hotplug_type, char_dev)
#4.confirm unhotplug result
confirm_unhotplug_result(char_dev)
else:
for i in range(test_count):
#1.hotplug serial device
hotplug_device(hotplug_type, "file")
hotplug_device(hotplug_type, "socket")
ptsid = utils_misc.aton(utils_misc.get_dev_pts_max_id()) + 1
hotplug_device(hotplug_type, "pty", id=ptsid)
#2.confirm hotplug result
confirm_hotplug_result("file")
confirm_hotplug_result("socket")
confirm_hotplug_result("pty", id=ptsid)
#3.unhotplug serial device
unhotplug_serial_device(hotplug_type, "file")
unhotplug_serial_device(hotplug_type, "socket")
unhotplug_serial_device(hotplug_type, "pty")
#4.confirm unhotplug result
confirm_unhotplug_result("file")
confirm_unhotplug_result("socket")
confirm_unhotplug_result("pty")
finally:
session.close()
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def verify_migration_speed(test, params, env):
"""
Check if migration speed is effective with twice migration.
"""
vms = env.get_all_vms()
src_uri = params.get("migrate_src_uri", "qemu+ssh://EXAMPLE/system")
dest_uri = params.get("migrate_dest_uri", "qemu+ssh://EXAMPLE/system")
if not len(vms):
test.cancel("Please provide migrate_vms for test.")
if src_uri.count('///') or src_uri.count('EXAMPLE'):
test.cancel("The src_uri '%s' is invalid" % src_uri)
if dest_uri.count('///') or dest_uri.count('EXAMPLE'):
test.cancel("The dest_uri '%s' is invalid" % dest_uri)
remote_host = params.get("migrate_dest_host")
username = params.get("migrate_dest_user", "root")
password = params.get("migrate_dest_pwd")
# Config ssh autologin for remote host
ssh_key.setup_ssh_key(remote_host, username, password, port=22)
# Check migrated vms' state
for vm in vms:
if vm.is_dead():
vm.start()
load_vm_names = params.get("load_vms").split()
# vms for load
load_vms = []
for vm_name in load_vm_names:
load_vms.append(
libvirt_vm.VM(vm_name, params, test.bindir,
env.get("address_cache")))
params["load_vms"] = load_vms
bandwidth = int(params.get("bandwidth", "4"))
stress_type = params.get("stress_type", "load_vms_booting")
migration_type = params.get("migration_type", "orderly")
thread_timeout = int(params.get("thread_timeout", "60"))
delta = float(params.get("allowed_delta", "0.1"))
virsh_migrate_timeout = int(params.get("virsh_migrate_timeout", "60"))
# virsh migrate options
virsh_migrate_options = "--live --unsafe --timeout %s" % virsh_migrate_timeout
# Migrate vms to remote host
mig_first = utlv.MigrationTest()
virsh_dargs = {"debug": True}
for vm in vms:
set_get_speed(vm.name, bandwidth, virsh_dargs=virsh_dargs)
vm.wait_for_login()
utils_test.load_stress(stress_type, vms, params)
mig_first.do_migration(vms,
src_uri,
dest_uri,
migration_type,
options=virsh_migrate_options,
thread_timeout=thread_timeout)
for vm in vms:
mig_first.cleanup_dest_vm(vm, None, dest_uri)
# Keep it clean for second migration
if vm.is_alive():
vm.destroy()
# Migrate vms again with new bandwidth
second_bandwidth = params.get("second_bandwidth", "times")
if second_bandwidth == "half":
second_bandwidth = bandwidth / 2
speed_times = 2
elif second_bandwidth == "times":
second_bandwidth = bandwidth * 2
speed_times = 0.5
elif second_bandwidth == "same":
second_bandwidth = bandwidth
speed_times = 1
# Migrate again
for vm in vms:
if vm.is_dead():
vm.start()
vm.wait_for_login()
set_get_speed(vm.name, second_bandwidth, virsh_dargs=virsh_dargs)
utils_test.load_stress(stress_type, vms, params)
mig_second = utlv.MigrationTest()
mig_second.do_migration(vms,
src_uri,
dest_uri,
migration_type,
options=virsh_migrate_options,
thread_timeout=thread_timeout)
for vm in vms:
mig_second.cleanup_dest_vm(vm, None, dest_uri)
fail_info = []
# Check whether migration failed
if len(fail_info):
test.fail(fail_info)
for vm in vms:
first_time = mig_first.mig_time[vm.name]
second_time = mig_second.mig_time[vm.name]
logging.debug(
"Migration time for %s:\n"
"Time with Bandwidth '%s' first: %s\n"
"Time with Bandwidth '%s' second: %s", vm.name, bandwidth,
first_time, second_bandwidth, second_time)
shift = float(abs(first_time * speed_times -
second_time)) / float(second_time)
logging.debug("Shift:%s", shift)
if delta < shift:
fail_info.append(
"Spent time for migrating %s is intolerable." % vm.name)
# Check again for speed result
if len(fail_info):
test.fail(fail_info)
def verify_migration_speed(test, params, env):
"""
Check if migration speed is effective with twice migration.
"""
vms = env.get_all_vms()
src_uri = params.get("migrate_src_uri", "qemu+ssh://EXAMPLE/system")
dest_uri = params.get("migrate_dest_uri", "qemu+ssh://EXAMPLE/system")
if src_uri.count('///') or src_uri.count('EXAMPLE'):
raise error.TestNAError("The src_uri '%s' is invalid", src_uri)
if dest_uri.count('///') or dest_uri.count('EXAMPLE'):
raise error.TestNAError("The dest_uri '%s' is invalid", dest_uri)
remote_host = params.get("migrate_dest_host")
username = params.get("migrate_dest_user", "root")
password = params.get("migrate_dest_pwd")
# Config ssh autologin for remote host
ssh_key.setup_ssh_key(remote_host, username, password, port=22)
# Check migrated vms' state
for vm in vms:
if vm.is_dead():
vm.start()
load_vm_names = params.get("load_vms").split()
# vms for load
load_vms = []
for vm_name in load_vm_names:
load_vms.append(libvirt_vm.VM(vm_name, params, test.bindir,
env.get("address_cache")))
bandwidth = int(params.get("bandwidth", "4"))
stress_type = params.get("stress_type", "load_vms_booting")
migration_type = params.get("migration_type", "orderly")
thread_timeout = int(params.get("thread_timeout", "60"))
delta = float(params.get("allowed_delta", "0.1"))
virsh_migrate_timeout = int(params.get("virsh_migrate_timeout", 60))
# virsh migrate options
virsh_migrate_options = "--live --timeout %s", virsh_migrate_timeout
# Migrate vms to remote host
mig_first = utlv.MigrationTest()
virsh_dargs = {"debug": True}
for vm in vms:
set_get_speed(vm.name, bandwidth, virsh_dargs=virsh_dargs)
vm.wait_for_login()
utils_test.load_stress(stress_type, vms, params)
mig_first.do_migration(vms, src_uri, dest_uri, migration_type,
options=virsh_migrate_options, thread_timeout=thread_timeout)
for vm in vms:
mig_first.cleanup_dest_vm(vm, None, dest_uri)
# Keep it clean for second migration
if vm.is_alive():
vm.destroy()
# Migrate vms again with new bandwidth
second_bandwidth = params.get("second_bandwidth", "times")
if second_bandwidth == "half":
second_bandwidth = bandwidth / 2
speed_times = 2
elif second_bandwidth == "times":
second_bandwidth = bandwidth * 2
speed_times = 0.5
elif second_bandwidth == "same":
second_bandwidth = bandwidth
speed_times = 1
# Migrate again
for vm in vms:
if vm.is_dead():
vm.start()
vm.wait_for_login()
set_get_speed(vm.name, second_bandwidth, virsh_dargs=virsh_dargs)
utils_test.load_stress(stress_type, vms, params)
mig_second = utlv.MigrationTest()
mig_second.do_migration(vms, src_uri, dest_uri, migration_type,
options="--live", thread_timeout=thread_timeout)
for vm in vms:
mig_second.cleanup_dest_vm(vm, None, dest_uri)
fail_info = []
# Check whether migration failed
if len(fail_info):
raise error.TestFail(fail_info)
for vm in vms:
first_time = mig_first.mig_time[vm.name]
second_time = mig_second.mig_time[vm.name]
logging.debug("Migration time for %s:\n"
"Time with Bandwidth '%s' first: %s\n"
"Time with Bandwidth '%s' second: %s", vm.name,
bandwidth, first_time, second_bandwidth, second_time)
shift = float(abs(first_time * speed_times - second_time)) / float(second_time)
logging.debug("Shift:%s", shift)
if delta < shift:
fail_info.append("Spent time for migrating %s is intolerable." % vm.name)
# Check again for speed result
if len(fail_info):
raise error.TestFail(fail_info)
def run(test, params, env):
"""
Test migration under stress.
"""
vm_names = params.get("vms").split()
if len(vm_names) < 2:
test.cancel("Provide enough vms for migration")
src_uri = "qemu:///system"
dest_uri = libvirt_vm.complete_uri(
params.get("migrate_dest_host", "EXAMPLE"))
if dest_uri.count('///') or dest_uri.count('EXAMPLE'):
test.cancel("The dest_uri '%s' is invalid" % dest_uri)
# Migrated vms' instance
vms = env.get_all_vms()
params["load_vms"] = list(vms)
cpu = int(params.get("smp", 1))
memory = int(params.get("mem")) * 1024
stress_tool = params.get("stress_tool", "")
stress_type = params.get("migration_stress_type")
require_stress_tool = "stress" in stress_tool
vm_bytes = params.get("stress_vm_bytes", "128M")
stress_args = params.get("stress_args")
migration_type = params.get("migration_type")
start_migration_vms = params.get("start_migration_vms", "yes") == "yes"
thread_timeout = int(params.get("thread_timeout", 120))
# Set vm_bytes for start_cmd
mem_total = utils_memory.memtotal()
vm_reserved = len(vms) * memory
if vm_bytes == "half":
vm_bytes = (mem_total - vm_reserved) / 2
elif vm_bytes == "shortage":
vm_bytes = mem_total - vm_reserved + 524288
if "vm-bytes" in stress_args:
params["stress_args"] = stress_args % vm_bytes
# Ensure stress tool is available in host
if require_stress_tool and stress_type == "stress_on_host":
utils_test.load_stress("stress_on_host", params)
for vm in vms:
# Keep vm dead for edit
if vm.is_alive():
vm.destroy()
set_cpu_memory(vm.name, cpu, memory)
try:
if start_migration_vms:
for vm in vms:
vm.start()
vm.wait_for_login()
# configure stress in VM
if require_stress_tool and stress_type == "stress_in_vms":
utils_test.load_stress("stress_in_vms", params, vms)
do_stress_migration(vms, src_uri, dest_uri, migration_type, test,
params, thread_timeout)
finally:
logging.debug("Cleanup vms...")
params["connect_uri"] = src_uri
for vm in vms:
utils_test.libvirt.MigrationTest().cleanup_dest_vm(
vm, None, dest_uri)
# Try to start vms in source once vms in destination are
# cleaned up
if not vm.is_alive():
vm.start()
vm.wait_for_login()
utils_test.unload_stress(stress_type, params, vms)
