def run(test, params, env):
"""
Test hotplug/hotunplug of vcpu device.
1) Boot up guest w/o vcpu device.
2) Hot plug/unplug vcpu devices and check successfully or not. (qemu side)
3) Check if the number of CPUs in guest changes accordingly. (guest side)
4) Do sub test after hot plug/unplug.
5) Recheck the number of CPUs in guest.
6) Check the CPU topology of guest. (if all vcpu plugged)
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def check_guest_cpu_count():
if not utils_misc.wait_for(
lambda: cpu_utils.check_if_vm_vcpus_match_qemu(vm),
verify_wait_timeout, first=sleep_after_change):
test.fail("Actual number of guest CPUs is not equal to expected")
def sub_hotunplug():
error_context.context("Hotunplug vcpu devices after vcpu %s"
% hotpluggable_test, logging.info)
for plugged_dev in pluggable_vcpu_dev[::-1]:
try:
vm.hotunplug_vcpu_device(plugged_dev)
except VMDeviceCheckError:
if not vm.is_paused():
raise
logging.warning("%s can not be unplugged directly because "
"guest is paused, will check again after "
"resume", plugged_dev)
vm.params["vcpu_enable_%s" % plugged_dev] = "no"
def sub_reboot():
error_context.context("Reboot guest after vcpu %s"
% hotpluggable_test, logging.info)
vm.reboot(session=session, method=params["reboot_method"],
timeout=login_timeout)
def sub_shutdown():
error_context.context("Shutdown guest after vcpu %s"
% hotpluggable_test, logging.info)
shutdown_method = params["shutdown_method"]
if shutdown_method == "shell":
session.sendline(params["shutdown_command"])
error_context.context("waiting VM to go down (guest shell cmd)",
logging.info)
elif shutdown_method == "system_powerdown":
vm.monitor.system_powerdown()
error_context.context("waiting VM to go down (qemu monitor cmd)",
logging.info)
if not vm.wait_for_shutdown(360):
test.fail("Guest refuses to go down after vcpu %s"
% hotpluggable_test)
def sub_migrate():
sub_migrate_reboot = sub_reboot
sub_migrate_hotunplug = sub_hotunplug
error_context.context("Migrate guest after vcpu %s"
% hotpluggable_test, logging.info)
vm.migrate()
vm.verify_alive()
sub_test_after_migrate = params.objects("sub_test_after_migrate")
while sub_test_after_migrate:
check_guest_cpu_count()
sub_test = sub_test_after_migrate.pop(0)
error_context.context("%s after migration completed" % sub_test)
eval("sub_migrate_%s" % sub_test)()
def sub_online_offline():
error_context.context("Offline then online guest CPUs after vcpu %s"
% hotpluggable_test, logging.info)
cpu_ids = list(current_guest_cpu_ids - guest_cpu_ids)
cpu_ids.sort()
cmd = "echo %d > /sys/devices/system/cpu/cpu%d/online"
try:
for cpu_id in cpu_ids[::-1]:
session.cmd(cmd % (0, cpu_id))
if not cpu_utils.check_if_vm_vcpu_match(cpu_count_before_test, vm):
test.fail(
"Actual number of guest CPUs is not equal to expected")
for cpu_id in cpu_ids:
session.cmd(cmd % (1, cpu_id))
except ShellCmdError as err:
logging.error(str(err))
test.error("Failed to change the CPU state on guest.")
def sub_pause_resume():
error_context.context("Pause guest to hotunplug all vcpu devices",
logging.info)
vm.pause()
sub_hotunplug()
error_context.context("Resume guest after hotunplug")
vm.resume()
login_timeout = params.get_numeric("login_timeout", 360)
sleep_after_change = params.get_numeric("sleep_after_cpu_change", 30)
os_type = params["os_type"]
hotpluggable_test = params["hotpluggable_test"]
verify_wait_timeout = params.get_numeric("verify_wait_timeout", 60)
sub_test_type = params.get("sub_test_type")
check_cpu_topology = params.get_boolean("check_cpu_topology", True)
vm = env.get_vm(params["main_vm"])
maxcpus = vm.cpuinfo.maxcpus
if not params.objects("vcpu_devices"):
vcpus_count = (vm.cpuinfo.threads if
params["machine_type"].startswith("pseries") else 1)
pluggable_cpus = vm.cpuinfo.maxcpus // vcpus_count // 2
params["vcpu_devices"] = " ".join(["vcpu%d" % (count + 1) for count in
range(pluggable_cpus)])
vm.destroy()
if len(params.objects("vcpu_devices")) < 2:
test.cancel("Insufficient maxcpus for multi-CPU hotplug")
params["paused_after_start_vm"] = "no"
vm.create(params=params)
if vm.is_paused():
vm.resume()
vcpu_devices = params.objects("vcpu_devices")
vm.verify_alive()
session = vm.wait_for_login(timeout=login_timeout)
cpu_count_before_test = vm.get_cpu_count()
guest_cpu_ids = cpu_utils.get_guest_cpu_ids(session, os_type)
error_context.context("Check the number of guest CPUs after startup",
logging.info)
if not cpu_utils.check_if_vm_vcpus_match_qemu(vm):
test.error("The number of guest CPUs is not equal to the qemu command "
"line configuration")
if hotpluggable_test == "hotplug":
pluggable_vcpu_dev = vcpu_devices
else:
pluggable_vcpu_dev = vcpu_devices[::-1]
if params.get_boolean("workaround_need"):
win_wora.modify_driver(params, session)
if params.get("pause_vm_before_hotplug", "no") == "yes":
error_context.context("Pause guest before %s" % hotpluggable_test,
logging.info)
vm.pause()
error_context.context("%s all vcpu devices" % hotpluggable_test,
logging.info)
for vcpu_dev in pluggable_vcpu_dev:
getattr(vm, "%s_vcpu_device" % hotpluggable_test)(vcpu_dev)
if vm.is_paused():
error_context.context("Resume guest after %s" % hotpluggable_test,
logging.info)
vm.resume()
check_guest_cpu_count()
current_guest_cpu_ids = cpu_utils.get_guest_cpu_ids(session, os_type)
if sub_test_type:
eval("sub_%s" % sub_test_type)()
# Close old session since guest maybe dead/reboot
if session:
session.close()
if vm.is_alive():
session = vm.wait_for_login(timeout=login_timeout)
check_guest_cpu_count()
if vm.get_cpu_count() == maxcpus and check_cpu_topology:
if not cpu_utils.check_if_vm_vcpu_topology_match(session, os_type,
vm.cpuinfo):
session.close()
test.fail("CPU topology of guest is inconsistent with "
"expectations.")
def run(test, params, env):
"""
Test hotplug/hotunplug of vcpu device.
1) Boot up guest w/o vcpu device.
2) Hot plug/unplug vcpu devices and check successfully or not. (qemu side)
3) Check if the number of CPUs in guest changes accordingly. (guest side)
4) Do sub test after hot plug/unplug.
5) Recheck the number of CPUs in guest.
6) Check the CPU topology of guest. (if all vcpu plugged)
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def check_guest_cpu_count(expected_count):
if not utils_misc.wait_for(
lambda: vm.get_cpu_count() == expected_count,
verify_wait_timeout,
first=sleep_after_change):
logging.error(
"CPU quantity mismatched! Guest got %s but the"
" expected is %s", vm.get_cpu_count(), expected_count)
test.fail("Actual number of guest CPUs is not equal to expected")
logging.info("CPU quantity matched: %s", expected_count)
def sub_hotunplug():
error_context.context(
"Hotunplug vcpu devices after vcpu %s" % hotpluggable_test,
logging.info)
for plugged_dev in pluggable_vcpu_dev[::-1]:
try:
vm.hotunplug_vcpu_device(plugged_dev)
except VMDeviceCheckError:
if not vm.is_paused():
raise
logging.warning(
"%s can not be unplugged directly because "
"guest is paused, will check again after "
"resume", plugged_dev)
def sub_reboot():
error_context.context("Reboot guest after vcpu %s" % hotpluggable_test,
logging.info)
vm.reboot(session=session,
method=params["reboot_method"],
timeout=login_timeout)
def sub_shutdown():
error_context.context(
"Shutdown guest after vcpu %s" % hotpluggable_test, logging.info)
shutdown_method = params["shutdown_method"]
if shutdown_method == "shell":
session.sendline(params["shutdown_command"])
error_context.context("waiting VM to go down (guest shell cmd)",
logging.info)
elif shutdown_method == "system_powerdown":
vm.monitor.system_powerdown()
error_context.context("waiting VM to go down (qemu monitor cmd)",
logging.info)
if not vm.wait_for_shutdown(360):
test.fail("Guest refuses to go down after vcpu %s" %
hotpluggable_test)
def sub_migrate():
sub_migrate_reboot = sub_reboot
sub_migrate_hotunplug = sub_hotunplug
error_context.context(
"Migrate guest after vcpu %s" % hotpluggable_test, logging.info)
vm.migrate()
vm.verify_alive()
sub_test_after_migrate = params.objects("sub_test_after_migrate")
while sub_test_after_migrate:
check_guest_cpu_count(expected_vcpus)
sub_test = sub_test_after_migrate.pop(0)
error_context.context("%s after migration completed" % sub_test)
eval("sub_migrate_%s" % sub_test)()
def sub_online_offline():
error_context.context(
"Offline then online guest CPUs after vcpu %s" % hotpluggable_test,
logging.info)
cpu_ids = list(current_guest_cpu_ids - guest_cpu_ids)
cpu_ids.sort()
cmd = "echo %d > /sys/devices/system/cpu/cpu%d/online"
try:
for cpu_id in cpu_ids[::-1]:
session.cmd(cmd % (0, cpu_id))
check_guest_cpu_count(smp)
for cpu_id in cpu_ids:
session.cmd(cmd % (1, cpu_id))
except ShellCmdError as err:
logging.error(str(err))
test.error("Failed to change the CPU state on guest.")
def sub_pause_resume():
error_context.context("Pause guest to hotunplug all vcpu devices",
logging.info)
vm.pause()
sub_hotunplug()
error_context.context("Resume guest after hotunplug")
vm.resume()
login_timeout = params.get_numeric("login_timeout", 360)
sleep_after_change = params.get_numeric("sleep_after_cpu_change", 30)
os_type = params["os_type"]
hotpluggable_test = params["hotpluggable_test"]
verify_wait_timeout = params.get_numeric("verify_wait_timeout", 60)
sub_test_type = params.get("sub_test_type")
check_cpu_topology = params.get_boolean("check_cpu_topology", True)
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session = vm.wait_for_login(timeout=login_timeout)
smp = vm.cpuinfo.smp
maxcpus = vm.cpuinfo.maxcpus
vcpus_count = vm.params.get_numeric("vcpus_count", 1)
vcpu_devices = params.objects("vcpu_devices")
guest_cpu_ids = cpu_utils.get_guest_cpu_ids(session, os_type)
if hotpluggable_test == "hotplug":
pluggable_vcpu_dev = vcpu_devices
pluggable_vcpu = vcpus_count * len(pluggable_vcpu_dev)
else:
pluggable_vcpu_dev = vcpu_devices[::-1]
pluggable_vcpu = -(vcpus_count * len(pluggable_vcpu_dev))
expected_vcpus = vm.get_cpu_count() + pluggable_vcpu
if params.get("pause_vm_before_hotplug", "no") == "yes":
error_context.context("Pause guest before %s" % hotpluggable_test,
logging.info)
vm.pause()
error_context.context("%s all vcpu devices" % hotpluggable_test,
logging.info)
for vcpu_dev in pluggable_vcpu_dev:
getattr(vm, "%s_vcpu_device" % hotpluggable_test)(vcpu_dev)
if vm.is_paused():
error_context.context("Resume guest after %s" % hotpluggable_test,
logging.info)
vm.resume()
check_guest_cpu_count(expected_vcpus)
current_guest_cpu_ids = cpu_utils.get_guest_cpu_ids(session, os_type)
if sub_test_type:
eval("sub_%s" % sub_test_type)()
# Close old session since guest maybe dead/reboot
if session:
session.close()
if ("hotunplug" in params.objects("sub_test_after_migrate")
or sub_test_type == "pause_resume"):
expected_vcpus -= pluggable_vcpu
if vm.is_alive():
session = vm.wait_for_login(timeout=login_timeout)
check_guest_cpu_count(expected_vcpus)
if expected_vcpus == maxcpus and check_cpu_topology:
if not cpu_utils.check_guest_cpu_topology(session, os_type,
vm.cpuinfo):
session.close()
test.fail("CPU topology of guest is inconsistent with "
"expectations.")
def run(test, params, env):
"""
Test hotplug vcpu devices and execute stress test.
1) Boot up guest without vcpu device.
2) Hotplug vcpu devices and check successfully or not. (qemu side)
3) Check if the number of CPUs in guest changes accordingly. (guest side)
4) Execute stress test on all hotplugged vcpu devices
5) Hotunplug vcpu devices during stress test
6) Recheck the number of CPUs in guest.
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def heavyload_install():
if session.cmd_status(test_installed_cmd) != 0:
logging.warning("Could not find installed heavyload in guest, will"
" install it via winutils.iso ")
winutil_drive = utils_misc.get_winutils_vol(session)
if not winutil_drive:
test.cancel("WIN_UTILS CDROM not found.")
install_cmd = params["install_cmd"] % winutil_drive
session.cmd(install_cmd)
os_type = params["os_type"]
vm_arch_name = params.get('vm_arch_name', arch.ARCH)
login_timeout = params.get_numeric("login_timeout", 360)
stress_duration = params.get_numeric("stress_duration", 180)
verify_wait_timeout = params.get_numeric("verify_wait_timeout", 60)
vcpu_devices = params.objects("vcpu_devices")
vcpus_count = params.get_numeric("vcpus_count", 1)
pluggable_count = len(vcpu_devices) * vcpus_count
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session = vm.wait_for_login(timeout=login_timeout)
if not cpu_utils.check_if_vm_vcpu_match(vm.cpuinfo.smp, vm):
test.error("The number of guest CPUs is not equal to the qemu command "
"line configuration")
cpu_count_before_test = vm.get_cpu_count()
expected_count = pluggable_count + cpu_count_before_test
guest_cpu_ids = cpu_utils.get_guest_cpu_ids(session, os_type)
for vcpu_dev in vcpu_devices:
error_context.context("Hotplug vcpu device: %s" % vcpu_dev,
logging.info)
vm.hotplug_vcpu_device(vcpu_dev)
if not utils_misc.wait_for(
lambda: cpu_utils.check_if_vm_vcpu_match(expected_count, vm),
verify_wait_timeout):
test.fail("Actual number of guest CPUs is not equal to expected")
if os_type == "linux":
stress_args = params["stress_args"]
stress_tool = cpu_utils.VMStressBinding(vm,
params,
stress_args=stress_args)
current_guest_cpu_ids = cpu_utils.get_guest_cpu_ids(session, os_type)
plugged_cpu_ids = list(current_guest_cpu_ids - guest_cpu_ids)
plugged_cpu_ids.sort()
for cpu_id in plugged_cpu_ids:
error_context.context(
"Run stress on vCPU(%d) inside guest." % cpu_id, logging.info)
stress_tool.load_stress_tool(cpu_id)
error_context.context(
"Successfully launched stress sessions, execute "
"stress test for %d seconds" % stress_duration, logging.info)
time.sleep(stress_duration)
if utils_package.package_install("sysstat", session):
error_context.context("Check usage of guest CPUs", logging.info)
mpstat_cmd = "mpstat 1 5 -P %s | cat" % ",".join(
map(str, plugged_cpu_ids))
mpstat_out = session.cmd_output(mpstat_cmd)
cpu_stat = dict(
re.findall(r"Average:\s+(\d+)\s+(\d+\.\d+)", mpstat_out, re.M))
for cpu_id in plugged_cpu_ids:
cpu_usage_rate = float(cpu_stat[str(cpu_id)])
if cpu_usage_rate < 50:
test.error("Stress test on vCPU(%s) failed, usage rate: "
"%.2f%%" % (cpu_id, cpu_usage_rate))
logging.info("Usage rate of vCPU(%s) is: %.2f%%", cpu_id,
cpu_usage_rate)
if not vm_arch_name.startswith("s390"):
for vcpu_dev in vcpu_devices:
error_context.context("Hotunplug vcpu device: %s" % vcpu_dev,
logging.info)
vm.hotunplug_vcpu_device(vcpu_dev)
# Drift the running stress task to other vCPUs
time.sleep(random.randint(5, 10))
if not cpu_utils.check_if_vm_vcpu_match(cpu_count_before_test, vm):
test.fail("Actual number of guest CPUs is not equal to "
"expected")
stress_tool.unload_stress()
stress_tool.clean()
else:
install_path = params["install_path"]
test_installed_cmd = 'dir "%s" | findstr /I heavyload' % install_path
heavyload_install()
error_context.context("Run heavyload inside guest.", logging.info)
heavyload_bin = r'"%s\heavyload.exe" ' % install_path
heavyload_options = [
"/CPU %d" % expected_count,
"/DURATION %d" % (stress_duration // 60), "/AUTOEXIT", "/START"
]
start_cmd = heavyload_bin + " ".join(heavyload_options)
stress_tool = BackgroundTest(
session.cmd, (start_cmd, stress_duration, stress_duration))
stress_tool.start()
if not utils_misc.wait_for(
stress_tool.is_alive, verify_wait_timeout, first=5):
test.error("Failed to start heavyload process.")
stress_tool.join(stress_duration)
session.close()