def run(test, params, env): """ :param test: kvm test object :param params: Dictionary with the test parameters :param env: Dictionary with test environment. """ guest_stress = params.get("guest_stress", "no") == "yes" host_stress = params.get("host_stress", "no") == "yes" stress_events = params.get("stress_events", "reboot") vms = env.get_all_vms() vms_uptime_init = {} if "reboot" not in stress_events: for vm in vms: vms_uptime_init[vm.name] = vm.uptime() stress_event = utils_stress.VMStressEvents(params, env) if guest_stress: try: utils_test.load_stress("stress_in_vms", params=params, vms=vms) except Exception as err: test.fail("Error running stress in vms: %s" % err) if host_stress: if params.get("host_stress_args", ""): params["stress_args"] = params.get("host_stress_args") try: utils_test.load_stress("stress_on_host", params=params) except Exception as err: test.fail("Error running stress in host: %s" % err) try: stress_event.run_threads() finally: stress_event.wait_for_threads() if guest_stress: utils_test.unload_stress("stress_in_vms", params=params, vms=vms) if host_stress: utils_test.unload_stress("stress_on_host", params=params) if "reboot" not in stress_events: fail = False for vm in vms: if vm.uptime() < vms_uptime_init[vm.name]: logging.error("Unexpected reboot of VM: %s between test", vm.name) fail = True if fail: test.fail("Unexpected VM reboot detected")
def do_stress_migration(vms, srcuri, desturi, stress_type, migration_type, params, thread_timeout=60): """ Migrate vms with stress. :param vms: migrated vms. :param srcuri: connect uri for source machine :param desturi: connect uri for destination machine :param stress_type: type of stress test in VM :param migration_type: type of migration to be performed :param params: Test dict params :param thread_timeout: default timeout for migration thread :raise: exceptions.TestFail if migration fails """ fail_info = utils_test.load_stress(stress_type, vms, params) migtest = utlv.MigrationTest() options = '' if migration_type == "compressed": options = "--compressed" migration_type = "orderly" shared_dir = os.path.dirname(data_dir.get_data_dir()) src_file = os.path.join(shared_dir, "scripts", "duplicate_pages.py") dest_dir = "/tmp" for vm in vms: session = vm.wait_for_login() vm.copy_files_to(src_file, dest_dir) status = session.cmd_status("cd /tmp;python duplicate_pages.py") if status: fail_info.append("Set duplicated pages for vm failed.") if len(fail_info): logging.warning("Add stress for migration failed:%s", fail_info) logging.debug("Starting migration...") migrate_options = ("--live --unsafe %s --timeout %s" % (options, params.get("virsh_migrate_timeout", 60))) migtest.do_migration(vms, srcuri, desturi, migration_type, options=migrate_options, thread_timeout=thread_timeout) # vms will be shutdown, so no need to do this cleanup # And migrated vms may be not login if the network is local lan if stress_type == "stress_on_host": utils_test.unload_stress(stress_type, vms) if not migtest.RET_MIGRATION: raise exceptions.TestFail()
def do_stress_migration(vms, srcuri, desturi, stress_type, migration_type, params, thread_timeout=60): """ Migrate vms with stress. :param vms: migrated vms. """ fail_info = utils_test.load_stress(stress_type, vms, params) if len(fail_info): logging.warning("Add stress for migration failed:%s", fail_info) migtest = utlv.MigrationTest() migtest.do_migration(vms, srcuri, desturi, migration_type, thread_timeout) utils_test.unload_stress(stress_type, vms) if not migtest.RET_MIGRATION: raise error.TestFail()
def load_stress(vms, stress_type, params={}): """ Load different stress in vm: unixbench, stress, running vms and so on """ fail_info = [] # Special operations for test if stress_type in ["stress_in_vms", "stress_on_host"]: logging.debug("Run stress %s", stress_type) fail_info = utils_test.load_stress(stress_type, vms, params) elif stress_type == "inject_nmi": inject_times = int(params.get("inject_times", 10)) for vm in vms: while inject_times > 0: try: inject_times -= 1 virsh.inject_nmi(vm.name, debug=True, ignore_status=False) except error.CmdError, detail: fail_info.append("Inject operations failed:%s", detail)
def do_stress_migration(vms, srcuri, desturi, stress_type, migration_type, params, thread_timeout=60): """ Migrate vms with stress. :param vms: migrated vms. """ fail_info = utils_test.load_stress(stress_type, vms, params) if len(fail_info): logging.warning("Add stress for migration failed:%s", fail_info) migtest = utlv.MigrationTest() migtest.do_migration(vms, srcuri, desturi, migration_type, options=None, thread_timeout=thread_timeout) # vms will be shutdown, so no need to do this cleanup # And migrated vms may be not login if the network is local lan if stress_type == "stress_on_host": utils_test.unload_stress(stress_type, vms) if not migtest.RET_MIGRATION: raise error.TestFail()
def manipulate_vm(vm, operation, params=None): """ Manipulate the VM. :param vm: VM instance :param operation: stress_in_vms, inject_nmi, dump, suspend_resume or save_restore :param params: Test parameters """ err_msg = '' # Special operations for test if operation == "stress": logging.debug("Load stress in VM") err_msg = utils_test.load_stress(operation, [vm], params)[0] elif operation == "inject_nmi": inject_times = int(params.get("inject_times", 10)) logging.info("Trying to inject nmi %s times", inject_times) while inject_times > 0: try: inject_times -= 1 virsh.inject_nmi(vm.name, debug=True, ignore_status=False) except error.CmdError, detail: err_msg = "Inject nmi failed: %s" % detail
def manipulate_vm(vm, operation, params=None): """ Manipulate the VM. :param vm: VM instance :param operation: stress_in_vms, inject_nmi, dump, suspend_resume or save_restore :param params: Test parameters """ err_msg = '' # Special operations for test if operation == "stress": logging.debug("Load stress in VM") err_msg = utils_test.load_stress(operation, [vm], params)[0] elif operation == "inject_nmi": inject_times = int(params.get("inject_times", 10)) logging.info("Trying to inject nmi %s times", inject_times) while inject_times > 0: try: inject_times -= 1 virsh.inject_nmi(vm.name, debug=True, ignore_status=False) except process.CmdError as detail: err_msg = "Inject nmi failed: %s" % detail elif operation == "dump": dump_times = int(params.get("dump_times", 10)) logging.info("Trying to dump vm %s times", dump_times) while dump_times > 0: dump_times -= 1 dump_path = os.path.join(data_dir.get_tmp_dir(), "dump.file") try: virsh.dump(vm.name, dump_path, debug=True, ignore_status=False) except (process.CmdError, OSError) as detail: err_msg = "Dump %s failed: %s" % (vm.name, detail) try: os.remove(dump_path) except OSError: pass elif operation == "suspend_resume": paused_times = int(params.get("paused_times", 10)) logging.info("Trying to suspend/resume vm %s times", paused_times) while paused_times > 0: paused_times -= 1 try: virsh.suspend(vm.name, debug=True, ignore_status=False) virsh.resume(vm.name, debug=True, ignore_status=False) except process.CmdError as detail: err_msg = "Suspend-Resume %s failed: %s" % (vm.name, detail) elif operation == "save_restore": save_times = int(params.get("save_times", 10)) logging.info("Trying to save/restore vm %s times", save_times) while save_times > 0: save_times -= 1 save_path = os.path.join(data_dir.get_tmp_dir(), "save.file") try: virsh.save(vm.name, save_path, debug=True, ignore_status=False) virsh.restore(save_path, debug=True, ignore_status=False) except process.CmdError as detail: err_msg = "Save-Restore %s failed: %s" % (vm.name, detail) try: os.remove(save_path) except OSError: pass else: err_msg = "Unsupport operation in this function: %s" % operation return err_msg
def manipulate_vm(vm, operation, params=None): """ Manipulate the VM. :param vm: VM instance :param operation: stress_in_vms, inject_nmi, dump, suspend_resume or save_restore :param params: Test parameters """ err_msg = '' # Special operations for test if operation == "stress": logging.debug("Load stress in VM") err_msg = utils_test.load_stress(operation, params=params, vms=[vm])[0] elif operation == "inject_nmi": inject_times = int(params.get("inject_times", 10)) logging.info("Trying to inject nmi %s times", inject_times) while inject_times > 0: try: inject_times -= 1 virsh.inject_nmi(vm.name, debug=True, ignore_status=False) except process.CmdError as detail: err_msg = "Inject nmi failed: %s" % detail elif operation == "dump": dump_times = int(params.get("dump_times", 10)) logging.info("Trying to dump vm %s times", dump_times) while dump_times > 0: dump_times -= 1 dump_path = os.path.join(data_dir.get_tmp_dir(), "dump.file") try: virsh.dump(vm.name, dump_path, debug=True, ignore_status=False) except (process.CmdError, OSError) as detail: err_msg = "Dump %s failed: %s" % (vm.name, detail) try: os.remove(dump_path) except OSError: pass elif operation == "suspend_resume": paused_times = int(params.get("paused_times", 10)) logging.info("Trying to suspend/resume vm %s times", paused_times) while paused_times > 0: paused_times -= 1 try: virsh.suspend(vm.name, debug=True, ignore_status=False) virsh.resume(vm.name, debug=True, ignore_status=False) except process.CmdError as detail: err_msg = "Suspend-Resume %s failed: %s" % (vm.name, detail) elif operation == "save_restore": save_times = int(params.get("save_times", 10)) logging.info("Trying to save/restore vm %s times", save_times) while save_times > 0: save_times -= 1 save_path = os.path.join(data_dir.get_tmp_dir(), "save.file") try: virsh.save(vm.name, save_path, debug=True, ignore_status=False) virsh.restore(save_path, debug=True, ignore_status=False) except process.CmdError as detail: err_msg = "Save-Restore %s failed: %s" % (vm.name, detail) try: os.remove(save_path) except OSError: pass else: err_msg = "Unsupport operation in this function: %s" % operation return err_msg
def copied_migration(vm, params, blockjob_type=None, block_target="vda"): """ Migrate vms with storage copied under some stress. And during it, some qemu-monitor-command will be sent. """ dest_uri = params.get("migrate_dest_uri") remote_host = params.get("migrate_dest_host") copy_option = params.get("copy_storage_option", "") username = params.get("remote_user") password = params.get("migrate_dest_pwd") timeout = int(params.get("thread_timeout", 1200)) options = "--live %s --unsafe" % copy_option # Get vm ip for remote checking if vm.is_dead(): vm.start() vm.wait_for_login() vms_ip = {} vms_ip[vm.name] = vm.get_address() logging.debug("VM %s IP: %s", vm.name, vms_ip[vm.name]) # Start to load stress stress_type = params.get("migrate_stress_type") if stress_type == "cpu": params['stress_args'] = "--cpu 2 --quiet --timeout 60" elif stress_type == "memory": params['stress_args'] = "--vm 2 --vm-bytes 256M --vm-keep --timeout 60" if stress_type is not None: utils_test.load_stress("stress_in_vms", [vm], params) cp_mig = utlv.MigrationTest() migration_thread = threading.Thread(target=cp_mig.thread_func_migration, args=(vm, dest_uri, options)) migration_thread.start() # Wait for migration launched time.sleep(5) job_ret = virsh.domjobinfo(vm.name, debug=True) if job_ret.exit_status: raise error.TestError("Prepare migration for blockjob failed.") # Execute some qemu monitor commands pause_cmd = "block-job-pause %s" % block_target resume_cmd = "block-job-resume %s" % block_target cancel_cmd = "block-job-cancel %s" % block_target complete_cmd = "block-job-complete %s" % block_target blockjob_failures = [] try: if blockjob_type == "cancel": virsh.qemu_monitor_command(vm.name, cancel_cmd, debug=True, ignore_status=False) elif blockjob_type == "pause_resume": virsh.qemu_monitor_command(vm.name, pause_cmd, debug=True, ignore_status=False) # TODO: Check whether it is paused. virsh.qemu_monitor_command(vm.name, resume_cmd, debug=True, ignore_status=False) elif blockjob_type == "complete": virsh.qemu_monitor_command(vm.name, complete_cmd, debug=True, ignore_status=False) except process.CmdError, detail: blockjob_failures.append(str(detail))
def copied_migration(vm, params, blockjob_type=None, block_target="vda"): """ Migrate vms with storage copied under some stress. And during it, some qemu-monitor-command will be sent. """ dest_uri = params.get("migrate_dest_uri") remote_host = params.get("migrate_dest_host") copy_option = params.get("copy_storage_option", "") username = params.get("remote_user") password = params.get("migrate_dest_pwd") timeout = int(params.get("thread_timeout", 1200)) options = "--live %s --unsafe" % copy_option # Get vm ip for remote checking if vm.is_dead(): vm.start() vm.wait_for_login() vms_ip = {} vms_ip[vm.name] = vm.get_address() logging.debug("VM %s IP: %s", vm.name, vms_ip[vm.name]) # Start to load stress stress_type = params.get("migrate_stress_type") if stress_type == "cpu": params['stress_args'] = "--cpu 2 --quiet --timeout 60" elif stress_type == "memory": params['stress_args'] = "--vm 2 --vm-bytes 256M --vm-keep --timeout 60" if stress_type is not None: utils_test.load_stress("stress_in_vms", [vm], params) cp_mig = utlv.MigrationTest() migration_thread = threading.Thread(target=cp_mig.thread_func_migration, args=(vm, dest_uri, options)) migration_thread.start() # Wait for migration launched time.sleep(5) job_ret = virsh.domjobinfo(vm.name, debug=True) if job_ret.exit_status: raise error.TestError("Prepare migration for blockjob failed.") # Execute some qemu monitor commands pause_cmd = "block-job-pause %s" % block_target resume_cmd = "block-job-resume %s" % block_target cancel_cmd = "block-job-cancel %s" % block_target complete_cmd = "block-job-complete %s" % block_target blockjob_failures = [] try: if blockjob_type == "cancel": virsh.qemu_monitor_command(vm.name, cancel_cmd, debug=True, ignore_status=False) elif blockjob_type == "pause_resume": virsh.qemu_monitor_command(vm.name, pause_cmd, debug=True, ignore_status=False) # TODO: Check whether it is paused. virsh.qemu_monitor_command(vm.name, resume_cmd, debug=True, ignore_status=False) elif blockjob_type == "complete": virsh.qemu_monitor_command(vm.name, complete_cmd, debug=True, ignore_status=False) except error.CmdError, detail: blockjob_failures.append(str(detail))
def copied_migration(test, vm, params, blockjob_type=None, block_target="vda"): """ Migrate vms with storage copied under some stress. And during it, some qemu-monitor-command will be sent. """ dest_uri = params.get("migrate_dest_uri") remote_host = params.get("migrate_dest_host") copy_option = params.get("copy_storage_option", "") username = params.get("remote_user") password = params.get("migrate_dest_pwd") timeout = int(params.get("thread_timeout", 1200)) options = "--live %s --unsafe" % copy_option # Get vm ip for remote checking if vm.is_dead(): vm.start() vm.wait_for_login() vms_ip = {} vms_ip[vm.name] = vm.get_address() logging.debug("VM %s IP: %s", vm.name, vms_ip[vm.name]) # Start to load stress stress_type = params.get("migrate_stress_type") if stress_type == "cpu": params['stress_args'] = "--cpu 2 --quiet --timeout 60" elif stress_type == "memory": params['stress_args'] = "--vm 2 --vm-bytes 256M --vm-keep --timeout 60" if stress_type is not None: utils_test.load_stress("stress_in_vms", [vm], params) cp_mig = utlv.MigrationTest() migration_thread = threading.Thread(target=cp_mig.thread_func_migration, args=(vm, dest_uri, options)) migration_thread.start() # Wait for migration launched time.sleep(5) job_ret = virsh.domjobinfo(vm.name, debug=True) if job_ret.exit_status: test.error("Prepare migration for blockjob failed.") # Execute some qemu monitor commands pause_cmd = "block-job-pause %s" % block_target resume_cmd = "block-job-resume %s" % block_target cancel_cmd = "block-job-cancel %s" % block_target complete_cmd = "block-job-complete %s" % block_target blockjob_failures = [] try: if blockjob_type == "cancel": virsh.qemu_monitor_command(vm.name, cancel_cmd, debug=True, ignore_status=False) elif blockjob_type == "pause_resume": virsh.qemu_monitor_command(vm.name, pause_cmd, debug=True, ignore_status=False) # TODO: Check whether it is paused. virsh.qemu_monitor_command(vm.name, resume_cmd, debug=True, ignore_status=False) elif blockjob_type == "complete": virsh.qemu_monitor_command(vm.name, complete_cmd, debug=True, ignore_status=False) except process.CmdError as detail: blockjob_failures.append(str(detail)) # Job info FYI virsh.domjobinfo(vm.name, debug=True) if len(blockjob_failures): timeout = 30 migration_thread.join(timeout) if migration_thread.isAlive(): logging.error("Migrate %s timeout.", migration_thread) cp_mig.RET_LOCK.acquire() cp_mig.RET_MIGRATION = False cp_mig.RET_LOCK.release() if len(blockjob_failures): cp_mig.cleanup_dest_vm(vm, None, dest_uri) test.fail("Run qemu monitor command failed %s" % blockjob_failures) check_ip_failures = [] if cp_mig.RET_MIGRATION: try: utils_test.check_dest_vm_network(vm, vms_ip[vm.name], remote_host, username, password) except exceptions.TestFail as detail: check_ip_failures.append(str(detail)) cp_mig.cleanup_dest_vm(vm, None, dest_uri) if blockjob_type in ["cancel", "complete"]: test.fail("Storage migration passed even after " "cancellation.") else: cp_mig.cleanup_dest_vm(vm, None, dest_uri) if blockjob_type in ["cancel", "complete"]: logging.error("Expected Migration Error for %s", blockjob_type) return else: test.fail("Command blockjob does not work well under " "storage copied migration.") if len(check_ip_failures): test.fail("Check IP failed:%s" % check_ip_failures)
def run(test, params, env): """ Test: vcpu hotplug. The command can change the number of virtual CPUs for VM. 1.Prepare test environment,destroy or suspend a VM. 2.Perform virsh setvcpus operation. 3.Recover test environment. 4.Confirm the test result. """ vm_name = params.get("main_vm") min_count = int(params.get("setvcpus_min_count", "1")) max_count = int(params.get("setvcpus_max_count", "2")) test_times = int(params.get("setvcpus_test_times", "1")) stress_type = params.get("stress_type", "") stress_param = params.get("stress_param", "") add_by_virsh = ("yes" == params.get("add_by_virsh")) del_by_virsh = ("yes" == params.get("del_by_virsh")) hotplug_timeout = int(params.get("hotplug_timeout", 30)) test_set_max = max_count * 2 # Save original configuration orig_config_xml = libvirt_xml.VMXML.new_from_inactive_dumpxml(vm_name) # Set min/max of vcpu libvirt_xml.VMXML.set_vm_vcpus(vm_name, test_set_max, min_count, topology_correction=True) # prepare VM instance vm = libvirt_vm.VM(vm_name, params, test.bindir, env.get("address_cache")) # prepare guest-agent service vm.prepare_guest_agent() # Increase the workload load_vms = [] if stress_type in ['cpu', 'memory', 'io']: params["stress_args"] = stress_param load_vms.append(vm) if stress_type in ['cpu', 'memory']: utils_test.load_stress("stress_in_vms", params, vms=load_vms) else: utils_test.load_stress("iozone_in_vms", params, vms=load_vms) session = vm.wait_for_login() try: # Clear dmesg before set vcpu session.cmd("dmesg -c") for i in range(test_times): # 1. Add vcpu add_result = cpu.hotplug_domain_vcpu(vm, max_count, add_by_virsh) add_status = add_result.exit_status # 1.1 check add status if add_status: if add_result.stderr.count("support"): test.cancel("vcpu hotplug not supported, " "no need to test any more:\n %s" % add_result.stderr.strip()) test.fail("Test failed for:\n %s" % add_result.stderr.strip()) if not utils_misc.wait_for( lambda: cpu.check_if_vm_vcpu_match(max_count, vm), hotplug_timeout, text="wait for vcpu online"): test.fail("vcpu hotplug failed") if 'ppc' not in platform.machine(): # 1.2 check dmesg domain_add_dmesg = session.cmd_output("dmesg -c") dmesg1 = "CPU%d has been hot-added" % (max_count - 1) dmesg2 = "CPU %d got hotplugged" % (max_count - 1) if (not domain_add_dmesg.count(dmesg1) and not domain_add_dmesg.count(dmesg2)): test.fail("Cannot find hotplug info in dmesg: %s" % domain_add_dmesg) # 1.3 check cpu related file online_cmd = "cat /sys/devices/system/cpu/cpu%d/online" \ % (max_count - 1) st, ot = session.cmd_status_output(online_cmd) if st: test.fail("Cannot find CPU%d after hotplug" % (max_count - 1)) # 1.4 check online if not ot.strip().count("1"): test.fail("CPU%d is not online after hotplug: %s" % ((max_count - 1), ot)) # 1.5 check online interrupts info inter_on_output = session.cmd_output("cat /proc/interrupts") if not inter_on_output.count("CPU%d" % (int(max_count) - 1)): test.fail("CPU%d can not be found in " "/proc/interrupts when it's online:%s" % ((int(max_count) - 1), inter_on_output)) # 1.6 offline vcpu off_st = session.cmd_status( "echo 0 > " "/sys/devices/system/cpu/cpu%d/online" % (max_count - 1)) if off_st: test.fail("Set cpu%d offline failed!" % (max_count - 1)) # 1.7 check offline interrupts info inter_off_output = session.cmd_output("cat /proc/interrupts") if inter_off_output.count("CPU%d" % (int(max_count) - 1)): test.fail("CPU%d can be found in /proc/interrupts" " when it's offline" % (int(max_count) - 1)) # 1.8 online vcpu on_st = session.cmd_status("echo 1 > " "/sys/devices/system/cpu/cpu%d/online" % (max_count - 1)) if on_st: test.fail("Set cpu%d online failed!" % (max_count - 1)) # 2. Del vcpu del_result = cpu.hotplug_domain_vcpu(vm, min_count, del_by_virsh, hotplug=False) del_status = del_result.exit_status if del_status: logging.info("del_result: %s" % del_result.stderr.strip()) # A qemu older than 1.5 or an unplug for 1.6 will result in # the following failure. # TODO: when CPU-hotplug feature becomes stable and strong, # remove these codes used to handle kinds of exceptions if re.search("The command cpu-del has not been found", del_result.stderr): test.cancel("vcpu hotunplug not supported") if re.search("cannot change vcpu count", del_result.stderr): test.cancel("unhotplug failed") if re.search("got wrong number of vCPU pids from QEMU monitor", del_result.stderr): test.cancel("unhotplug failed") # process all tips that contains keyword 'support' # for example, "unsupported"/"hasn't been support" and so on if re.search("support", del_result.stderr): test.cancel("vcpu hotunplug not supported") # besides above, regard it failed test.fail("Test fail for:\n %s" % del_result.stderr.strip()) if not utils_misc.wait_for( lambda: cpu.check_if_vm_vcpu_match(min_count, vm), hotplug_timeout, text="wait for vcpu offline"): test.fail("vcpu hotunplug failed") if 'ppc' not in platform.machine(): domain_del_dmesg = session.cmd_output("dmesg -c") if not domain_del_dmesg.count("CPU %d is now offline" % (max_count - 1)): test.fail("Cannot find hot-unplug info in dmesg: %s" % domain_del_dmesg) except exceptions.TestCancel: # So far, QEMU doesn't support unplug vcpu, # unplug operation will encounter kind of errors. pass finally: utils_test.unload_stress("stress_in_vms", params, load_vms) if session: session.close() # Cleanup orig_config_xml.sync()
def run(test, params, env): """ Test different hmi injections with guest :param test: QEMU test object :param params: Dictionary with the test parameters :param env: Dictionary with test environment. """ def set_condn(action, recover=False): """ Set/reset guest state/action :param action: Guest state change/action :param recover: whether to recover given state default: False """ if not recover: if action == "pin_vcpu": for i in range(cur_vcpu): virsh.vcpupin(vm_name, i, hmi_cpu, "--live", ignore_status=False, debug=True) virsh.emulatorpin(vm_name, hmi_cpu, "live", ignore_status=False, debug=True) elif action == "filetrans": utils_test.run_file_transfer(test, params, env) elif action == "save": save_file = os.path.join(data_dir.get_tmp_dir(), vm_name + ".save") result = virsh.save(vm_name, save_file, ignore_status=True, debug=True) utils_test.libvirt.check_exit_status(result) time.sleep(10) if os.path.exists(save_file): result = virsh.restore(save_file, ignore_status=True, debug=True) utils_test.libvirt.check_exit_status(result) os.remove(save_file) elif action == "suspend": result = virsh.suspend(vm_name, ignore_status=True, debug=True) utils_test.libvirt.check_exit_status(result) time.sleep(10) result = virsh.resume(vm_name, ignore_status=True, debug=True) utils_test.libvirt.check_exit_status(result) return host_version = params.get("host_version") guest_version = params.get("guest_version", "") max_vcpu = int(params.get("ppchmi_vcpu_max", '1')) cur_vcpu = int(params.get("ppchmi_vcpu_cur", "1")) cores = int(params.get("ppchmi_cores", '1')) sockets = int(params.get("ppchmi_sockets", '1')) threads = int(params.get("ppchmi_threads", '1')) status_error = "yes" == params.get("status_error", "no") condition = params.get("condn", "") inject_code = params.get("inject_code", "") scom_base = params.get("scom_base", "") hmi_name = params.get("hmi_name", "") hmi_iterations = int(params.get("hmi_iterations", 1)) if host_version not in cpu.get_cpu_arch(): test.cancel("Unsupported Host cpu version") vm_name = params.get("main_vm") vm = env.get_vm(vm_name) sm = SoftwareManager() if not sm.check_installed("opal-utils") and not sm.install("opal-utils"): test.cancel("opal-utils package install failed") cpus_list = cpu.cpu_online_list() cpu_idle_state = cpu.get_cpuidle_state() cpu.set_cpuidle_state() # Lets use second available host cpu hmi_cpu = cpus_list[1] pir = int( open('/sys/devices/system/cpu/cpu%s/pir' % hmi_cpu).read().strip(), 16) if host_version == 'power9': coreid = (((pir) >> 2) & 0x3f) nodeid = (((pir) >> 8) & 0x7f) & 0xf hmi_scom_addr = hex(((coreid & 0x1f + 0x20) << 24) | int(scom_base, 16)) if host_version == 'power8': coreid = (((pir) >> 3) & 0xf) nodeid = (((pir) >> 7) & 0x3f) hmi_scom_addr = hex(((coreid & 0xf) << 24) | int(scom_base, 16)) hmi_cmd = "putscom -c %s %s %s" % (nodeid, hmi_scom_addr, inject_code) vmxml = libvirt_xml.VMXML.new_from_inactive_dumpxml(vm_name) org_xml = vmxml.copy() # Destroy the vm vm.destroy() try: session = None bgt = None libvirt_xml.VMXML.set_vm_vcpus(vm_name, max_vcpu, cur_vcpu, sockets=sockets, cores=cores, threads=threads, add_topology=True) if guest_version: libvirt_xml.VMXML.set_cpu_mode(vm_name, model=guest_version) vm.start() # Lets clear host and guest dmesg process.system("dmesg -C", verbose=False) session = vm.wait_for_login() session.cmd("dmesg -C") # Set condn if "vcpupin" in condition: set_condn("pin_vcpu") if "stress" in condition: utils_test.load_stress("stress_in_vms", params=params, vms=[vm]) if "save" in condition: set_condn("save") if "suspend" in condition: set_condn("suspend") # hmi inject logging.debug("Injecting %s HMI on cpu %s", hmi_name, hmi_cpu) logging.debug("HMI Command: %s", hmi_cmd) process.run(hmi_cmd) # Check host and guest dmesg host_dmesg = process.system_output("dmesg -c", verbose=False) guest_dmesg = session.cmd_output("dmesg") if "Unrecovered" in host_dmesg: test.fail("Unrecovered host hmi\n%s", host_dmesg) else: logging.debug("Host dmesg: %s", host_dmesg) logging.debug("Guest dmesg: %s", guest_dmesg) if "save" in condition: set_condn("save") if "suspend" in condition: set_condn("suspend") finally: if "stress" in condition: utils_test.unload_stress("stress_in_vms", params=params, vms=[vm]) if session: session.close() org_xml.sync() cpu.set_cpuidle_state(setstate=cpu_idle_state)
def set_condition(vm_name, condn, reset=False, guestbt=None): """ Set domain to given state or reset it. """ bt = None if not reset: if condn == "avocadotest": bt = utils_test.run_avocado_bg(vm, params, test) if not bt: test.cancel("guest stress failed to start") # Allow stress to start time.sleep(condn_sleep_sec) return bt elif condn == "stress": utils_test.load_stress("stress_in_vms", params=params, vms=[vm]) elif condn in ["save", "managedsave"]: # No action pass elif condn == "suspend": result = virsh.suspend(vm_name, ignore_status=True, debug=True) libvirt.check_exit_status(result) elif condn == "hotplug": result = virsh.setvcpus(vm_name, max_vcpu, "--live", ignore_status=True, debug=True) libvirt.check_exit_status(result) exp_vcpu = {'max_config': max_vcpu, 'max_live': max_vcpu, 'cur_config': current_vcpu, 'cur_live': max_vcpu, 'guest_live': max_vcpu} result = cpu.check_vcpu_value(vm, exp_vcpu, option="--live") elif condn == "host_smt": if cpuutil.get_cpu_vendor_name() == 'power9': result = process.run("ppc64_cpu --smt=4", shell=True) else: test.cancel("Host SMT changes not allowed during guest live") else: logging.debug("No operation for the domain") else: if condn == "save": save_file = os.path.join(data_dir.get_tmp_dir(), vm_name + ".save") result = virsh.save(vm_name, save_file, ignore_status=True, debug=True) libvirt.check_exit_status(result) time.sleep(condn_sleep_sec) if os.path.exists(save_file): result = virsh.restore(save_file, ignore_status=True, debug=True) libvirt.check_exit_status(result) os.remove(save_file) else: test.error("No save file for domain restore") elif condn == "managedsave": result = virsh.managedsave(vm_name, ignore_status=True, debug=True) libvirt.check_exit_status(result) time.sleep(condn_sleep_sec) result = virsh.start(vm_name, ignore_status=True, debug=True) libvirt.check_exit_status(result) elif condn == "suspend": result = virsh.resume(vm_name, ignore_status=True, debug=True) libvirt.check_exit_status(result) elif condn == "avocadotest": guestbt.join(ignore_status=True) elif condn == "stress": utils_test.unload_stress("stress_in_vms", params=params, vms=[vm]) elif condn == "hotplug": result = virsh.setvcpus(vm_name, current_vcpu, "--live", ignore_status=True, debug=True) libvirt.check_exit_status(result) exp_vcpu = {'max_config': max_vcpu, 'max_live': current_vcpu, 'cur_config': current_vcpu, 'cur_live': current_vcpu, 'guest_live': current_vcpu} result = cpu.check_vcpu_value(vm, exp_vcpu, option="--live") elif condn == "host_smt": result = process.run("ppc64_cpu --smt=2", shell=True) # Change back the host smt result = process.run("ppc64_cpu --smt=4", shell=True) # Work around due to known cgroup issue after cpu hot(un)plug # sequence root_cpuset_path = utils_cgroup.get_cgroup_mountpoint("cpuset") machine_cpuset_paths = [] if os.path.isdir(os.path.join(root_cpuset_path, "machine.slice")): machine_cpuset_paths.append(os.path.join(root_cpuset_path, "machine.slice")) if os.path.isdir(os.path.join(root_cpuset_path, "machine")): machine_cpuset_paths.append(os.path.join(root_cpuset_path, "machine")) if not machine_cpuset_paths: logging.warning("cgroup cpuset might not recover properly " "for guests after host smt changes, " "restore it manually") root_cpuset_cpus = os.path.join(root_cpuset_path, "cpuset.cpus") for path in machine_cpuset_paths: machine_cpuset_cpus = os.path.join(path, "cpuset.cpus") # check if file content differs cmd = "diff %s %s" % (root_cpuset_cpus, machine_cpuset_cpus) if process.system(cmd, verbose=True, ignore_status=True): cmd = "cp %s %s" % (root_cpuset_cpus, machine_cpuset_cpus) process.system(cmd, verbose=True) else: logging.debug("No need recover the domain") return bt
def copied_migration(test, vm, params, blockjob_type=None, block_target="vda"): """ Migrate vms with storage copied under some stress. And during it, some qemu-monitor-command will be sent. """ dest_uri = params.get("migrate_dest_uri") remote_host = params.get("migrate_dest_host") copy_option = params.get("copy_storage_option", "") username = params.get("remote_user") password = params.get("migrate_dest_pwd") timeout = int(params.get("thread_timeout", 1200)) options = "--live %s --unsafe" % copy_option # Get vm ip for remote checking if vm.is_dead(): vm.start() vm.wait_for_login() vms_ip = {} vms_ip[vm.name] = vm.get_address() logging.debug("VM %s IP: %s", vm.name, vms_ip[vm.name]) # Start to load stress stress_type = params.get("migrate_stress_type") if stress_type == "cpu": params['stress_args'] = "--cpu 2 --quiet --timeout 60" elif stress_type == "memory": params['stress_args'] = "--vm 2 --vm-bytes 256M --vm-keep --timeout 60" if stress_type is not None: utils_test.load_stress("stress_in_vms", params=params, vms=[vm]) cp_mig = utlv.MigrationTest() migration_thread = threading.Thread(target=cp_mig.thread_func_migration, args=(vm, dest_uri, options)) migration_thread.start() # Wait for migration launched time.sleep(5) job_ret = virsh.domjobinfo(vm.name, debug=True) if job_ret.exit_status: test.error("Prepare migration for blockjob failed.") # Execute some qemu monitor commands pause_cmd = "block-job-pause %s" % block_target resume_cmd = "block-job-resume %s" % block_target cancel_cmd = "block-job-cancel %s" % block_target complete_cmd = "block-job-complete %s" % block_target blockjob_failures = [] try: if blockjob_type == "cancel": virsh.qemu_monitor_command(vm.name, cancel_cmd, debug=True, ignore_status=False) elif blockjob_type == "pause_resume": virsh.qemu_monitor_command(vm.name, pause_cmd, debug=True, ignore_status=False) # TODO: Check whether it is paused. virsh.qemu_monitor_command(vm.name, resume_cmd, debug=True, ignore_status=False) elif blockjob_type == "complete": virsh.qemu_monitor_command(vm.name, complete_cmd, debug=True, ignore_status=False) except process.CmdError as detail: blockjob_failures.append(str(detail)) # Job info FYI virsh.domjobinfo(vm.name, debug=True) if len(blockjob_failures): timeout = 30 migration_thread.join(timeout) if migration_thread.isAlive(): logging.error("Migrate %s timeout.", migration_thread) cp_mig.RET_LOCK.acquire() cp_mig.RET_MIGRATION = False cp_mig.RET_LOCK.release() if len(blockjob_failures): cp_mig.cleanup_dest_vm(vm, None, dest_uri) test.fail("Run qemu monitor command failed %s" % blockjob_failures) check_ip_failures = [] if cp_mig.RET_MIGRATION: try: utils_test.check_dest_vm_network(vm, vms_ip[vm.name], remote_host, username, password) except exceptions.TestFail as detail: check_ip_failures.append(str(detail)) cp_mig.cleanup_dest_vm(vm, None, dest_uri) if blockjob_type in ["cancel", "complete"]: test.fail("Storage migration passed even after " "cancellation.") else: cp_mig.cleanup_dest_vm(vm, None, dest_uri) if blockjob_type in ["cancel", "complete"]: logging.error("Expected Migration Error for %s", blockjob_type) return else: test.fail("Command blockjob does not work well under " "storage copied migration.") if len(check_ip_failures): test.fail("Check IP failed:%s" % check_ip_failures)
def run(test, params, env): """ Test: vcpu hotplug. The command can change the number of virtual CPUs for VM. 1.Prepare test environment,destroy or suspend a VM. 2.Perform virsh setvcpus operation. 3.Recover test environment. 4.Confirm the test result. """ vm_name = params.get("main_vm") min_count = int(params.get("setvcpus_min_count", "1")) max_count = int(params.get("setvcpus_max_count", "2")) test_times = int(params.get("setvcpus_test_times", "1")) stress_type = params.get("stress_type", "") stress_param = params.get("stress_param", "") add_by_virsh = ("yes" == params.get("add_by_virsh")) del_by_virsh = ("yes" == params.get("del_by_virsh")) hotplug_timeout = int(params.get("hotplug_timeout", 30)) test_set_max = max_count * 2 # Save original configuration orig_config_xml = libvirt_xml.VMXML.new_from_inactive_dumpxml(vm_name) # Set min/max of vcpu libvirt_xml.VMXML.set_vm_vcpus(vm_name, test_set_max, min_count, topology_correction=True) # prepare VM instance vm = libvirt_vm.VM(vm_name, params, test.bindir, env.get("address_cache")) # prepare guest-agent service vm.prepare_guest_agent() # Increase the workload load_vms = [] if stress_type in ['cpu', 'memory', 'io']: params["stress_args"] = stress_param load_vms.append(vm) if stress_type in ['cpu', 'memory']: utils_test.load_stress("stress_in_vms", params, vms=load_vms) else: utils_test.load_stress("iozone_in_vms", params, vms=load_vms) session = vm.wait_for_login() try: # Clear dmesg before set vcpu session.cmd("dmesg -c") for i in range(test_times): # 1. Add vcpu add_result = libvirt.hotplug_domain_vcpu(vm, max_count, add_by_virsh) add_status = add_result.exit_status # 1.1 check add status if add_status: if add_result.stderr.count("support"): test.cancel("vcpu hotplug not supported, " "no need to test any more:\n %s" % add_result.stderr.strip()) test.fail("Test failed for:\n %s" % add_result.stderr.strip()) if not utils_misc.wait_for(lambda: utils_misc.check_if_vm_vcpu_match(max_count, vm), hotplug_timeout, text="wait for vcpu online"): test.fail("vcpu hotplug failed") if 'ppc' not in platform.machine(): # 1.2 check dmesg domain_add_dmesg = session.cmd_output("dmesg -c") dmesg1 = "CPU%d has been hot-added" % (max_count - 1) dmesg2 = "CPU %d got hotplugged" % (max_count - 1) if (not domain_add_dmesg.count(dmesg1) and not domain_add_dmesg.count(dmesg2)): test.fail("Cannot find hotplug info in dmesg: %s" % domain_add_dmesg) # 1.3 check cpu related file online_cmd = "cat /sys/devices/system/cpu/cpu%d/online" \ % (max_count - 1) st, ot = session.cmd_status_output(online_cmd) if st: test.fail("Cannot find CPU%d after hotplug" % (max_count - 1)) # 1.4 check online if not ot.strip().count("1"): test.fail("CPU%d is not online after hotplug: %s" % ((max_count - 1), ot)) # 1.5 check online interrupts info inter_on_output = session.cmd_output("cat /proc/interrupts") if not inter_on_output.count("CPU%d" % (int(max_count) - 1)): test.fail("CPU%d can not be found in " "/proc/interrupts when it's online:%s" % ((int(max_count) - 1), inter_on_output)) # 1.6 offline vcpu off_st = session.cmd_status("echo 0 > " "/sys/devices/system/cpu/cpu%d/online" % (max_count - 1)) if off_st: test.fail("Set cpu%d offline failed!" % (max_count - 1)) # 1.7 check offline interrupts info inter_off_output = session.cmd_output("cat /proc/interrupts") if inter_off_output.count("CPU%d" % (int(max_count) - 1)): test.fail("CPU%d can be found in /proc/interrupts" " when it's offline" % (int(max_count) - 1)) # 1.8 online vcpu on_st = session.cmd_status("echo 1 > " "/sys/devices/system/cpu/cpu%d/online" % (max_count - 1)) if on_st: test.fail("Set cpu%d online failed!" % (max_count - 1)) # 2. Del vcpu del_result = libvirt.hotplug_domain_vcpu(vm, min_count, del_by_virsh, hotplug=False) del_status = del_result.exit_status if del_status: logging.info("del_result: %s" % del_result.stderr.strip()) # A qemu older than 1.5 or an unplug for 1.6 will result in # the following failure. # TODO: when CPU-hotplug feature becomes stable and strong, # remove these codes used to handle kinds of exceptions if re.search("The command cpu-del has not been found", del_result.stderr): test.cancel("vcpu hotunplug not supported") if re.search("cannot change vcpu count", del_result.stderr): test.cancel("unhotplug failed") if re.search("got wrong number of vCPU pids from QEMU monitor", del_result.stderr): test.cancel("unhotplug failed") # process all tips that contains keyword 'support' # for example, "unsupported"/"hasn't been support" and so on if re.search("support", del_result.stderr): test.cancel("vcpu hotunplug not supported") # besides above, regard it failed test.fail("Test fail for:\n %s" % del_result.stderr.strip()) if not utils_misc.wait_for(lambda: utils_misc.check_if_vm_vcpu_match(min_count, vm), hotplug_timeout, text="wait for vcpu offline"): test.fail("vcpu hotunplug failed") if 'ppc' not in platform.machine(): domain_del_dmesg = session.cmd_output("dmesg -c") if not domain_del_dmesg.count("CPU %d is now offline" % (max_count - 1)): test.fail("Cannot find hot-unplug info in dmesg: %s" % domain_del_dmesg) except exceptions.TestCancel: # So far, QEMU doesn't support unplug vcpu, # unplug operation will encounter kind of errors. pass finally: utils_test.unload_stress("stress_in_vms", params, load_vms) if session: session.close() # Cleanup orig_config_xml.sync()
def set_condition(vm_name, condn, reset=False, guestbt=None): """ Set domain to given state or reset it. """ bt = None if not reset: if condn == "avocadotest": bt = utils_test.run_avocado_bg(vm, params, test) if not bt: test.cancel("guest stress failed to start") # Allow stress to start time.sleep(condn_sleep_sec) return bt elif condn == "stress": utils_test.load_stress("stress_in_vms", params=params, vms=[vm]) elif condn in ["save", "managedsave"]: # No action pass elif condn == "suspend": result = virsh.suspend(vm_name, ignore_status=True, debug=True) libvirt.check_exit_status(result) elif condn == "hotplug": result = virsh.setvcpus(vm_name, max_vcpu, "--live", ignore_status=True, debug=True) libvirt.check_exit_status(result) exp_vcpu = {'max_config': max_vcpu, 'max_live': max_vcpu, 'cur_config': current_vcpu, 'cur_live': max_vcpu, 'guest_live': max_vcpu} result = utils_hotplug.check_vcpu_value(vm, exp_vcpu, option="--live") elif condn == "host_smt": if cpu.get_cpu_arch() == 'power9': result = process.run("ppc64_cpu --smt=4", shell=True) else: test.cancel("Host SMT changes not allowed during guest live") else: logging.debug("No operation for the domain") else: if condn == "save": save_file = os.path.join(data_dir.get_tmp_dir(), vm_name + ".save") result = virsh.save(vm_name, save_file, ignore_status=True, debug=True) libvirt.check_exit_status(result) time.sleep(condn_sleep_sec) if os.path.exists(save_file): result = virsh.restore(save_file, ignore_status=True, debug=True) libvirt.check_exit_status(result) os.remove(save_file) else: test.error("No save file for domain restore") elif condn == "managedsave": result = virsh.managedsave(vm_name, ignore_status=True, debug=True) libvirt.check_exit_status(result) time.sleep(condn_sleep_sec) result = virsh.start(vm_name, ignore_status=True, debug=True) libvirt.check_exit_status(result) elif condn == "suspend": result = virsh.resume(vm_name, ignore_status=True, debug=True) libvirt.check_exit_status(result) elif condn == "avocadotest": guestbt.join(ignore_status=True) elif condn == "stress": utils_test.unload_stress("stress_in_vms", params=params, vms=[vm]) elif condn == "hotplug": result = virsh.setvcpus(vm_name, current_vcpu, "--live", ignore_status=True, debug=True) libvirt.check_exit_status(result) exp_vcpu = {'max_config': max_vcpu, 'max_live': current_vcpu, 'cur_config': current_vcpu, 'cur_live': current_vcpu, 'guest_live': current_vcpu} result = utils_hotplug.check_vcpu_value(vm, exp_vcpu, option="--live") elif condn == "host_smt": result = process.run("ppc64_cpu --smt=2", shell=True) # Change back the host smt result = process.run("ppc64_cpu --smt=4", shell=True) # Work around due to known cgroup issue after cpu hot(un)plug # sequence root_cpuset_path = utils_cgroup.get_cgroup_mountpoint("cpuset") machine_cpuset_paths = [] if os.path.isdir(os.path.join(root_cpuset_path, "machine.slice")): machine_cpuset_paths.append(os.path.join(root_cpuset_path, "machine.slice")) if os.path.isdir(os.path.join(root_cpuset_path, "machine")): machine_cpuset_paths.append(os.path.join(root_cpuset_path, "machine")) if not machine_cpuset_paths: logging.warning("cgroup cpuset might not recover properly " "for guests after host smt changes, " "restore it manually") root_cpuset_cpus = os.path.join(root_cpuset_path, "cpuset.cpus") for path in machine_cpuset_paths: machine_cpuset_cpus = os.path.join(path, "cpuset.cpus") # check if file content differs cmd = "diff %s %s" % (root_cpuset_cpus, machine_cpuset_cpus) if process.system(cmd, verbose=True, ignore_status=True): cmd = "cp %s %s" % (root_cpuset_cpus, machine_cpuset_cpus) process.system(cmd, verbose=True) else: logging.debug("No need recover the domain") return bt
def run(test, params, env): """ Stress test for the hotplug feature of serial device """ vm_name = params.get("main_vm", "vm1") char_dev = params.get("char_dev", "file") hotplug_type = params.get("hotplug_type", "qmp") load_type = params.get("load_type", "") load_params = params.get("load_params", "") test_count = int(params.get("test_count", 5)) test_type = params.get("test_type", "multi") tmp_dir = os.path.join(test.tmpdir, "hotplug_serial_load") if not os.path.exists(tmp_dir): os.mkdir(tmp_dir) os.chmod(tmp_dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) load_vms = [] if load_type in ['cpu', 'memory', 'io']: params["stress_args"] = load_params load_vms.append( libvirt_vm.VM(vm_name, params, test.bindir, env.get("address_cache"))) vm = env.get_vm(vm_name) session = vm.wait_for_login() def prepare_channel_xml(to_file, char_type, index=1, id=0): params = {} mode = '' if char_type == "file": channel_type = char_type channel_path = ("%s/%s%s" % (tmp_dir, char_type, index)) elif char_type == "socket": channel_type = 'unix' channel_path = ("%s/%s%s" % (tmp_dir, char_type, index)) mode = 'bind' elif char_type == "pty": channel_type = char_type channel_path = ("/dev/pts/%s" % id) params = { 'channel_type_name': channel_type, 'source_path': channel_path, 'source_mode': mode, 'target_type': 'virtio', 'target_name': char_type + str(index) } channel_xml = utlv.create_channel_xml(params, alias=True, address=True) shutil.copyfile(channel_xml.xml, to_file) def hotplug_device(hotplug_type, char_dev, index=1, id=0): if hotplug_type == "qmp": char_add_opt = "chardev-add " dev_add_opt = "device_add virtserialport,chardev=" if char_dev == "file": char_add_opt += ("file,path=%s/file%s,id=file%s" % (tmp_dir, index, index)) dev_add_opt += ( "file%s,name=file%s,bus=virtio-serial0.0,id=file%s" % (index, index, index)) elif char_dev == "socket": char_add_opt += ( "socket,path=%s/socket%s,server,nowait,id=socket%s" % (tmp_dir, index, index)) dev_add_opt += ( "socket%s,name=socket%s,bus=virtio-serial0.0,id=socket%s" % (index, index, index)) elif char_dev == "pty": char_add_opt += "pty,path=/dev/pts/%s,id=pty%s" % (id, index) dev_add_opt += ( "pty%s,name=pty%s,bus=virtio-serial0.0,id=pty%s" % (index, index, index)) virsh.qemu_monitor_command(vm_name, char_add_opt, "--hmp") virsh.qemu_monitor_command(vm_name, dev_add_opt, "--hmp") elif hotplug_type == "attach": xml_file = "%s/xml_%s%s" % (tmp_dir, char_dev, index) if char_dev in ["file", "socket"]: prepare_channel_xml(xml_file, char_dev, index) elif char_dev == "pty": prepare_channel_xml(xml_file, char_dev, index, id) virsh.attach_device(vm_name, xml_file, flagstr="--live") def confirm_hotplug_result(char_dev, index=1, id=0): result = virsh.qemu_monitor_command(vm_name, "info qtree", "--hmp") h_o = result.stdout.strip() chardev_c = h_o.count("chardev = %s%s" % (char_dev, index)) name_c = h_o.count("name = \"%s%s\"" % (char_dev, index)) if chardev_c == 0 and name_c == 0: raise error.TestFail("Cannot get serial device info: '%s'" % h_o) tmp_file = "%s/%s%s" % (tmp_dir, char_dev, index) serial_file = "/dev/virtio-ports/%s%s" % (char_dev, index) if char_dev == "file": session.cmd("echo test > %s" % serial_file) f = open(tmp_file, "r") output = f.read() f.close() elif char_dev == "socket": session.cmd("echo test > /tmp/file") sock = socket.socket(socket.AF_UNIX) sock.connect(tmp_file) session.cmd("dd if=/tmp/file of=%s" % serial_file) output = sock.recv(1024) sock.close() elif char_dev == "pty": session.cmd("echo test > /tmp/file") session.cmd("dd if=/tmp/file of=%s &" % serial_file) dev_file = "/dev/pts/%s" % id if not os.path.exists(dev_file): raise error.TestFail("%s doesn't exist." % dev_file) p = subprocess.Popen(["/usr/bin/cat", dev_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) while True: output = p.stdout.readline() if output or p.poll(): break time.sleep(0.2) p.kill() if not output.count("test"): err_info = "%s device file doesn't match 'test':%s" % (char_dev, output) raise error.TestFail(err_info) def unhotplug_serial_device(hotplug_type, char_dev, index=1): if hotplug_type == "qmp": del_dev_opt = "device_del %s%s" % (char_dev, index) del_char_opt = "chardev-remove %s%s" % (char_dev, index) virsh.qemu_monitor_command(vm_name, del_dev_opt, "--hmp") virsh.qemu_monitor_command(vm_name, del_char_opt, "--hmp") elif hotplug_type == "attach": xml_file = "%s/xml_%s%s" % (tmp_dir, char_dev, index) virsh.detach_device(vm_name, xml_file, flagstr="--live") def confirm_unhotplug_result(char_dev, index=1): serial_file = "/dev/virtio-ports/%s%s" % (char_dev, index) result = virsh.qemu_monitor_command(vm_name, "info qtree", "--hmp") uh_o = result.stdout.strip() if uh_o.count("chardev = %s%s" % (char_dev, index)): raise error.TestFail("Still can get serial device info: '%s'" % uh_o) if not session.cmd_status("test -e %s" % serial_file): raise error.TestFail("File '%s' still exists after unhotplug" % serial_file) # run test case try: # increase workload if load_type in ['cpu', 'memory']: utils_test.load_stress("stress_in_vms", load_vms, params) else: utils_test.load_stress("iozone_in_vms", load_vms, params) if test_type == "multi": for i in range(test_count): if char_dev == "pty": ptsid = utils_misc.aton( utils_misc.get_dev_pts_max_id()) + 1 else: ptsid = 0 hotplug_device(hotplug_type, char_dev, i + 1, id=ptsid) confirm_hotplug_result(char_dev, i + 1, id=ptsid) unhotplug_serial_device(hotplug_type, char_dev, i + 1) confirm_unhotplug_result(char_dev, i + 1) elif test_type == "circle": if char_dev != "all": for i in range(test_count): if char_dev == "pty": ptsid = utils_misc.aton( utils_misc.get_dev_pts_max_id()) + 1 else: ptsid = 0 #1.hotplug serial device hotplug_device(hotplug_type, char_dev, id=ptsid) #2.confirm hotplug result confirm_hotplug_result(char_dev, id=ptsid) #3.unhotplug serial device unhotplug_serial_device(hotplug_type, char_dev) #4.confirm unhotplug result confirm_unhotplug_result(char_dev) else: for i in range(test_count): #1.hotplug serial device hotplug_device(hotplug_type, "file") hotplug_device(hotplug_type, "socket") ptsid = utils_misc.aton( utils_misc.get_dev_pts_max_id()) + 1 hotplug_device(hotplug_type, "pty", id=ptsid) #2.confirm hotplug result confirm_hotplug_result("file") confirm_hotplug_result("socket") confirm_hotplug_result("pty", id=ptsid) #3.unhotplug serial device unhotplug_serial_device(hotplug_type, "file") unhotplug_serial_device(hotplug_type, "socket") unhotplug_serial_device(hotplug_type, "pty") #4.confirm unhotplug result confirm_unhotplug_result("file") confirm_unhotplug_result("socket") confirm_unhotplug_result("pty") finally: session.close() if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir)
def run(test, params, env): """ Stress test for the hotplug feature of serial device """ vm_name = params.get("main_vm", "vm1") status_error = "yes" == params.get("status_error", "no") char_dev = params.get("char_dev", "file") hotplug_type = params.get("hotplug_type", "qmp") load_type = params.get("load_type", "") load_params = params.get("load_params", "") test_count = int(params.get("test_count", 5)) test_type = params.get("test_type", "multi") tmp_dir = os.path.join(test.tmpdir, "hotplug_serial_load") if not os.path.exists(tmp_dir): os.mkdir(tmp_dir) os.chmod(tmp_dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) load_vms = [] if load_type in ['cpu', 'memory', 'io']: params["stress_args"] = load_params load_vms.append(libvirt_vm.VM(vm_name, params, test.bindir, env.get("address_cache"))) vm = env.get_vm(vm_name) session = vm.wait_for_login() def prepare_channel_xml(to_file, char_type, index=1, id=0): params = {} mode = '' if char_type == "file": channel_type = char_type channel_path = ("%s/%s%s" % (tmp_dir, char_type, index)) elif char_type == "socket": channel_type = 'unix' channel_path = ("%s/%s%s" % (tmp_dir, char_type, index)) mode = 'bind' elif char_type == "pty": channel_type = char_type channel_path = ("/dev/pts/%s" % id) params = {'channel_type_name': channel_type, 'source_path': channel_path, 'source_mode': mode, 'target_type': 'virtio', 'target_name': char_type + str(index)} channel_xml = utlv.create_channel_xml(params, alias=True, address=True) shutil.copyfile(channel_xml.xml, to_file) def hotplug_device(hotplug_type, char_dev, index=1, id=0): if hotplug_type == "qmp": char_add_opt = "chardev-add " dev_add_opt = "device_add virtserialport,chardev=" if char_dev == "file": char_add_opt += ("file,path=%s/file%s,id=file%s" % (tmp_dir, index, index)) dev_add_opt += ("file%s,name=file%s,bus=virtio-serial0.0,id=file%s" % (index, index, index)) elif char_dev == "socket": char_add_opt += ("socket,path=%s/socket%s,server,nowait,id=socket%s" % (tmp_dir, index, index)) dev_add_opt += ("socket%s,name=socket%s,bus=virtio-serial0.0,id=socket%s" % (index, index, index)) elif char_dev == "pty": char_add_opt += "pty,path=/dev/pts/%s,id=pty%s" % (id, index) dev_add_opt += ("pty%s,name=pty%s,bus=virtio-serial0.0,id=pty%s" % (index, index, index)) virsh.qemu_monitor_command(vm_name, char_add_opt, "--hmp") virsh.qemu_monitor_command(vm_name, dev_add_opt, "--hmp") elif hotplug_type == "attach": xml_file = "%s/xml_%s%s" % (tmp_dir, char_dev, index) if char_dev in ["file", "socket"]: prepare_channel_xml(xml_file, char_dev, index) elif char_dev == "pty": prepare_channel_xml(xml_file, char_dev, index, id) virsh.attach_device(vm_name, xml_file, flagstr="--live") def confirm_hotplug_result(char_dev, index=1, id=0): result = virsh.qemu_monitor_command(vm_name, "info qtree", "--hmp") h_o = result.stdout.strip() chardev_c = h_o.count("chardev = %s%s" % (char_dev, index)) name_c = h_o.count("name = \"%s%s\"" % (char_dev, index)) if chardev_c == 0 and name_c == 0: raise error.TestFail("Cannot get serial device info: '%s'" % h_o) tmp_file = "%s/%s%s" % (tmp_dir, char_dev, index) serial_file = "/dev/virtio-ports/%s%s" % (char_dev, index) if char_dev == "file": session.cmd("echo test > %s" % serial_file) f = open(tmp_file, "r") output = f.read() f.close() elif char_dev == "socket": session.cmd("echo test > /tmp/file") sock = socket.socket(socket.AF_UNIX) sock.connect(tmp_file) session.cmd("dd if=/tmp/file of=%s" % serial_file) output = sock.recv(1024) sock.close() elif char_dev == "pty": session.cmd("echo test > /tmp/file") session.cmd("dd if=/tmp/file of=%s &" % serial_file) dev_file = "/dev/pts/%s" % id if not os.path.exists(dev_file): raise error.TestFail("%s doesn't exist." % dev_file) p = subprocess.Popen(["/usr/bin/cat", dev_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) while True: output = p.stdout.readline() if output or p.poll(): break time.sleep(0.2) p.kill() if not output.count("test"): err_info = "%s device file doesn't match 'test':%s" % (char_dev, output) raise error.TestFail(err_info) def unhotplug_serial_device(hotplug_type, char_dev, index=1): if hotplug_type == "qmp": del_dev_opt = "device_del %s%s" % (char_dev, index) del_char_opt = "chardev-remove %s%s" % (char_dev, index) virsh.qemu_monitor_command(vm_name, del_dev_opt, "--hmp") virsh.qemu_monitor_command(vm_name, del_char_opt, "--hmp") elif hotplug_type == "attach": xml_file = "%s/xml_%s%s" % (tmp_dir, char_dev, index) virsh.detach_device(vm_name, xml_file, flagstr="--live") def confirm_unhotplug_result(char_dev, index=1): serial_file = "/dev/virtio-ports/%s%s" % (char_dev, index) result = virsh.qemu_monitor_command(vm_name, "info qtree", "--hmp") uh_o = result.stdout.strip() if uh_o.count("chardev = %s%s" % (char_dev, index)): raise error.TestFail("Still can get serial device info: '%s'" % uh_o) if not session.cmd_status("test -e %s" % serial_file): raise error.TestFail("File '%s' still exists after unhotplug" % serial_file) # run test case try: # increase workload if load_type in ['cpu', 'memory']: utils_test.load_stress("stress_in_vms", load_vms, params) else: utils_test.load_stress("iozone_in_vms", load_vms, params) if test_type == "multi": for i in range(test_count): if char_dev == "pty": ptsid = utils_misc.aton(utils_misc.get_dev_pts_max_id()) + 1 else: ptsid = 0 hotplug_device(hotplug_type, char_dev, i + 1, id=ptsid) confirm_hotplug_result(char_dev, i + 1, id=ptsid) unhotplug_serial_device(hotplug_type, char_dev, i + 1) confirm_unhotplug_result(char_dev, i + 1) elif test_type == "circle": if char_dev != "all": for i in range(test_count): if char_dev == "pty": ptsid = utils_misc.aton(utils_misc.get_dev_pts_max_id()) + 1 else: ptsid = 0 #1.hotplug serial device hotplug_device(hotplug_type, char_dev, id=ptsid) #2.confirm hotplug result confirm_hotplug_result(char_dev, id=ptsid) #3.unhotplug serial device unhotplug_serial_device(hotplug_type, char_dev) #4.confirm unhotplug result confirm_unhotplug_result(char_dev) else: for i in range(test_count): #1.hotplug serial device hotplug_device(hotplug_type, "file") hotplug_device(hotplug_type, "socket") ptsid = utils_misc.aton(utils_misc.get_dev_pts_max_id()) + 1 hotplug_device(hotplug_type, "pty", id=ptsid) #2.confirm hotplug result confirm_hotplug_result("file") confirm_hotplug_result("socket") confirm_hotplug_result("pty", id=ptsid) #3.unhotplug serial device unhotplug_serial_device(hotplug_type, "file") unhotplug_serial_device(hotplug_type, "socket") unhotplug_serial_device(hotplug_type, "pty") #4.confirm unhotplug result confirm_unhotplug_result("file") confirm_unhotplug_result("socket") confirm_unhotplug_result("pty") finally: session.close() if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir)
def verify_migration_speed(test, params, env): """ Check if migration speed is effective with twice migration. """ vms = env.get_all_vms() src_uri = params.get("migrate_src_uri", "qemu+ssh://EXAMPLE/system") dest_uri = params.get("migrate_dest_uri", "qemu+ssh://EXAMPLE/system") if not len(vms): test.cancel("Please provide migrate_vms for test.") if src_uri.count('///') or src_uri.count('EXAMPLE'): test.cancel("The src_uri '%s' is invalid" % src_uri) if dest_uri.count('///') or dest_uri.count('EXAMPLE'): test.cancel("The dest_uri '%s' is invalid" % dest_uri) remote_host = params.get("migrate_dest_host") username = params.get("migrate_dest_user", "root") password = params.get("migrate_dest_pwd") # Config ssh autologin for remote host ssh_key.setup_ssh_key(remote_host, username, password, port=22) # Check migrated vms' state for vm in vms: if vm.is_dead(): vm.start() load_vm_names = params.get("load_vms").split() # vms for load load_vms = [] for vm_name in load_vm_names: load_vms.append( libvirt_vm.VM(vm_name, params, test.bindir, env.get("address_cache"))) params["load_vms"] = load_vms bandwidth = int(params.get("bandwidth", "4")) stress_type = params.get("stress_type", "load_vms_booting") migration_type = params.get("migration_type", "orderly") thread_timeout = int(params.get("thread_timeout", "60")) delta = float(params.get("allowed_delta", "0.1")) virsh_migrate_timeout = int(params.get("virsh_migrate_timeout", "60")) # virsh migrate options virsh_migrate_options = "--live --unsafe --timeout %s" % virsh_migrate_timeout # Migrate vms to remote host mig_first = utlv.MigrationTest() virsh_dargs = {"debug": True} for vm in vms: set_get_speed(vm.name, bandwidth, virsh_dargs=virsh_dargs) vm.wait_for_login() utils_test.load_stress(stress_type, vms, params) mig_first.do_migration(vms, src_uri, dest_uri, migration_type, options=virsh_migrate_options, thread_timeout=thread_timeout) for vm in vms: mig_first.cleanup_dest_vm(vm, None, dest_uri) # Keep it clean for second migration if vm.is_alive(): vm.destroy() # Migrate vms again with new bandwidth second_bandwidth = params.get("second_bandwidth", "times") if second_bandwidth == "half": second_bandwidth = bandwidth / 2 speed_times = 2 elif second_bandwidth == "times": second_bandwidth = bandwidth * 2 speed_times = 0.5 elif second_bandwidth == "same": second_bandwidth = bandwidth speed_times = 1 # Migrate again for vm in vms: if vm.is_dead(): vm.start() vm.wait_for_login() set_get_speed(vm.name, second_bandwidth, virsh_dargs=virsh_dargs) utils_test.load_stress(stress_type, vms, params) mig_second = utlv.MigrationTest() mig_second.do_migration(vms, src_uri, dest_uri, migration_type, options=virsh_migrate_options, thread_timeout=thread_timeout) for vm in vms: mig_second.cleanup_dest_vm(vm, None, dest_uri) fail_info = [] # Check whether migration failed if len(fail_info): test.fail(fail_info) for vm in vms: first_time = mig_first.mig_time[vm.name] second_time = mig_second.mig_time[vm.name] logging.debug( "Migration time for %s:\n" "Time with Bandwidth '%s' first: %s\n" "Time with Bandwidth '%s' second: %s", vm.name, bandwidth, first_time, second_bandwidth, second_time) shift = float(abs(first_time * speed_times - second_time)) / float(second_time) logging.debug("Shift:%s", shift) if delta < shift: fail_info.append( "Spent time for migrating %s is intolerable." % vm.name) # Check again for speed result if len(fail_info): test.fail(fail_info)
def verify_migration_speed(test, params, env): """ Check if migration speed is effective with twice migration. """ vms = env.get_all_vms() src_uri = params.get("migrate_src_uri", "qemu+ssh://EXAMPLE/system") dest_uri = params.get("migrate_dest_uri", "qemu+ssh://EXAMPLE/system") if src_uri.count('///') or src_uri.count('EXAMPLE'): raise error.TestNAError("The src_uri '%s' is invalid", src_uri) if dest_uri.count('///') or dest_uri.count('EXAMPLE'): raise error.TestNAError("The dest_uri '%s' is invalid", dest_uri) remote_host = params.get("migrate_dest_host") username = params.get("migrate_dest_user", "root") password = params.get("migrate_dest_pwd") # Config ssh autologin for remote host ssh_key.setup_ssh_key(remote_host, username, password, port=22) # Check migrated vms' state for vm in vms: if vm.is_dead(): vm.start() load_vm_names = params.get("load_vms").split() # vms for load load_vms = [] for vm_name in load_vm_names: load_vms.append(libvirt_vm.VM(vm_name, params, test.bindir, env.get("address_cache"))) bandwidth = int(params.get("bandwidth", "4")) stress_type = params.get("stress_type", "load_vms_booting") migration_type = params.get("migration_type", "orderly") thread_timeout = int(params.get("thread_timeout", "60")) delta = float(params.get("allowed_delta", "0.1")) virsh_migrate_timeout = int(params.get("virsh_migrate_timeout", 60)) # virsh migrate options virsh_migrate_options = "--live --timeout %s", virsh_migrate_timeout # Migrate vms to remote host mig_first = utlv.MigrationTest() virsh_dargs = {"debug": True} for vm in vms: set_get_speed(vm.name, bandwidth, virsh_dargs=virsh_dargs) vm.wait_for_login() utils_test.load_stress(stress_type, vms, params) mig_first.do_migration(vms, src_uri, dest_uri, migration_type, options=virsh_migrate_options, thread_timeout=thread_timeout) for vm in vms: mig_first.cleanup_dest_vm(vm, None, dest_uri) # Keep it clean for second migration if vm.is_alive(): vm.destroy() # Migrate vms again with new bandwidth second_bandwidth = params.get("second_bandwidth", "times") if second_bandwidth == "half": second_bandwidth = bandwidth / 2 speed_times = 2 elif second_bandwidth == "times": second_bandwidth = bandwidth * 2 speed_times = 0.5 elif second_bandwidth == "same": second_bandwidth = bandwidth speed_times = 1 # Migrate again for vm in vms: if vm.is_dead(): vm.start() vm.wait_for_login() set_get_speed(vm.name, second_bandwidth, virsh_dargs=virsh_dargs) utils_test.load_stress(stress_type, vms, params) mig_second = utlv.MigrationTest() mig_second.do_migration(vms, src_uri, dest_uri, migration_type, options="--live", thread_timeout=thread_timeout) for vm in vms: mig_second.cleanup_dest_vm(vm, None, dest_uri) fail_info = [] # Check whether migration failed if len(fail_info): raise error.TestFail(fail_info) for vm in vms: first_time = mig_first.mig_time[vm.name] second_time = mig_second.mig_time[vm.name] logging.debug("Migration time for %s:\n" "Time with Bandwidth '%s' first: %s\n" "Time with Bandwidth '%s' second: %s", vm.name, bandwidth, first_time, second_bandwidth, second_time) shift = float(abs(first_time * speed_times - second_time)) / float(second_time) logging.debug("Shift:%s", shift) if delta < shift: fail_info.append("Spent time for migrating %s is intolerable." % vm.name) # Check again for speed result if len(fail_info): raise error.TestFail(fail_info)
def run(test, params, env): """ Test migration under stress. """ vm_names = params.get("vms").split() if len(vm_names) < 2: test.cancel("Provide enough vms for migration") src_uri = "qemu:///system" dest_uri = libvirt_vm.complete_uri( params.get("migrate_dest_host", "EXAMPLE")) if dest_uri.count('///') or dest_uri.count('EXAMPLE'): test.cancel("The dest_uri '%s' is invalid" % dest_uri) # Migrated vms' instance vms = env.get_all_vms() params["load_vms"] = list(vms) cpu = int(params.get("smp", 1)) memory = int(params.get("mem")) * 1024 stress_tool = params.get("stress_tool", "") stress_type = params.get("migration_stress_type") require_stress_tool = "stress" in stress_tool vm_bytes = params.get("stress_vm_bytes", "128M") stress_args = params.get("stress_args") migration_type = params.get("migration_type") start_migration_vms = params.get("start_migration_vms", "yes") == "yes" thread_timeout = int(params.get("thread_timeout", 120)) # Set vm_bytes for start_cmd mem_total = utils_memory.memtotal() vm_reserved = len(vms) * memory if vm_bytes == "half": vm_bytes = (mem_total - vm_reserved) / 2 elif vm_bytes == "shortage": vm_bytes = mem_total - vm_reserved + 524288 if "vm-bytes" in stress_args: params["stress_args"] = stress_args % vm_bytes # Ensure stress tool is available in host if require_stress_tool and stress_type == "stress_on_host": utils_test.load_stress("stress_on_host", params) for vm in vms: # Keep vm dead for edit if vm.is_alive(): vm.destroy() set_cpu_memory(vm.name, cpu, memory) try: if start_migration_vms: for vm in vms: vm.start() vm.wait_for_login() # configure stress in VM if require_stress_tool and stress_type == "stress_in_vms": utils_test.load_stress("stress_in_vms", params, vms) do_stress_migration(vms, src_uri, dest_uri, migration_type, test, params, thread_timeout) finally: logging.debug("Cleanup vms...") params["connect_uri"] = src_uri for vm in vms: utils_test.libvirt.MigrationTest().cleanup_dest_vm( vm, None, dest_uri) # Try to start vms in source once vms in destination are # cleaned up if not vm.is_alive(): vm.start() vm.wait_for_login() utils_test.unload_stress(stress_type, params, vms)