def install_virt_pkgs(vm_session): """ Install required virt packages for nested virt test :param vm_session: session to the vm :raises: exceptions.TestError if installation fails """ pkg_names = ['libvirt', 'qemu-kvm'] LOG.info("Virt packages will be installed") pkg_mgr = utils_package.package_manager(vm_session, pkg_names) if not pkg_mgr.install(): raise exceptions.TestError("Package '%s' installation " "fails" % pkg_names)
def run(test, params, env): """ Test migration with glusterfs. """ def create_or_clean_backend_dir(g_uri, params, session=None, is_clean=False): """ Create/cleanup backend directory :params g_uri: glusterfs uri :params params: the parameters to be checked :params session: VM/remote session object :params is_cleanup: True for cleanup backend directory; False for create one. :return: gluster_img if is_clean is equal to True """ mount_point = params.get("gluster_mount_dir") is_symlink = params.get("gluster_create_symlink") == "yes" symlink_name = params.get("gluster_symlink") gluster_img = None if not is_clean: if not utils_misc.check_exists(mount_point, session): utils_misc.make_dirs(mount_point, session) if gluster.glusterfs_is_mounted(mount_point, session): gluster.glusterfs_umount(g_uri, mount_point, session) gluster.glusterfs_mount(g_uri, mount_point, session) gluster_img = os.path.join(mount_point, disk_img) if is_symlink: utils_misc.make_symlink(mount_point, symlink_name) utils_misc.make_symlink(mount_point, symlink_name, remote_session) gluster_img = os.path.join(symlink_name, disk_img) return gluster_img else: if is_symlink: utils_misc.rm_link(symlink_name, session) gluster.glusterfs_umount(g_uri, mount_point, session) if utils_misc.check_exists(mount_point, session): utils_misc.safe_rmdir(gluster_mount_dir, session=session) # Local variables virsh_args = {"debug": True} server_ip = params["server_ip"] = params.get("remote_ip") server_user = params["server_user"] = params.get("remote_user", "root") server_pwd = params["server_pwd"] = params.get("remote_pwd") client_ip = params["client_ip"] = params.get("local_ip") client_pwd = params["client_pwd"] = params.get("local_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options", "--live --p2p --verbose") virsh_options = params.get("virsh_options", "") vol_name = params.get("vol_name") disk_format = params.get("disk_format", "qcow2") gluster_mount_dir = params.get("gluster_mount_dir") status_error = "yes" == params.get("status_error", "no") err_msg = params.get("err_msg") host_ip = params.get("gluster_server_ip", "") migrate_vm_back = params.get("migrate_vm_back", "no") == "yes" selinux_local = params.get('set_sebool_local', 'yes') == "yes" selinux_remote = params.get('set_sebool_remote', 'no') == "yes" sebool_fusefs_local = params.get('set_sebool_fusefs_local', 'yes') sebool_fusefs_remote = params.get('set_sebool_fusefs_remote', 'yes') test_dict = dict(params) test_dict["local_boolean_varible"] = "virt_use_fusefs" test_dict["remote_boolean_varible"] = "virt_use_fusefs" remote_dargs = { 'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd, 'file_path': "/etc/libvirt/libvirt.conf" } remove_pkg = False seLinuxBool = None seLinuxfusefs = None gluster_uri = None mig_result = None remove_dict = {} remote_libvirt_file = None src_libvirt_file = None # Make sure all of parameters are assigned a valid value migrate_test = migration.MigrationTest() migrate_test.check_parameters(params) extra_args = migrate_test.update_virsh_migrate_extra_args(params) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) params["virsh_migrate_connect_uri"] = libvirt_vm.complete_uri( params.get("migrate_source_host")) src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (virsh_options, postcopy_options) func_name = virsh.migrate_postcopy vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # Back up xml file. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Configure selinux if selinux_local or selinux_remote: seLinuxBool = utils_misc.SELinuxBoolean(params) seLinuxBool.setup() if sebool_fusefs_local or sebool_fusefs_remote: seLinuxfusefs = utils_misc.SELinuxBoolean(test_dict) seLinuxfusefs.setup() # Setup glusterfs disk_img = "gluster.%s" % disk_format params['disk_img'] = disk_img host_ip = gluster.setup_or_cleanup_gluster(is_setup=True, **params) logging.debug("host ip: %s ", host_ip) # Check if gluster server is deployed locally if not host_ip: logging.debug("Enable port 24007 and 49152:49216") migrate_test.migrate_pre_setup(src_uri, params, ports="24007") migrate_test.migrate_pre_setup(src_uri, params) gluster_uri = "{}:{}".format(client_ip, vol_name) else: gluster_uri = "{}:{}".format(host_ip, vol_name) remote_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") if gluster_mount_dir: # The package 'glusterfs-fuse' is not installed on target # which makes issue when trying to 'mount -t glusterfs' pkg_name = 'glusterfs-fuse' logging.debug("Check if glusterfs-fuse is installed") pkg_mgr = utils_package.package_manager(remote_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("glusterfs-fuse will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) else: remove_pkg = True gluster_img = create_or_clean_backend_dir(gluster_uri, params) create_or_clean_backend_dir(gluster_uri, params, remote_session) # Get the image path image_source = vm.get_first_disk_devices()['source'] image_info = utils_misc.get_image_info(image_source) if image_info["format"] == disk_format: disk_cmd = "cp -f %s %s" % (image_source, gluster_img) else: # Convert the disk format disk_cmd = ("qemu-img convert -f %s -O %s %s %s" % (image_info["format"], disk_format, image_source, gluster_img)) process.run("%s; chmod a+rw %s" % (disk_cmd, gluster_mount_dir), shell=True) logging.debug("Gluster Image is %s", gluster_img) gluster_backend_disk = {'disk_source_name': gluster_img} # Update disk xml with gluster image in backend dir libvirt.set_vm_disk(vm, gluster_backend_disk) remote_session.close() vm_xml_cxt = virsh.dumpxml(vm_name).stdout_text.strip() logging.debug("The VM XML with gluster disk source: \n%s", vm_xml_cxt) vm.wait_for_login().close() migrate_test.ping_vm(vm, params) remove_dict = {"do_search": '{"%s": "ssh:/"}' % dest_uri} src_libvirt_file = libvirt_config.remove_key_for_modular_daemon( remove_dict) vms = [vm] migrate_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, extra_opts=extra, **extra_args) migrate_test.ping_vm(vm, params, dest_uri) if migrate_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=client_ip, server_pwd=client_pwd, client_ip=server_ip, client_pwd=server_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine migrate_test.migrate_pre_setup(src_uri, params) remove_dict = {"do_search": ('{"%s": "ssh:/"}' % src_uri)} remote_libvirt_file = libvirt_config\ .remove_key_for_modular_daemon(remove_dict, remote_dargs) cmd = "virsh migrate %s %s %s %s" % (vm_name, options, virsh_options, src_uri) logging.debug("Start migrating: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) logging.info(cmd_result) if cmd_result.exit_status: destroy_cmd = "virsh destroy %s" % vm_name remote.run_remote_cmd(destroy_cmd, params, runner_on_target, ignore_status=False) test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) finally: logging.info("Recover test environment") migrate_test.cleanup_vm(vm, dest_uri) orig_config_xml.sync() if src_libvirt_file: src_libvirt_file.restore() if remote_libvirt_file: del remote_libvirt_file # Clean up of pre migration setup for local machine if migrate_vm_back: if 'ssh_connection' in locals(): ssh_connection.auto_recover = True migrate_test.migrate_pre_setup(src_uri, params, cleanup=True) # Cleanup selinu configuration if seLinuxBool: seLinuxBool.cleanup() if seLinuxfusefs: seLinuxfusefs.cleanup() # Disable ports 24007 and 49152:49216 if not host_ip: logging.debug("Disable 24007 and 49152:49216 in Firewall") migrate_test.migrate_pre_setup(src_uri, params, cleanup=True, ports="24007") migrate_test.migrate_pre_setup(src_uri, params, cleanup=True) gluster.setup_or_cleanup_gluster(False, **params) # Cleanup backend directory/symlink if gluster_mount_dir and gluster_uri: remote_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") create_or_clean_backend_dir(gluster_uri, params, is_clean=True) create_or_clean_backend_dir(gluster_uri, params, remote_session, True) if remove_pkg: pkg_mgr = utils_package.package_manager( remote_session, pkg_name) if pkg_mgr.is_installed(pkg_name): logging.debug("glusterfs-fuse will be uninstalled") if not pkg_mgr.remove(): logging.error("Package '%s' un-installation fails", pkg_name) remote_session.close()
def run(test, params, env): """ Test virsh migrate command. """ def set_feature(vmxml, feature, value): """ Set guest features for PPC :param state: the htm status :param vmxml: guest xml """ features_xml = vm_xml.VMFeaturesXML() if feature == 'hpt': features_xml.hpt_resizing = value elif feature == 'htm': features_xml.htm = value vmxml.features = features_xml vmxml.sync() def trigger_hpt_resize(session): """ Check the HPT order file and dmesg :param session: the session to guest :raise: test.fail if required message is not found """ hpt_order_path = "/sys/kernel/debug/powerpc/hpt_order" hpt_order = session.cmd_output('cat %s' % hpt_order_path).strip() hpt_order = int(hpt_order) logging.info('Current hpt_order is %d', hpt_order) hpt_order += 1 cmd = 'echo %d > %s' % (hpt_order, hpt_order_path) cmd_result = session.cmd_status_output(cmd) result = process.CmdResult(stderr=cmd_result[1], stdout=cmd_result[1], exit_status=cmd_result[0]) libvirt.check_exit_status(result) dmesg = session.cmd('dmesg') dmesg_content = params.get('dmesg_content').split('|') for content in dmesg_content: if content % hpt_order not in dmesg: test.fail("'%s' is missing in dmesg" % (content % hpt_order)) else: logging.info("'%s' is found in dmesg", content % hpt_order) def check_qemu_cmd_line(content, err_ignore=False): """ Check the specified content in the qemu command line :param content: the desired string to search :param err_ignore: True to return False when fail False to raise exception when fail :return: True if exist, False otherwise """ cmd = 'ps -ef|grep qemu|grep -v grep' qemu_line = results_stdout_52lts(process.run(cmd, shell=True)) if content not in qemu_line: if err_ignore: return False else: test.fail("Expected '%s' was not found in " "qemu command line" % content) return True def check_vm_network_accessed(session=None): """ The operations to the VM need to be done before or after migration happens :param session: The session object to the host :raise: test.error when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") s_ping, _ = utils_test.ping(vm.get_address(), count=10, timeout=20, output_func=logging.debug, session=session) if s_ping != 0: if session: session.close() test.fail("%s did not respond after %d sec." % (vm.name, 20)) def check_virsh_command_and_option(command, option=None): """ Check if virsh command exists :param command: the command to be checked :param option: the command option to be checked """ msg = "This version of libvirt does not support " if not virsh.has_help_command(command): test.cancel(msg + "virsh command '%s'" % command) if option and not virsh.has_command_help_match(command, option): test.cancel(msg + "virsh command '%s' with option '%s'" % (command, option)) def add_ctrls(vm_xml, dev_type="pci", dev_index="0", dev_model="pci-root"): """ Add multiple devices :param dev_type: the type of the device to be added :param dev_index: the maximum index of the device to be added :param dev_model: the model of the device to be added """ for inx in range(0, int(dev_index) + 1): newcontroller = Controller("controller") newcontroller.type = dev_type newcontroller.index = inx newcontroller.model = dev_model logging.debug("New device is added:\n%s", newcontroller) vm_xml.add_device(newcontroller) vm_xml.sync() def do_migration(vm, dest_uri, options, extra): """ Execute the migration with given parameters :param vm: the guest to be migrated :param dest_uri: the destination uri for migration :param options: options next to 'migrate' command :param extra: options in the end of the migrate command line :return: CmdResult object """ logging.info("Sleeping 10 seconds before migration") time.sleep(10) # Migrate the guest. migration_res = vm.migrate(dest_uri, options, extra, **virsh_args) logging.info("Migration out: %s", results_stdout_52lts(migration_res).strip()) logging.info("Migration error: %s", results_stderr_52lts(migration_res).strip()) if int(migration_res.exit_status) != 0: logging.error("Migration failed for %s.", vm_name) return migration_res if vm.is_alive(): # vm.connect_uri was updated logging.info("VM is alive on destination %s.", dest_uri) else: test.fail("VM is not alive on destination %s" % dest_uri) # Throws exception if console shows panic message vm.verify_kernel_crash() return migration_res def cleanup_libvirtd_log(log_file): """ Remove existing libvirtd log file on source and target host. :param log_file: log file with absolute path """ if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) cmd = "rm -f %s" % log_file logging.debug("Delete remote libvirt log file '%s'", log_file) run_remote_cmd(cmd) def cleanup_dest(vm): """ Clean up the destination host environment when doing the uni-direction migration. :param vm: the guest to be cleaned up """ logging.info("Cleaning up VMs on %s", vm.connect_uri) try: if virsh.domain_exists(vm.name, uri=vm.connect_uri): vm_state = vm.state() if vm_state == "paused": vm.resume() elif vm_state == "shut off": vm.start() vm.destroy(gracefully=False) if vm.is_persistent(): vm.undefine() except Exception as detail: logging.error("Cleaning up destination failed.\n%s", detail) def run_remote_cmd(cmd): """ A function to run a command on remote host. :param cmd: the command to be executed :return: CmdResult object """ remote_runner = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) cmdResult = remote_runner.run(cmd, ignore_status=True) if cmdResult.exit_status: test.fail("Failed to run '%s' on remote: %s" % (cmd, results_stderr_52lts(cmdResult).strip())) return cmdResult def run_stress_in_vm(): """ The function to load stress in VM """ stress_args = params.get( "stress_args", "--cpu 8 --io 4 " "--vm 2 --vm-bytes 128M " "--timeout 20s") try: vm_session.cmd('stress %s' % stress_args) except Exception as detail: logging.debug(detail) def check_timeout_postcopy(params): """ Check the vm state on target host after timeout when --postcopy and --timeout-postcopy are used. The vm state is expected as running. :param params: the parameters used """ timeout = int(params.get("timeout_postcopy", 10)) time.sleep(timeout + 1) remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) vm_state = results_stdout_52lts( remote_virsh_session.domstate(vm_name)).strip() if vm_state != "running": remote_virsh_session.close_session() test.fail( "After timeout '%s' seconds, " "the vm state on target host should " "be 'running', but '%s' found", timeout, vm_state) remote_virsh_session.close_session() def get_usable_compress_cache(pagesize): """ Get a number which is bigger than pagesize and is power of two. :param pagesize: the given integer :return: an integer satisfying the criteria """ def calculate(num): result = num & (num - 1) return (result == 0) item = pagesize found = False while (not found): item += 1 found = calculate(item) logging.debug( "%d is smallest one that is bigger than '%s' and " "is power of 2", item, pagesize) return item check_parameters(test, params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"ignore_status": True, "debug": True} server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") log_file = params.get("libvirt_log", "/var/log/libvirt/libvirtd.log") check_complete_job = "yes" == params.get("check_complete_job", "no") config_libvirtd = "yes" == params.get("config_libvirtd", "no") contrl_index = params.get("new_contrl_index", None) grep_str_remote_log = params.get("grep_str_remote_log", "") grep_str_local_log = params.get("grep_str_local_log", "") stress_in_vm = "yes" == params.get("stress_in_vm", "no") remote_virsh_dargs = { 'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True } hpt_resize = params.get("hpt_resize", None) htm_state = params.get("htm_state", None) qemu_check = params.get("qemu_check", None) xml_check_after_mig = params.get("guest_xml_check_after_mig", None) arch = platform.machine() if any([hpt_resize, contrl_index, htm_state]) and 'ppc64' not in arch: test.cancel("The case is PPC only.") # For TLS tls_recovery = params.get("tls_auto_recovery", "yes") # qemu config qemu_conf_dict = None # libvirtd config libvirtd_conf_dict = None remote_virsh_session = None vm = None vm_session = None libvirtd_conf = None qemu_conf = None mig_result = None test_exception = None is_TestError = False is_TestFail = False is_TestSkip = False asynch_migration = False # Objects to be cleaned up in the end objs_list = [] tls_obj = None # Local variables vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() if not orig_config_xml: test.error("Backing up xmlfile failed.") try: # Change VM xml in below part if contrl_index: new_xml.remove_all_device_by_type('controller') logging.debug("After removing controllers, current XML:\n%s\n", new_xml) add_ctrls(new_xml, dev_index=contrl_index) if extra.count("--tls"): qemu_conf_dict = {"migrate_tls_x509_verify": "1"} # Setup TLS tls_obj = TLSConnection(params) if tls_recovery == "yes": objs_list.append(tls_obj) tls_obj.auto_recover = True tls_obj.conn_setup() # Setup qemu configure logging.debug("Configure the qemu") cleanup_libvirtd_log(log_file) qemu_conf = libvirt.customize_libvirt_config(qemu_conf_dict, config_type="qemu", remote_host=True, extra_params=params) # Setup libvirtd if config_libvirtd: logging.debug("Configure the libvirtd") cleanup_libvirtd_log(log_file) libvirtd_conf_dict = setup_libvirtd_conf_dict(params) libvirtd_conf = libvirt.customize_libvirt_config( libvirtd_conf_dict, remote_host=True, extra_params=params) if hpt_resize: set_feature(new_xml, 'hpt', hpt_resize) if htm_state: set_feature(new_xml, 'htm', htm_state) # Change the disk of the vm to shared disk and then start VM libvirt.set_vm_disk(vm, params) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) if qemu_check: check_content = qemu_check if hpt_resize: check_content = "%s%s" % (qemu_check, hpt_resize) if htm_state: check_content = "%s%s" % (qemu_check, htm_state) check_qemu_cmd_line(check_content) vm_session = vm.wait_for_login() check_vm_network_accessed() if hpt_resize and hpt_resize != 'disabled': trigger_hpt_resize(vm_session) if stress_in_vm: pkg_name = 'stress' logging.debug("Check if stress tool is installed") pkg_mgr = utils_package.package_manager(vm_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("Stress tool will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) stress_thread = threading.Thread(target=run_stress_in_vm, args=()) stress_thread.start() if extra.count("timeout-postcopy"): asynch_migration = True func_name = check_timeout_postcopy if extra.count("comp-xbzrle-cache"): cache = get_usable_compress_cache(memory.get_page_size()) extra = "%s %s" % (extra, cache) # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options and not extra.count(postcopy_options): extra = "%s %s" % (extra, postcopy_options) if not asynch_migration: mig_result = do_migration(vm, dest_uri, options, extra) else: migration_test = libvirt.MigrationTest() logging.debug("vm.connect_uri=%s", vm.connect_uri) vms = [vm] try: migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, func=func_name, extra_opts=extra, func_params=params) mig_result = migration_test.ret except exceptions.TestFail as fail_detail: test.fail(fail_detail) except exceptions.TestSkipError as skip_detail: test.cancel(skip_detail) except exceptions.TestError as error_detail: test.error(error_detail) except Exception as details: mig_result = migration_test.ret logging.error(details) if int(mig_result.exit_status) != 0: test.fail(results_stderr_52lts(mig_result).strip()) if check_complete_job: search_str_domjobinfo = params.get("search_str_domjobinfo", None) opts = "--completed" args = vm_name + " " + opts check_virsh_command_and_option("domjobinfo", opts) jobinfo = results_stdout_52lts( virsh.domjobinfo(args, debug=True, ignore_status=True)).strip() logging.debug("Local job info on completion:\n%s", jobinfo) if extra.count("comp-xbzrle-cache") and search_str_domjobinfo: search_str_domjobinfo = "%s %s" % (search_str_domjobinfo, cache // 1024) if search_str_domjobinfo: if not re.search(search_str_domjobinfo, jobinfo): test.fail("Fail to search '%s' on local:\n%s" % (search_str_domjobinfo, jobinfo)) # Check remote host if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent( **remote_virsh_dargs) jobinfo = results_stdout_52lts( remote_virsh_session.domjobinfo(args, debug=True, ignore_status=True)).strip() logging.debug("Remote job info on completion:\n%s", jobinfo) if search_str_domjobinfo: if not re.search(search_str_domjobinfo, jobinfo): remote_virsh_session.close_session() test.fail("Fail to search '%s' on remote:\n%s" % (search_str_domjobinfo, jobinfo)) remote_virsh_session.close_session() if grep_str_local_log: cmd = "grep -E '%s' %s" % (grep_str_local_log, log_file) cmdRes = process.run(cmd, shell=True, ignore_status=True) if cmdRes.exit_status: test.fail(results_stderr_52lts(cmdRes).strip()) if grep_str_remote_log: cmd = "grep -E '%s' %s" % (grep_str_remote_log, log_file) run_remote_cmd(cmd) if xml_check_after_mig: if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent( **remote_virsh_dargs) target_guest_dumpxml = results_stdout_52lts( remote_virsh_session.dumpxml(vm_name, debug=True, ignore_status=True)).strip() if hpt_resize: check_str = hpt_resize elif htm_state: check_str = htm_state if hpt_resize or htm_state: xml_check_after_mig = "%s'%s'" % (xml_check_after_mig, check_str) if not re.search(xml_check_after_mig, target_guest_dumpxml): remote_virsh_session.close_session() test.fail("Fail to search '%s' in target guest XML:\n%s" % (xml_check_after_mig, target_guest_dumpxml)) if contrl_index: all_ctrls = re.findall(xml_check_after_mig, target_guest_dumpxml) if len(all_ctrls) != int(contrl_index) + 1: remote_virsh_session.close_session() test.fail( "%s pci-root controllers are expected in guest XML, " "but found %s" % (int(contrl_index) + 1, len(all_ctrls))) remote_virsh_session.close_session() server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") check_vm_network_accessed(server_session) server_session.close() except exceptions.TestFail as details: is_TestFail = True test_exception = details except exceptions.TestSkipError as details: is_TestSkip = True test_exception = details except exceptions.TestError as details: is_TestError = True test_exception = details except Exception as details: test_exception = details finally: logging.debug("Recover test environment") try: # Clean VM on destination vm.connect_uri = dest_uri cleanup_dest(vm) vm.connect_uri = src_uri logging.info("Recovery VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) if remote_virsh_session: remote_virsh_session.close_session() if extra.count("--tls"): logging.debug("Recover the qemu configuration") libvirt.customize_libvirt_config(None, config_type="qemu", remote_host=True, extra_params=params, is_recover=True, config_object=qemu_conf) if config_libvirtd: logging.debug("Recover the libvirtd configuration") libvirt.customize_libvirt_config(None, remote_host=True, extra_params=params, is_recover=True, config_object=libvirtd_conf) logging.info("Remove local NFS image") source_file = params.get("source_file") libvirt.delete_local_disk("file", path=source_file) if objs_list: for obj in objs_list: logging.debug("Clean up local objs") del obj except Exception as exception_detail: if (not test_exception and not is_TestError and not is_TestFail and not is_TestSkip): raise exception_detail else: # if any of above exceptions has been raised, only print # error log here to avoid of hiding the original issue logging.error(exception_detail) # Check result if is_TestFail: test.fail(test_exception) if is_TestSkip: test.cancel(test_exception) if is_TestError: test.error(test_exception) if not test_exception: logging.info("Case execution is done.") else: test.error(test_exception)
def run(test, params, env): """ 1. prepare env 2. check if nwfilter binding 3. run test 4. destroy vm and restore the status """ vm_name = params.get("main_vm") vm = env.get_vm(vm_name) status_error = "yes" == params.get("status_error") vmxml_backup = libvirt_xml.vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) filter_name = params.get("filter_name") is_nwfilter_define = "yes" == params.get("is_nwfilter_define") vnet0_binding = os.path.join(data_dir.get_tmp_dir(), "vnet0_binding.xml") filter_binding_name = params.get("filter_binding_name") failed_msg = params.get("expected_failed") target_dev = params.get("target_dev") source_network = params.get("source_network") source_bridge = params.get("source_bridge") alias_name = params.get("alias_name") def set_env(): """ prepare the vm interface xml this xml can both use in two scenario. but little different for two scenario """ vmxml = libvirt_xml.VMXML.new_from_inactive_dumpxml(vm_name) iface_xml = vmxml.get_devices('interface')[0] vmxml.del_device(iface_xml) new_iface = interface.Interface('network') new_iface.xml = iface_xml.xml new_iface.type_name = "network" iface_target = {'dev': target_dev} new_iface.target = iface_target source = {'network': source_network, 'bridge': source_bridge} new_iface.source = source filterrefs_dict = {} filterrefs_dict['name'] = filter_name filterrefs_dict['parameters'] = [] new_filterref = new_iface.new_filterref(**filterrefs_dict) new_iface.filterref = new_filterref alias_dict = {'name': alias_name} new_iface.alias = alias_dict vmxml.add_device(new_iface) logging.debug("new interface xml is: %s" % new_iface) vmxml.sync() return new_iface def check_binding_port(cmd_res, match, is_match=True): """ check the list binding ports """ list_res = cmd_res.stdout_text.strip() if list_res and re.search(match, list_res): if not is_match: test.fail("expected not match %s" % match) elif is_match: test.fail("expected match %s but not match" % match) try: # set new interface env new_iface = set_env() # create binding dump file pkg_mgr = utils_package.package_manager(None, 'libvirt-daemon-config-nwfilter') pkg_mgr.install() virsh.start(vm_name, debug=True) ret = virsh.nwfilter_binding_dumpxml(new_iface.target['dev'], to_file=vnet0_binding, debug=True) utlv.check_exit_status(ret, status_error) binding = nwfilter_binding.NwfilterBinding() binding.xml = vnet0_binding filterrefs_dict = {} filterrefs_dict['name'] = filter_binding_name filterrefs_dict['parameters'] = [ {'name': "MAC", 'value': new_iface.mac_address}] binding.filterref = binding.new_filterref(**filterrefs_dict) logging.debug("binding is %s" % binding) # list filter if not is_nwfilter_define: virsh.nwfilter_binding_delete(new_iface.target['dev'], debug=True) if is_nwfilter_define: ret = virsh.nwfilter_binding_list(debug=True) utlv.check_exit_status(ret, status_error) check_binding_port(ret, filter_name, is_match=True) ret_create = virsh.nwfilter_binding_create(binding.xml, debug=True) # two Scenario if is_nwfilter_define: utlv.check_result(ret_create, failed_msg) elif not is_nwfilter_define: # get params for senario2 check_cmd = params.get("check_cmd") expected_match = params.get("expected_match") filter_binding_copy = params.get("filter_binding_copy") ret = virsh.nwfilter_binding_list(debug=True) check_binding_port(ret, filter_binding_name, is_match=True) utlv.check_cmd_output(check_cmd, expected_match, True) utils_libvirtd.libvirtd_restart() ret = virsh.nwfilter_binding_list(debug=True) check_binding_port(ret, filter_binding_name, is_match=True) # use check command to check result utlv.check_cmd_output(check_cmd, expected_match, True) new_binding = nwfilter_binding.NwfilterBinding() new_binding.xml = binding.xml filterrefs_dict = {} filterrefs_dict['name'] = filter_binding_copy filterrefs_dict['parameters'] = [ {'name': "MAC", 'value': new_iface.mac_address}] binding.filterref = binding.new_filterref(**filterrefs_dict) logging.debug("binding is %s" % new_binding) ret_create = virsh.nwfilter_binding_create(new_binding.xml, debug=True) utlv.check_result(ret_create, failed_msg) finally: if vm.is_alive(): vm.destroy(gracefully=False) vmxml_backup.sync()
def run(test, params, env): """ Test virsh migrate command. """ def cleanup_vm(vm, vm_name='', uri=''): """ Clean up vm in the src or destination host environment when doing the uni-direction migration. """ # Backup vm name and uri uri_bak = vm.connect_uri vm_name_bak = vm.name # Destroy and undefine vm vm.connect_uri = uri if uri else uri_bak vm.name = vm_name if vm_name else vm_name_bak logging.info("Cleaning up VM %s on %s", vm.name, vm.connect_uri) if vm.is_alive(): vm.destroy() if vm.is_persistent(): vm.undefine() # Restore vm connect_uri vm.connect_uri = uri_bak vm.name = vm_name_bak # Check whether there are unset parameters for v in list(itervalues(params)): if isinstance(v, string_types) and v.count("EXAMPLE"): test.cancel("Please set real value for %s" % v) # Params for virsh migrate options: live_migration = params.get("live_migration") == "yes" offline_migration = params.get("offline_migration") == "yes" persistent = params.get("persistent") == "yes" undefinesource = params.get("undefinesource") == "yes" p2p = params.get("p2p") == "yes" tunnelled = params.get("tunnelled") == "yes" postcopy = params.get("postcopy") == "yes" dname = params.get("dname") xml_option = params.get("xml_option") == "yes" persistent_xml_option = params.get("persistent_xml_option") == "yes" extra_options = params.get("virsh_migrate_extra", "") if live_migration and not extra_options.count("--live"): extra_options = "%s --live" % extra_options if offline_migration and not extra_options.count("--offline"): extra_options = "%s --offline" % extra_options if persistent and not extra_options.count("--persistent"): extra_options = "%s --persistent" % extra_options if undefinesource and not extra_options.count("--undefinesource"): extra_options = "%s --undefinesource" % extra_options if p2p and not extra_options.count("--p2p"): extra_options = "%s --p2p" % extra_options if tunnelled and not extra_options.count("--tunnelled"): extra_options = "%s --tunnelled" % extra_options if tunnelled and not extra_options.count("--p2p"): extra_options = "%s --p2p" % extra_options if postcopy and not extra_options.count("--postcopy"): extra_options = "%s --postcopy" % extra_options if dname and not extra_options.count("--dname"): extra_options = "%s --dname %s" % (extra_options, dname) if xml_option: pass if persistent_xml_option and not extra_options.count("--persistent"): extra_options = "%s --persistent" % extra_options if persistent_xml_option: pass # Set param migrate_options in case it is used somewhere: params.setdefault("migrate_options", extra_options) # Params for postcopy migration postcopy_timeout = int(params.get("postcopy_migration_timeout", "180")) # Params for migrate hosts: server_cn = params.get("server_cn") client_cn = params.get("client_cn") migrate_source_host = client_cn if client_cn else params.get( "migrate_source_host") migrate_dest_host = server_cn if server_cn else params.get( "migrate_dest_host") # Params for migrate uri transport = params.get("transport", "tls") transport_port = params.get("transport_port") uri_port = ":%s" % transport_port if transport_port else '' hypervisor_driver = params.get("hypervisor_driver", "qemu") hypervisor_mode = params.get("hypervisor_mode", 'system') if "virsh_migrate_desturi" not in list(params.keys()): params["virsh_migrate_desturi"] = "%s+%s://%s%s/%s" % ( hypervisor_driver, transport, migrate_dest_host, uri_port, hypervisor_mode) if "virsh_migrate_srcuri" not in list(params.keys()): params["virsh_migrate_srcuri"] = "%s:///%s" % (hypervisor_driver, hypervisor_mode) dest_uri = params.get("virsh_migrate_desturi") src_uri = params.get("virsh_migrate_srcuri") # Params for src vm cfg: src_vm_cfg = params.get("src_vm_cfg") src_vm_status = params.get("src_vm_status") with_graphic_passwd = params.get("with_graphic_passwd") graphic_passwd = params.get("graphic_passwd") # For test result check cancel_exception = False fail_exception = False exception = False result_check_pass = True # Objects(SSH, TLS and TCP, etc) to be cleaned up in finally objs_list = [] # VM objects for migration test vms = [] try: # Get a MigrationTest() Object logging.debug("Get a MigrationTest() object") obj_migration = migration.MigrationTest() # Setup libvirtd remote connection TLS connection env if transport == "tls": tls_obj = TLSConnection(params) # Setup CA, server(on dest host) and client(on src host) tls_obj.conn_setup() # Add tls_obj to objs_list objs_list.append(tls_obj) # Enable libvirtd remote connection transport port if transport == 'tls': transport_port = '16514' elif transport == 'tcp': transport_port = '16509' obj_migration.migrate_pre_setup(dest_uri, params, ports=transport_port) # Back up vm name for recovery in finally vm_name_backup = params.get("migrate_main_vm") # Get a vm object for migration logging.debug("Get a vm object for migration") vm = env.get_vm(vm_name_backup) # Back up vm xml for recovery in finally logging.debug("Backup vm xml before migration") vm_xml_backup = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) if not vm_xml_backup: test.error("Backing up xmlfile failed.") # Prepare shared disk in vm xml for live migration: # Change the source of the first disk of vm to shared disk if live_migration: logging.debug("Prepare shared disk in vm xml for live migration") storage_type = params.get("storage_type") if storage_type == 'nfs': logging.debug("Prepare nfs shared disk in vm xml") nfs_mount_dir = params.get("nfs_mount_dir") libvirt.update_vm_disk_source(vm.name, nfs_mount_dir) libvirt.update_vm_disk_driver_cache(vm.name, driver_cache="none") else: # TODO:Other storage types test.cancel("Other storage type is not supported for now") pass # Prepare graphic password in vm xml if with_graphic_passwd in ["yes", "no"]: logging.debug("Set VM graphic passwd in vm xml") # Get graphics list in vm xml vmxml_tmp = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) graphics_list = vmxml_tmp.get_graphics_devices if not graphics_list: # Add spice graphic with passwd to vm xml logging.debug("Add spice graphic to vm xml") graphics.Graphics.add_graphic(vm.name, graphic_passwd, "spice") elif graphic_passwd: # Graphics already exist in vm xml and passwd is required # Add passwd to the first graphic device in vm xml logging.debug("Add graphic passwd to vm xml") vm_xml.VMXML.add_security_info(vmxml_tmp, graphic_passwd) vmxml_tmp.sync() else: # Graphics already exist in vm xml and non-passwd is required # Do nothing here as passwd has been removed by new_from_inactive_dumpxml() pass # Prepare for required src vm status. logging.debug("Turning %s into certain state.", vm.name) if src_vm_status == "running" and not vm.is_alive(): vm.start() elif src_vm_status == "shut off" and not vm.is_dead(): vm.destroy() # Prepare for required src vm persistency. logging.debug("Prepare for required src vm persistency") if src_vm_cfg == "persistent" and not vm.is_persistent(): logging.debug("Make src vm persistent") vm_xml_backup.define() elif src_vm_cfg == "transient" and vm.is_persistent(): logging.debug("Make src vm transient") vm.undefine() # Prepare for postcopy migration: install and run stress in VM if postcopy and src_vm_status == "running": logging.debug( "Install and run stress in vm for postcopy migration") pkg_name = 'stress' # Get a vm session logging.debug("Get a vm session") vm_session = vm.wait_for_login() if not vm_session: test.error("Can't get a vm session successfully") # Install package stress if it is not installed in vm logging.debug( "Check if stress tool is installed for postcopy migration") pkg_mgr = utils_package.package_manager(vm_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("Stress tool will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) # Run stress in vm logging.debug("Run stress in vm") stress_args = params.get("stress_args") vm_session.cmd('stress %s' % stress_args) # Prepare for --xml <updated_xml_file>. if xml_option: logging.debug("Preparing new xml file for --xml option.") # Get the vm xml vmxml_tmp = vm_xml.VMXML.new_from_dumpxml( vm.name, "--security-info --migratable") # Update something in the xml file: e.g. title # Note: VM ABI shall not be broken when migrating with updated_xml updated_title = "VM Title in updated xml" vmxml_tmp.title = updated_title # Add --xml to migrate extra_options extra_options = ("%s --xml=%s" % (extra_options, vmxml_tmp.xml)) # Prepare for --persistent-xml <updated_xml_file>. if persistent_xml_option: logging.debug( "Preparing new xml file for --persistent-xml option.") # Get the vm xml vmxml_persist_tmp = vm_xml.VMXML.new_from_inactive_dumpxml( vm.name, "--security-info") # Update something in the xml file: e.g. title # Note: VM ABI shall not be broken when migrating with updated_xml updated_persist_title = "VM Title in updated persist xml" vmxml_persist_tmp.title = updated_persist_title # Add --persistent-xml to migrate extra_options extra_options = ("%s --persistent-xml=%s" % (extra_options, vmxml_persist_tmp.xml)) # Prepare host env: clean up vm on dest host logging.debug("Clean up vm on dest host before migration") if dname: cleanup_vm(vm, dname, dest_uri) else: cleanup_vm(vm, vm.name, dest_uri) # Prepare host env: set selinux state before migration logging.debug("Set selinux to enforcing before migration") utils_selinux.set_status(params.get("selinux_state", "enforcing")) # Check vm network connectivity by ping before migration logging.debug("Check vm network before migration") if src_vm_status == "running": obj_migration.ping_vm(vm, params) # Get VM uptime before migration if src_vm_status == "running": vm_uptime = vm.uptime() logging.info("Check VM uptime before migration: %s", vm_uptime) # Print vm active xml before migration process.system_output("virsh dumpxml %s --security-info" % vm.name, shell=True) # Print vm inactive xml before migration process.system_output("virsh dumpxml %s --security-info --inactive" % vm.name, shell=True) # Do uni-direction migration. # NOTE: vm.connect_uri will be set to dest_uri once migration is complete successfully logging.debug("Start to do migration test.") vms.append(vm) if postcopy: # Monitor the qemu monitor event of "postcopy-active" for postcopy migration logging.debug( "Monitor the qemu monitor event for postcopy migration") virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True) cmd = "qemu-monitor-event --loop --domain %s --event MIGRATION" % vm.name virsh_session.sendline(cmd) # Do live migration and switch to postcopy by "virsh migrate-postcopy" logging.debug("Start to do postcopy migration") obj_migration.do_migration(vms, src_uri, dest_uri, "orderly", options="", thread_timeout=postcopy_timeout, ignore_status=True, func=virsh.migrate_postcopy, extra_opts=extra_options, shell=True) # Check migration result obj_migration.check_result(obj_migration.ret, params) # Check "postcopy-active" event after postcopy migration logging.debug( "Check postcopy-active event after postcopy migration") virsh_session.send_ctrl("^C") events_output = virsh_session.get_stripped_output() logging.debug("events_output are %s", events_output) pattern = "postcopy-active" if not re.search(pattern, events_output): test.fail("Migration didn't switch to postcopy mode") virsh_session.close() virsh_session.close() else: logging.debug("Start to do precopy migration") obj_migration.do_migration(vms, src_uri, dest_uri, "orderly", options="", ignore_status=True, extra_opts=extra_options) # Check migration result obj_migration.check_result(obj_migration.ret, params) """ # Check src vm after migration # First, update vm name and connect_uri to src vm's """ vm.name = vm_name_backup vm.connect_uri = src_uri logging.debug("Start to check %s state on src %s after migration.", vm.name, src_uri) # Check src vm status after migration: existence, running, shutoff, etc logging.debug("Check vm status on source after migration") if offline_migration: if src_vm_status == "shut off" and undefinesource: if vm.exists(): result_check_pass = False logging.error( "Src vm should not exist after offline migration" " with --undefinesource") logging.debug("Src vm state is %s" % vm.state()) elif not libvirt.check_vm_state( vm.name, src_vm_status, uri=vm.connect_uri): result_check_pass = False logging.error("Src vm should be %s after offline migration" % src_vm_status) logging.debug("Src vm state is %s" % vm.state()) if live_migration: if not undefinesource and src_vm_cfg == "persistent": if not libvirt.check_vm_state( vm.name, "shut off", uri=vm.connect_uri): result_check_pass = False logging.error( "Src vm should be shutoff after live migration") logging.debug("Src vm state is %s" % vm.state()) elif vm.exists(): result_check_pass = False logging.error("Src vm should not exist after live migration") logging.debug("Src vm state is %s" % vm.state()) # Check src vm status after migration: persistency logging.debug("Check vm persistency on source after migration") if src_vm_cfg == "persistent" and not undefinesource: if not vm.is_persistent(): # Src vm should be persistent after migration without --undefinesource result_check_pass = False logging.error("Src vm should be persistent after migration") elif vm.is_persistent(): result_check_pass = False logging.error("Src vm should be not be persistent after migration") """ # Check dst vm after migration # First, update vm name and connect_uri to dst vm's """ vm.name = dname if dname else vm.name vm.connect_uri = dest_uri logging.debug("Start to check %s state on target %s after migration.", vm.name, vm.connect_uri) # Check dst vm status after migration: running, shutoff, etc logging.debug("Check vm status on target after migration") if live_migration: if not libvirt.check_vm_state( vm.name, src_vm_status, uri=vm.connect_uri): result_check_pass = False logging.error("Dst vm should be %s after live migration", src_vm_status) elif vm.is_alive(): result_check_pass = False logging.error("Dst vm should not be alive after offline migration") # Print vm active xml after migration process.system_output("virsh -c %s dumpxml %s --security-info" % (vm.connect_uri, vm.name), shell=True) # Print vm inactive xml after migration process.system_output( "virsh -c %s dumpxml %s --security-info --inactive" % (vm.connect_uri, vm.name), shell=True) # Check dst vm xml after migration logging.debug("Check vm xml on target after migration") remote_virsh = virsh.Virsh(uri=vm.connect_uri) vmxml_active_tmp = vm_xml.VMXML.new_from_dumpxml( vm.name, "--security-info", remote_virsh) vmxml_inactive_tmp = vm_xml.VMXML.new_from_inactive_dumpxml( vm.name, "--security-info", remote_virsh) # Check dst vm xml after migration: --xml <updated_xml_file> if xml_option and not offline_migration: logging.debug("Check vm active xml for --xml") if not vmxml_active_tmp.title == updated_title: print("vmxml active tmp title is %s" % vmxml_active_tmp.title) result_check_pass = False logging.error("--xml doesn't take effect in migration") if xml_option and offline_migration: logging.debug("Check vm inactive xml for --xml") if not vmxml_active_tmp.title == updated_title: result_check_pass = False logging.error("--xml doesn't take effect in migration") # Check dst vm xml after migration: --persistent-xml <updated_xml_file> if persistent_xml_option: logging.debug("Check vm inactive xml for --persistent-xml") if not offline_migration and not vmxml_inactive_tmp.title == updated_persist_title: print("vmxml inactive tmp title is %s" % vmxml_inactive_tmp.title) result_check_pass = False logging.error( "--persistent-xml doesn't take effect in live migration") elif offline_migration and vmxml_inactive_tmp.title == updated_persist_title: result_check_pass = False logging.error( "--persistent-xml should not take effect in offline " "migration") # Check dst vm xml after migration: graphic passwd if with_graphic_passwd == "yes": logging.debug("Check graphic passwd in vm xml after migration") graphic_active = vmxml_active_tmp.devices.by_device_tag( 'graphics')[0] graphic_inactive = vmxml_inactive_tmp.devices.by_device_tag( 'graphics')[0] try: logging.debug("Check graphic passwd in active vm xml") if graphic_active.passwd != graphic_passwd: result_check_pass = False logging.error( "Graphic passwd in active xml of dst vm should be %s", graphic_passwd) logging.debug("Check graphic passwd in inactive vm xml") if graphic_inactive.passwd != graphic_passwd: result_check_pass = False logging.error( "Graphic passwd in inactive xml of dst vm should be %s", graphic_passwd) except LibvirtXMLNotFoundError: result_check_pass = False logging.error("Graphic passwd lost in dst vm xml") # Check dst vm uptime, network, etc after live migration if live_migration: # Check dst VM uptime after migration # Note: migrated_vm_uptime should be greater than the vm_uptime got # before migration migrated_vm_uptime = vm.uptime(connect_uri=dest_uri) logging.info( "Check VM uptime in destination after " "migration: %s", migrated_vm_uptime) if not migrated_vm_uptime: result_check_pass = False logging.error("Failed to check vm uptime after migration") elif vm_uptime > migrated_vm_uptime: result_check_pass = False logging.error( "VM went for a reboot while migrating to destination") # Check dst VM network connectivity after migration logging.debug("Check VM network connectivity after migrating") obj_migration.ping_vm(vm, params, uri=dest_uri) # Restore vm.connect_uri as it is set to src_uri in ping_vm() logging.debug( "Restore vm.connect_uri as it is set to src_uri in ping_vm()") vm.connect_uri = dest_uri # Check dst vm status after migration: persistency logging.debug("Check vm persistency on target after migration") if persistent: if not vm.is_persistent(): result_check_pass = False logging.error("Dst vm should be persistent after migration " "with --persistent") time.sleep(10) # Destroy vm and check vm state should be shutoff. BZ#1076354 vm.destroy() if not libvirt.check_vm_state( vm.name, "shut off", uri=vm.connect_uri): result_check_pass = False logging.error( "Dst vm with name %s should exist and be shutoff", vm.name) elif vm.is_persistent(): result_check_pass = False logging.error("Dst vm should not be persistent after migration " "without --persistent") finally: logging.debug("Start to clean up env") # Clean up vm on dest and src host for vm in vms: cleanup_vm(vm, vm_name=dname, uri=dest_uri) cleanup_vm(vm, vm_name=vm_name_backup, uri=src_uri) # Recover source vm defination (just in case). logging.info("Recover vm defination on source") if vm_xml_backup: vm_xml_backup.define() # Clean up SSH, TCP, TLS test env if objs_list and len(objs_list) > 0: logging.debug("Clean up test env: SSH, TCP, TLS, etc") for obj in objs_list: obj.auto_recover = True obj.__del__() # Disable libvirtd remote connection transport port obj_migration.migrate_pre_setup(dest_uri, params, cleanup=True, ports=transport_port) # Check test result. if not result_check_pass: test.fail("Migration succeed, but some check points didn't pass." "Please check the error log for details")
def run(test, params, env): """ Test virsh migrate command. """ def set_feature(vmxml, feature, value): """ Set guest features for PPC :param state: the htm status :param vmxml: guest xml """ features_xml = vm_xml.VMFeaturesXML() if feature == 'hpt': features_xml.hpt_resizing = value elif feature == 'htm': features_xml.htm = value vmxml.features = features_xml vmxml.sync() def trigger_hpt_resize(session): """ Check the HPT order file and dmesg :param session: the session to guest :raise: test.fail if required message is not found """ hpt_order_path = "/sys/kernel/debug/powerpc/hpt_order" hpt_order = session.cmd_output('cat %s' % hpt_order_path).strip() hpt_order = int(hpt_order) logging.info('Current hpt_order is %d', hpt_order) hpt_order += 1 cmd = 'echo %d > %s' % (hpt_order, hpt_order_path) cmd_result = session.cmd_status_output(cmd) result = process.CmdResult(stderr=cmd_result[1], stdout=cmd_result[1], exit_status=cmd_result[0]) libvirt.check_exit_status(result) dmesg = session.cmd('dmesg') dmesg_content = params.get('dmesg_content').split('|') for content in dmesg_content: if content % hpt_order not in dmesg: test.fail("'%s' is missing in dmesg" % (content % hpt_order)) else: logging.info("'%s' is found in dmesg", content % hpt_order) def check_vm_network_accessed(session=None): """ The operations to the VM need to be done before or after migration happens :param session: The session object to the host :raise: test.error when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") s_ping, _ = utils_test.ping(vm.get_address(), count=10, timeout=20, output_func=logging.debug, session=session) if s_ping != 0: if session: session.close() test.fail("%s did not respond after %d sec." % (vm.name, 20)) def check_virsh_command_and_option(command, option=None): """ Check if virsh command exists :param command: the command to be checked :param option: the command option to be checked """ msg = "This version of libvirt does not support " if not virsh.has_help_command(command): test.cancel(msg + "virsh command '%s'" % command) if option and not virsh.has_command_help_match(command, option): test.cancel(msg + "virsh command '%s' with option '%s'" % (command, option)) def add_ctrls(vm_xml, dev_type="pci", dev_index="0", dev_model="pci-root"): """ Add multiple devices :param dev_type: the type of the device to be added :param dev_index: the maximum index of the device to be added :param dev_model: the model of the device to be added """ for inx in range(0, int(dev_index) + 1): newcontroller = Controller("controller") newcontroller.type = dev_type newcontroller.index = inx newcontroller.model = dev_model logging.debug("New device is added:\n%s", newcontroller) vm_xml.add_device(newcontroller) vm_xml.sync() def do_migration(vm, dest_uri, options, extra): """ Execute the migration with given parameters :param vm: the guest to be migrated :param dest_uri: the destination uri for migration :param options: options next to 'migrate' command :param extra: options in the end of the migrate command line :return: CmdResult object """ logging.info("Sleeping 10 seconds before migration") time.sleep(10) # Migrate the guest. virsh_args.update({"ignore_status": True}) migration_res = vm.migrate(dest_uri, options, extra, **virsh_args) if int(migration_res.exit_status) != 0: logging.error("Migration failed for %s.", vm_name) return migration_res if vm.is_alive(): # vm.connect_uri was updated logging.info("VM is alive on destination %s.", dest_uri) else: test.fail("VM is not alive on destination %s" % dest_uri) # Throws exception if console shows panic message vm.verify_kernel_crash() return migration_res def cleanup_libvirtd_log(log_file): """ Remove existing libvirtd log file on source and target host. :param log_file: log file with absolute path """ if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) cmd = "rm -f %s" % log_file logging.debug("Delete remote libvirt log file '%s'", log_file) cmd_parms = { 'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd } remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) def cleanup_dest(vm): """ Clean up the destination host environment when doing the uni-direction migration. :param vm: the guest to be cleaned up """ logging.info("Cleaning up VMs on %s", vm.connect_uri) try: if virsh.domain_exists(vm.name, uri=vm.connect_uri): vm_state = vm.state() if vm_state == "paused": vm.resume() elif vm_state == "shut off": vm.start() vm.destroy(gracefully=False) if vm.is_persistent(): vm.undefine() except Exception as detail: logging.error("Cleaning up destination failed.\n%s", detail) def run_stress_in_vm(): """ The function to load stress in VM """ stress_args = params.get( "stress_args", "--cpu 8 --io 4 " "--vm 2 --vm-bytes 128M " "--timeout 20s") try: vm_session.cmd('stress %s' % stress_args) except Exception as detail: logging.debug(detail) def control_migrate_speed(to_speed=1): """ Control migration duration :param to_speed: the speed value in Mbps to be set for migration :return int: the new migration speed after setting """ virsh_args.update({"ignore_status": False}) old_speed = virsh.migrate_getspeed(vm_name, **virsh_args) logging.debug("Current migration speed is %s MiB/s\n", old_speed.stdout.strip()) logging.debug("Set migration speed to %d MiB/s\n", to_speed) cmd_result = virsh.migrate_setspeed(vm_name, to_speed, "", **virsh_args) actual_speed = virsh.migrate_getspeed(vm_name, **virsh_args) logging.debug("New migration speed is %s MiB/s\n", actual_speed.stdout.strip()) return int(actual_speed.stdout.strip()) def check_setspeed(params): """ Set/get migration speed :param params: the parameters used :raise: test.fail if speed set does not take effect """ expected_value = int(params.get("migrate_speed", '41943040')) // (1024 * 1024) actual_value = control_migrate_speed(to_speed=expected_value) params.update({'compare_to_value': actual_value}) if actual_value != expected_value: test.fail( "Migration speed is expected to be '%d MiB/s', but '%d MiB/s' " "found" % (expected_value, actual_value)) def check_domjobinfo(params, option=""): """ Check given item in domjobinfo of the guest is as expected :param params: the parameters used :param option: options for domjobinfo :raise: test.fail if the value of given item is unexpected """ def search_jobinfo(jobinfo): """ Find value of given item in domjobinfo :param jobinfo: cmdResult object :raise: test.fail if not found """ for item in jobinfo.stdout.splitlines(): if item.count(jobinfo_item): groups = re.findall(r'[0-9.]+', item.strip()) logging.debug("In '%s' search '%s'\n", item, groups[0]) if (math.fabs(float(groups[0]) - float(compare_to_value)) // float(compare_to_value) > diff_rate): test.fail("{} {} has too much difference from " "{}".format(jobinfo_item, groups[0], compare_to_value)) break jobinfo_item = params.get("jobinfo_item") compare_to_value = params.get("compare_to_value") logging.debug("compare_to_value:%s", compare_to_value) diff_rate = float(params.get("diff_rate", "0")) if not jobinfo_item or not compare_to_value: return vm_ref = '{}{}'.format(vm_name, option) jobinfo = virsh.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) check_domjobinfo_remote = params.get("check_domjobinfo_remote") if check_domjobinfo_remote: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) jobinfo = remote_virsh_session.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) remote_virsh_session.close_session() def check_maxdowntime(params): """ Set/get migration maxdowntime :param params: the parameters used :raise: test.fail if maxdowntime set does not take effect """ expected_value = int( float(params.get("migrate_maxdowntime", '0.3')) * 1000) virsh_args.update({"ignore_status": False}) old_value = int(virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("Current migration maxdowntime is %d ms", old_value) logging.debug("Set migration maxdowntime to %d ms", expected_value) virsh.migrate_setmaxdowntime(vm_name, expected_value, **virsh_args) actual_value = int( virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("New migration maxdowntime is %d ms", actual_value) if actual_value != expected_value: test.fail( "Migration maxdowntime is expected to be '%d ms', but '%d ms' " "found" % (expected_value, actual_value)) params.update({'compare_to_value': actual_value}) def do_actions_during_migrate(params): """ The entry point to execute action list during migration :param params: the parameters used """ actions_during_migration = params.get("actions_during_migration") if not actions_during_migration: return for action in actions_during_migration.split(","): if action == 'setspeed': check_setspeed(params) elif action == 'domjobinfo': check_domjobinfo(params) elif action == 'setmaxdowntime': check_maxdowntime(params) time.sleep(3) def attach_channel_xml(): """ Create channel xml and attach it to guest configuration """ # Check if pty channel exists already for elem in new_xml.devices.by_device_tag('channel'): if elem.type_name == channel_type_name: logging.debug( "{0} channel already exists in guest. " "No need to add new one".format(channel_type_name)) return params = { 'channel_type_name': channel_type_name, 'target_type': target_type, 'target_name': target_name } channel_xml = libvirt.create_channel_xml(params) virsh.attach_device(domain_opt=vm_name, file_opt=channel_xml.xml, flagstr="--config", ignore_status=False) logging.debug("New VMXML with channel:\n%s", virsh.dumpxml(vm_name)) def check_timeout_postcopy(params): """ Check the vm state on target host after timeout when --postcopy and --timeout-postcopy are used. The vm state is expected as running. :param params: the parameters used """ timeout = int(params.get("timeout_postcopy", 10)) time.sleep(timeout + 1) remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) vm_state = results_stdout_52lts( remote_virsh_session.domstate(vm_name)).strip() if vm_state != "running": remote_virsh_session.close_session() test.fail( "After timeout '%s' seconds, " "the vm state on target host should " "be 'running', but '%s' found", timeout, vm_state) remote_virsh_session.close_session() def get_usable_compress_cache(pagesize): """ Get a number which is bigger than pagesize and is power of two. :param pagesize: the given integer :return: an integer satisfying the criteria """ def calculate(num): result = num & (num - 1) return (result == 0) item = pagesize found = False while (not found): item += 1 found = calculate(item) logging.debug( "%d is smallest one that is bigger than '%s' and " "is power of 2", item, pagesize) return item def check_migration_res(result): """ Check if the migration result is as expected :param result: the output of migration :raise: test.fail if test is failed """ logging.info("Migration out: %s", results_stdout_52lts(result).strip()) logging.info("Migration error: %s", results_stderr_52lts(result).strip()) if status_error: # Migration should fail if err_msg: # Special error messages are expected if not re.search(err_msg, results_stderr_52lts(result).strip()): test.fail("Can not find the expected patterns '%s' in " "output '%s'" % (err_msg, results_stderr_52lts(result).strip())) else: logging.debug("It is the expected error message") else: if int(result.exit_status) != 0: logging.debug("Migration failure is expected result") else: test.fail("Migration success is unexpected result") else: if int(result.exit_status) != 0: test.fail(results_stderr_52lts(result).strip()) check_parameters(test, params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} virsh_opt = params.get("virsh_opt", "") server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") log_file = params.get("libvirt_log", "/var/log/libvirt/libvirtd.log") check_complete_job = "yes" == params.get("check_complete_job", "no") config_libvirtd = "yes" == params.get("config_libvirtd", "no") contrl_index = params.get("new_contrl_index", None) asynch_migration = "yes" == params.get("asynch_migrate", "no") grep_str_remote_log = params.get("grep_str_remote_log", "") grep_str_local_log = params.get("grep_str_local_log", "") disable_verify_peer = "yes" == params.get("disable_verify_peer", "no") status_error = "yes" == params.get("status_error", "no") stress_in_vm = "yes" == params.get("stress_in_vm", "no") low_speed = params.get("low_speed", None) remote_virsh_dargs = { 'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True } hpt_resize = params.get("hpt_resize", None) htm_state = params.get("htm_state", None) # For pty channel test add_channel = "yes" == params.get("add_channel", "no") channel_type_name = params.get("channel_type_name", None) target_type = params.get("target_type", None) target_name = params.get("target_name", None) cmd_run_in_remote_guest = params.get("cmd_run_in_remote_guest", None) cmd_run_in_remote_guest_1 = params.get("cmd_run_in_remote_guest_1", None) cmd_run_in_remote_host = params.get("cmd_run_in_remote_host", None) cmd_run_in_remote_host_1 = params.get("cmd_run_in_remote_host_1", None) cmd_run_in_remote_host_2 = params.get("cmd_run_in_remote_host_2", None) # For qemu command line checking qemu_check = params.get("qemu_check", None) xml_check_after_mig = params.get("guest_xml_check_after_mig", None) # params for cache matrix test cache = params.get("cache") remove_cache = "yes" == params.get("remove_cache", "no") err_msg = params.get("err_msg") arch = platform.machine() if any([hpt_resize, contrl_index, htm_state]) and 'ppc64' not in arch: test.cancel("The case is PPC only.") # For TLS tls_recovery = params.get("tls_auto_recovery", "yes") # qemu config qemu_conf_dict = None # libvirtd config libvirtd_conf_dict = None remote_virsh_session = None vm = None vm_session = None libvirtd_conf = None qemu_conf = None mig_result = None test_exception = None is_TestError = False is_TestFail = False is_TestSkip = False # Objects to be cleaned up in the end objs_list = [] tls_obj = None # Local variables vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() if not orig_config_xml: test.error("Backing up xmlfile failed.") try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Change the configuration files if needed before starting guest # For qemu.conf if extra.count("--tls"): # Setup TLS tls_obj = TLSConnection(params) if tls_recovery == "yes": objs_list.append(tls_obj) tls_obj.auto_recover = True tls_obj.conn_setup() if not disable_verify_peer: qemu_conf_dict = {"migrate_tls_x509_verify": "1"} # Setup qemu configure logging.debug("Configure the qemu") cleanup_libvirtd_log(log_file) qemu_conf = libvirt.customize_libvirt_config( qemu_conf_dict, config_type="qemu", remote_host=True, extra_params=params) # Setup libvirtd if config_libvirtd: logging.debug("Configure the libvirtd") cleanup_libvirtd_log(log_file) libvirtd_conf_dict = setup_libvirtd_conf_dict(params) libvirtd_conf = libvirt.customize_libvirt_config( libvirtd_conf_dict, remote_host=True, extra_params=params) # Prepare required guest xml before starting guest if contrl_index: new_xml.remove_all_device_by_type('controller') logging.debug("After removing controllers, current XML:\n%s\n", new_xml) add_ctrls(new_xml, dev_index=contrl_index) if add_channel: attach_channel_xml() if hpt_resize: set_feature(new_xml, 'hpt', hpt_resize) if htm_state: set_feature(new_xml, 'htm', htm_state) if cache: params["driver_cache"] = cache if remove_cache: params["enable_cache"] = "no" # Change the disk of the vm to shared disk and then start VM libvirt.set_vm_disk(vm, params) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check qemu command line after guest is started if qemu_check: check_content = qemu_check if hpt_resize: check_content = "%s%s" % (qemu_check, hpt_resize) if htm_state: check_content = "%s%s" % (qemu_check, htm_state) libvirt.check_qemu_cmd_line(check_content) # Check local guest network connection before migration vm_session = vm.wait_for_login() check_vm_network_accessed() # Preparation for the running guest before migration if hpt_resize and hpt_resize != 'disabled': trigger_hpt_resize(vm_session) if low_speed: control_migrate_speed(int(low_speed)) if stress_in_vm: pkg_name = 'stress' logging.debug("Check if stress tool is installed") pkg_mgr = utils_package.package_manager(vm_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("Stress tool will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) stress_thread = threading.Thread(target=run_stress_in_vm, args=()) stress_thread.start() if extra.count("timeout-postcopy"): func_name = check_timeout_postcopy if params.get("actions_during_migration"): func_name = do_actions_during_migrate if extra.count("comp-xbzrle-cache"): cache = get_usable_compress_cache(memory.get_page_size()) extra = "%s %s" % (extra, cache) # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (extra, postcopy_options) # Execute migration process if not asynch_migration: mig_result = do_migration(vm, dest_uri, options, extra) else: migration_test = libvirt.MigrationTest() logging.debug("vm.connect_uri=%s", vm.connect_uri) vms = [vm] try: migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_opt, func=func_name, extra_opts=extra, func_params=params) mig_result = migration_test.ret except exceptions.TestFail as fail_detail: test.fail(fail_detail) except exceptions.TestSkipError as skip_detail: test.cancel(skip_detail) except exceptions.TestError as error_detail: test.error(error_detail) except Exception as details: mig_result = migration_test.ret logging.error(details) check_migration_res(mig_result) if add_channel: # Get the channel device source path of remote guest if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent( **remote_virsh_dargs) file_path = tempfile.mktemp(dir=data_dir.get_tmp_dir()) remote_virsh_session.dumpxml(vm_name, to_file=file_path, debug=True, ignore_status=True) local_vmxml = vm_xml.VMXML.new_from_dumpxml(vm_name) local_vmxml.xmltreefile = xml_utils.XMLTreeFile(file_path) for elem in local_vmxml.devices.by_device_tag('channel'): logging.debug("Found channel device {}".format(elem)) if elem.type_name == channel_type_name: host_source = elem.source.get('path') logging.debug( "Remote guest uses {} for channel device".format( host_source)) break remote_virsh_session.close_session() if not host_source: test.fail("Can not find source for %s channel on remote host" % channel_type_name) # Prepare to wait for message on remote host from the channel cmd_parms = { 'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd } cmd_result = remote.run_remote_cmd( cmd_run_in_remote_host % host_source, cmd_parms, runner_on_target) # Send message from remote guest to the channel file remote_vm_obj = utils_test.RemoteVMManager(cmd_parms) vm_ip = vm.get_address() vm_pwd = params.get("password") remote_vm_obj.setup_ssh_auth(vm_ip, vm_pwd) cmd_result = remote_vm_obj.run_command(vm_ip, cmd_run_in_remote_guest_1) remote_vm_obj.run_command( vm_ip, cmd_run_in_remote_guest % results_stdout_52lts(cmd_result).strip()) logging.debug("Sending message is done") # Check message on remote host from the channel remote.run_remote_cmd(cmd_run_in_remote_host_1, cmd_parms, runner_on_target) logging.debug("Receiving message is done") remote.run_remote_cmd(cmd_run_in_remote_host_2, cmd_parms, runner_on_target) if check_complete_job: opts = " --completed" check_virsh_command_and_option("domjobinfo", opts) if extra.count("comp-xbzrle-cache"): params.update({'compare_to_value': cache // 1024}) check_domjobinfo(params, option=opts) if grep_str_local_log: cmd = "grep -E '%s' %s" % (grep_str_local_log, log_file) cmdRes = process.run(cmd, shell=True, ignore_status=True) if cmdRes.exit_status: test.fail(results_stderr_52lts(cmdRes).strip()) if grep_str_remote_log: cmd = "grep -E '%s' %s" % (grep_str_remote_log, log_file) cmd_parms = { 'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd } remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if xml_check_after_mig: if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent( **remote_virsh_dargs) target_guest_dumpxml = results_stdout_52lts( remote_virsh_session.dumpxml(vm_name, debug=True, ignore_status=True)).strip() if hpt_resize: check_str = hpt_resize elif htm_state: check_str = htm_state if hpt_resize or htm_state: xml_check_after_mig = "%s'%s'" % (xml_check_after_mig, check_str) if not re.search(xml_check_after_mig, target_guest_dumpxml): remote_virsh_session.close_session() test.fail("Fail to search '%s' in target guest XML:\n%s" % (xml_check_after_mig, target_guest_dumpxml)) if contrl_index: all_ctrls = re.findall(xml_check_after_mig, target_guest_dumpxml) if len(all_ctrls) != int(contrl_index) + 1: remote_virsh_session.close_session() test.fail( "%s pci-root controllers are expected in guest XML, " "but found %s" % (int(contrl_index) + 1, len(all_ctrls))) remote_virsh_session.close_session() if int(mig_result.exit_status) == 0: server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") check_vm_network_accessed(server_session) server_session.close() except exceptions.TestFail as details: is_TestFail = True test_exception = details except exceptions.TestSkipError as details: is_TestSkip = True test_exception = details except exceptions.TestError as details: is_TestError = True test_exception = details except Exception as details: test_exception = details finally: logging.debug("Recover test environment") try: # Clean VM on destination vm.connect_uri = dest_uri cleanup_dest(vm) vm.connect_uri = src_uri logging.info("Recovery VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) if remote_virsh_session: remote_virsh_session.close_session() if extra.count("--tls") and not disable_verify_peer: logging.debug("Recover the qemu configuration") libvirt.customize_libvirt_config(None, config_type="qemu", remote_host=True, extra_params=params, is_recover=True, config_object=qemu_conf) if config_libvirtd: logging.debug("Recover the libvirtd configuration") libvirt.customize_libvirt_config(None, remote_host=True, extra_params=params, is_recover=True, config_object=libvirtd_conf) logging.info("Remove local NFS image") source_file = params.get("source_file") libvirt.delete_local_disk("file", path=source_file) if objs_list: for obj in objs_list: logging.debug("Clean up local objs") del obj except Exception as exception_detail: if (not test_exception and not is_TestError and not is_TestFail and not is_TestSkip): raise exception_detail else: # if any of above exceptions has been raised, only print # error log here to avoid of hiding the original issue logging.error(exception_detail) # Check result if is_TestFail: test.fail(test_exception) if is_TestSkip: test.cancel(test_exception) if is_TestError: test.error(test_exception) if not test_exception: logging.info("Case execution is done.") else: test.error(test_exception)
def run(test, params, env): """ Test migration with glusterfs. """ def create_or_clean_backend_dir(g_uri, params, session=None, is_clean=False): """ Create/cleanup backend directory :params g_uri: glusterfs uri :params params: the parameters to be checked :params session: VM/remote session object :params is_cleanup: True for cleanup backend directory; False for create one. :return: gluster_img if is_clean is equal to True """ mount_point = params.get("gluster_mount_dir") is_symlink = params.get("gluster_create_symlink") == "yes" symlink_name = params.get("gluster_symlink") gluster_img = None if not is_clean: if not utils_misc.check_exists(mount_point, session): utils_misc.make_dirs(mount_point, session) if gluster.glusterfs_is_mounted(mount_point, session): gluster.glusterfs_umount(g_uri, mount_point, session) gluster.glusterfs_mount(g_uri, mount_point, session) gluster_img = os.path.join(mount_point, disk_img) if is_symlink: utils_misc.make_symlink(mount_point, symlink_name) utils_misc.make_symlink(mount_point, symlink_name, remote_session) gluster_img = os.path.join(symlink_name, disk_img) return gluster_img else: if is_symlink: utils_misc.rm_link(symlink_name, session) gluster.glusterfs_umount(g_uri, mount_point, session) if utils_misc.check_exists(mount_point, session): utils_misc.safe_rmdir(gluster_mount_dir, session=session) def do_migration(vm, dest_uri, options, extra): """ Execute the migration with given parameters :param vm: the guest to be migrated :param dest_uri: the destination uri for migration :param options: options next to 'migrate' command :param extra: options in the end of the migrate command line :return: CmdResult object """ # Migrate the guest. virsh_args.update({"ignore_status": True}) migration_res = vm.migrate(dest_uri, options, extra, **virsh_args) if int(migration_res.exit_status) != 0: logging.error("Migration failed for %s.", vm_name) return migration_res if vm.is_alive(): logging.info("VM is alive on destination %s.", dest_uri) else: test.fail("VM is not alive on destination %s" % dest_uri) # Throws exception if console shows panic message vm.verify_kernel_crash() return migration_res def check_migration_res(result): """ Check if the migration result is as expected :param result: the output of migration :raise: test.fail if test is failed """ if not result: test.error("No migration result is returned.") logging.info("Migration out: %s", result.stdout_text.strip()) logging.info("Migration error: %s", result.stderr_text.strip()) if status_error: # Migration should fail if err_msg: # Special error messages are expected if not re.search(err_msg, result.stderr_text.strip()): test.fail("Can not find the expected patterns '%s' in " "output '%s'" % (err_msg, result.stderr_text.strip())) else: logging.debug("It is the expected error message") else: if int(result.exit_status) != 0: logging.debug("Migration failure is expected result") else: test.fail("Migration success is unexpected result") else: if int(result.exit_status) != 0: test.fail(result.stderr_text.strip()) # Local variables virsh_args = {"debug": True} server_ip = params["server_ip"] = params.get("remote_ip") server_user = params["server_user"] = params.get("remote_user", "root") server_pwd = params["server_pwd"] = params.get("remote_pwd") client_ip = params["client_ip"] = params.get("local_ip") client_pwd = params["client_pwd"] = params.get("local_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") virsh_options = params.get("virsh_options", "--verbose --live") vol_name = params.get("vol_name") disk_format = params.get("disk_format", "qcow2") gluster_mount_dir = params.get("gluster_mount_dir") status_error = "yes" == params.get("status_error", "no") err_msg = params.get("err_msg") host_ip = params.get("gluster_server_ip", "") migr_vm_back = params.get("migrate_vm_back", "no") == "yes" selinux_local = params.get('set_sebool_local', 'yes') == "yes" selinux_remote = params.get('set_sebool_remote', 'no') == "yes" sebool_fusefs_local = params.get('set_sebool_fusefs_local', 'yes') sebool_fusefs_remote = params.get('set_sebool_fusefs_remote', 'yes') test_dict = dict(params) test_dict["local_boolean_varible"] = "virt_use_fusefs" test_dict["remote_boolean_varible"] = "virt_use_fusefs" remove_pkg = False seLinuxBool = None seLinuxfusefs = None gluster_uri = None mig_result = None # Make sure all of parameters are assigned a valid value check_parameters(test, params) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) params["virsh_migrate_connect_uri"] = libvirt_vm.complete_uri( params.get("migrate_source_host")) src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options: virsh_options = "%s %s" % (virsh_options, postcopy_options) params['virsh_options'] = virsh_options vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # Back up xml file. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() migrate_setup = libvirt.MigrationTest() try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Configure selinux if selinux_local or selinux_remote: seLinuxBool = utils_misc.SELinuxBoolean(params) seLinuxBool.setup() if sebool_fusefs_local or sebool_fusefs_remote: seLinuxfusefs = utils_misc.SELinuxBoolean(test_dict) seLinuxfusefs.setup() # Setup glusterfs and disk xml. disk_img = "gluster.%s" % disk_format params['disk_img'] = disk_img libvirt.set_vm_disk(vm, params) vm_xml_cxt = virsh.dumpxml(vm_name).stdout_text.strip() logging.debug("The VM XML with gluster disk source: \n%s", vm_xml_cxt) # Check if gluster server is deployed locally if not host_ip: logging.debug("Enable port 24007 and 49152:49216") migrate_setup.migrate_pre_setup(src_uri, params, ports="24007") migrate_setup.migrate_pre_setup(src_uri, params) gluster_uri = "{}:{}".format(client_ip, vol_name) else: gluster_uri = "{}:{}".format(host_ip, vol_name) remote_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") if gluster_mount_dir: # The package 'glusterfs-fuse' is not installed on target # which makes issue when trying to 'mount -t glusterfs' pkg_name = 'glusterfs-fuse' logging.debug("Check if glusterfs-fuse is installed") pkg_mgr = utils_package.package_manager(remote_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("glusterfs-fuse will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) else: remove_pkg = True gluster_img = create_or_clean_backend_dir(gluster_uri, params) create_or_clean_backend_dir(gluster_uri, params, remote_session) logging.debug("Gluster Image is %s", gluster_img) gluster_backend_disk = {'disk_source_name': gluster_img} # Update disk xml with gluster image in backend dir libvirt.set_vm_disk(vm, gluster_backend_disk) remote_session.close() mig_result = do_migration(vm, dest_uri, options, extra) check_migration_res(mig_result) if migr_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=client_ip, server_pwd=client_pwd, client_ip=server_ip, client_pwd=server_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine migrate_setup.migrate_pre_setup(src_uri, params) cmd = "virsh migrate %s %s %s" % (vm_name, virsh_options, src_uri) logging.debug("Start migrating: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) logging.info(cmd_result) if cmd_result.exit_status: destroy_cmd = "virsh destroy %s" % vm_name remote.run_remote_cmd(destroy_cmd, params, runner_on_target, ignore_status=False) test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) finally: logging.info("Recovery test environment") orig_config_xml.sync() # Clean up of pre migration setup for local machine if migr_vm_back: if 'ssh_connection' in locals(): ssh_connection.auto_recover = True migrate_setup.migrate_pre_setup(src_uri, params, cleanup=True) # Cleanup selinu configuration if seLinuxBool: seLinuxBool.cleanup() if seLinuxfusefs: seLinuxfusefs.cleanup() # Disable ports 24007 and 49152:49216 if not host_ip: logging.debug("Disable 24007 and 49152:49216 in Firewall") migrate_setup.migrate_pre_setup(src_uri, params, cleanup=True, ports="24007") migrate_setup.migrate_pre_setup(src_uri, params, cleanup=True) gluster.setup_or_cleanup_gluster(False, **params) # Cleanup backend directory/symlink if gluster_mount_dir and gluster_uri: remote_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") create_or_clean_backend_dir(gluster_uri, params, is_clean=True) create_or_clean_backend_dir(gluster_uri, params, remote_session, True) if remove_pkg: pkg_mgr = utils_package.package_manager(remote_session, pkg_name) if pkg_mgr.is_installed(pkg_name): logging.debug("glusterfs-fuse will be uninstalled") if not pkg_mgr.remove(): logging.error("Package '%s' un-installation fails", pkg_name) remote_session.close()
def run(test, params, env): def check_vm_network_accessed(): """ The operations to the VM need to be done before migration happens """ # 1. Confirm local VM can be accessed through network. logging.info("Check local VM network connectivity before migrating") s_ping, o_ping = utils_test.ping(vm.get_address(), count=10, timeout=20) logging.info(o_ping) if s_ping != 0: test.error("%s did not respond after %d sec." % (vm.name, 20)) def check_virsh_command_and_option(command, option=None): """ Check if virsh command exists :param command: the command to be checked :param option: the command option to be checked """ msg = "This version of libvirt does not support " if not virsh.has_help_command(command): test.cancel(msg + "virsh command '%s'" % command) if option and not virsh.has_command_help_match(command, option): test.cancel(msg + "virsh command '%s' with option '%s'" % (command, option)) def do_migration(vm, dest_uri, options, extra): """ Execute the migration with given parameters :param vm: the guest to be migrated :param dest_uri: the destination uri for migration :param options: options next to 'migrate' command :param extra: options in the end of the migrate command line :return: CmdResult object """ logging.info("Sleeping 10 seconds before migration") time.sleep(10) # Migrate the guest. migration_res = vm.migrate(dest_uri, options, extra, **virsh_args) logging.info("Migration out: %s", results_stdout_52lts(migration_res).strip()) logging.info("Migration error: %s", results_stderr_52lts(migration_res).strip()) if int(migration_res.exit_status) != 0: logging.error("Migration failed for %s.", vm_name) return migration_res if vm.is_alive(): # vm.connect_uri was updated logging.info("VM is alive on destination %s.", dest_uri) else: test.fail("VM is not alive on destination %s" % dest_uri) # Throws exception if console shows panic message vm.verify_kernel_crash() return migration_res def cleanup_libvirtd_log(log_file): """ Remove existing libvirtd log file on source and target host. :param log_file: log file with absolute path """ if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) cmd = "rm -f %s" % log_file logging.debug("Delete remote libvirt log file '%s'", log_file) run_remote_cmd(cmd) def cleanup_dest(vm): """ Clean up the destination host environment when doing the uni-direction migration. :param vm: the guest to be cleaned up """ logging.info("Cleaning up VMs on %s", vm.connect_uri) try: if virsh.domain_exists(vm.name, uri=vm.connect_uri): vm_state = vm.state() if vm_state == "paused": vm.resume() elif vm_state == "shut off": vm.start() vm.destroy(gracefully=False) if vm.is_persistent(): vm.undefine() except Exception as detail: logging.error("Cleaning up destination failed.\n%s", detail) def run_remote_cmd(cmd): """ A function to run a command on remote host. :param cmd: the command to be executed :return: CmdResult object """ remote_runner = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) cmdResult = remote_runner.run(cmd, ignore_status=True) if cmdResult.exit_status: test.fail("Failed to run '%s' on remote: %s" % (cmd, results_stderr_52lts(cmdResult).strip())) return cmdResult def run_stress_in_vm(): """ The function to load stress in VM """ stress_args = params.get( "stress_args", "--cpu 8 --io 4 " "--vm 2 --vm-bytes 128M " "--timeout 20s") try: vm_session.cmd('stress %s' % stress_args) except Exception as detail: logging.debug(detail) def check_timeout_postcopy(params): """ Check the vm state on target host after timeout when --postcopy and --timeout-postcopy are used. The vm state is expected as running. :param params: the parameters used """ timeout = int(params.get("timeout_postcopy", 10)) time.sleep(timeout + 1) remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) vm_state = results_stdout_52lts( remote_virsh_session.domstate(vm_name)).strip() if vm_state != "running": remote_virsh_session.close_session() test.fail( "After timeout '%s' seconds, " "the vm state on target host should " "be 'running', but '%s' found", timeout, vm_state) remote_virsh_session.close_session() def get_usable_compress_cache(pagesize): """ Get a number which is bigger than pagesize and is power of two. :param pagesize: the given integer :return: an integer satisfying the criteria """ def calculate(num): result = num & (num - 1) return (result == 0) item = pagesize found = False while (not found): item += 1 found = calculate(item) logging.debug( "%d is smallest one that is bigger than '%s' and " "is power of 2", item, pagesize) return item check_parameters(test, params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"ignore_status": True, "debug": True} server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") log_file = params.get("libvirt_log", "/var/log/libvirt/libvirtd.log") check_complete_job = "yes" == params.get("check_complete_job", "no") config_libvirtd = "yes" == params.get("config_libvirtd", "no") grep_str_remote_log = params.get("grep_str_remote_log", "") grep_str_local_log = params.get("grep_str_local_log", "") stress_in_vm = "yes" == params.get("stress_in_vm", "no") remote_virsh_dargs = { 'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True } # For TLS tls_recovery = params.get("tls_auto_recovery", "yes") # qemu config qemu_conf_dict = None # libvirtd config libvirtd_conf_dict = None remote_virsh_session = None vm = None vm_session = None libvirtd_conf = None qemu_conf = None mig_result = None test_exception = None is_TestError = False is_TestFail = False is_TestSkip = False asynch_migration = False # Objects to be cleaned up in the end objs_list = [] tls_obj = None # Local variables vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. orig_config_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) if not orig_config_xml: test.error("Backing up xmlfile failed.") try: # Change the disk of the vm to shared disk libvirt.set_vm_disk(vm, params) if extra.count("--tls"): qemu_conf_dict = {"migrate_tls_x509_verify": "1"} # Setup TLS tls_obj = TLSConnection(params) if tls_recovery == "yes": objs_list.append(tls_obj) tls_obj.auto_recover = True tls_obj.conn_setup() # Setup qemu configure logging.debug("Configure the qemu") cleanup_libvirtd_log(log_file) qemu_conf = libvirt.customize_libvirt_config(qemu_conf_dict, config_type="qemu", remote_host=True, extra_params=params) # Setup libvirtd if config_libvirtd: logging.debug("Configure the libvirtd") cleanup_libvirtd_log(log_file) libvirtd_conf_dict = setup_libvirtd_conf_dict(params) libvirtd_conf = libvirt.customize_libvirt_config( libvirtd_conf_dict, remote_host=True, extra_params=params) if not vm.is_alive(): vm.start() vm_session = vm.wait_for_login() check_vm_network_accessed() if stress_in_vm: pkg_name = 'stress' logging.debug("Check if stress tool is installed") pkg_mgr = utils_package.package_manager(vm_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("Stress tool will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) stress_thread = threading.Thread(target=run_stress_in_vm, args=()) stress_thread.start() if extra.count("timeout-postcopy"): asynch_migration = True func_name = check_timeout_postcopy if extra.count("comp-xbzrle-cache"): cache = get_usable_compress_cache(memory.get_page_size()) extra = "%s %s" % (extra, cache) # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options and not extra.count(postcopy_options): extra = "%s %s" % (extra, postcopy_options) if not asynch_migration: mig_result = do_migration(vm, dest_uri, options, extra) else: migration_test = libvirt.MigrationTest() logging.debug("vm.connect_uri=%s", vm.connect_uri) vms = [vm] try: migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, func=func_name, extra_opts=extra, func_params=params) mig_result = migration_test.ret except exceptions.TestFail as fail_detail: test.fail(fail_detail) except exceptions.TestSkipError as skip_detail: test.cancel(skip_detail) except exceptions.TestError as error_detail: test.error(error_detail) except Exception as details: mig_result = migration_test.ret logging.error(details) if int(mig_result.exit_status) != 0: test.fail(results_stderr_52lts(mig_result).strip()) if check_complete_job: search_str_domjobinfo = params.get("search_str_domjobinfo", None) opts = "--completed" args = vm_name + " " + opts check_virsh_command_and_option("domjobinfo", opts) jobinfo = results_stdout_52lts( virsh.domjobinfo(args, debug=True, ignore_status=True)).strip() logging.debug("Local job info on completion:\n%s", jobinfo) if extra.count("comp-xbzrle-cache") and search_str_domjobinfo: search_str_domjobinfo = "%s %s" % (search_str_domjobinfo, cache // 1024) if search_str_domjobinfo: if not re.search(search_str_domjobinfo, jobinfo): test.fail("Fail to search '%s' on local:\n%s" % (search_str_domjobinfo, jobinfo)) # Check remote host remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) jobinfo = results_stdout_52lts( remote_virsh_session.domjobinfo(args, debug=True, ignore_status=True)).strip() logging.debug("Remote job info on completion:\n%s", jobinfo) if search_str_domjobinfo: if not re.search(search_str_domjobinfo, jobinfo): remote_virsh_session.close_session() test.fail("Fail to search '%s' on remote:\n%s" % (search_str_domjobinfo, jobinfo)) remote_virsh_session.close_session() if grep_str_local_log: cmd = "grep -E '%s' %s" % (grep_str_local_log, log_file) cmdRes = process.run(cmd, shell=True, ignore_status=True) if cmdRes.exit_status: test.fail(results_stderr_52lts(cmdRes).strip()) if grep_str_remote_log: cmd = "grep -E '%s' %s" % (grep_str_remote_log, log_file) run_remote_cmd(cmd) except exceptions.TestFail as details: is_TestFail = True test_exception = details except exceptions.TestSkipError as details: is_TestSkip = True test_exception = details except exceptions.TestError as details: is_TestError = True test_exception = details except Exception as details: test_exception = details finally: logging.debug("Recover test environment") try: # Clean VM on destination vm.connect_uri = dest_uri cleanup_dest(vm) vm.connect_uri = src_uri logging.info("Recovery VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) if remote_virsh_session: remote_virsh_session.close_session() if extra.count("--tls"): logging.debug("Recover the qemu configuration") libvirt.customize_libvirt_config(None, config_type="qemu", remote_host=True, extra_params=params, is_recover=True, config_object=qemu_conf) if config_libvirtd: logging.debug("Recover the libvirtd configuration") libvirt.customize_libvirt_config(None, remote_host=True, extra_params=params, is_recover=True, config_object=libvirtd_conf) logging.info("Remove local NFS image") source_file = params.get("source_file") libvirt.delete_local_disk("file", path=source_file) if objs_list: for obj in objs_list: logging.debug("Clean up local objs") del obj except Exception as exception_detail: if (not test_exception and not is_TestError and not is_TestFail and not is_TestSkip): raise exception_detail else: # if any of above exceptions has been raised, only print # error log here to avoid of hiding the original issue logging.error(exception_detail) # Check result if is_TestFail: test.fail(test_exception) if is_TestSkip: test.cancel(test_exception) if is_TestError: test.error(test_exception) if not test_exception: logging.info("Case execution is done.") else: test.error(test_exception)
def run(test, params, env): """ Test virsh migrate command. """ def set_feature(vmxml, feature, value): """ Set guest features for PPC :param state: the htm status :param vmxml: guest xml """ features_xml = vm_xml.VMFeaturesXML() if feature == 'hpt': hpt_xml = vm_xml.VMFeaturesHptXML() hpt_xml.resizing = value features_xml.hpt = hpt_xml elif feature == 'htm': features_xml.htm = value vmxml.features = features_xml vmxml.sync() def trigger_hpt_resize(session): """ Check the HPT order file and dmesg :param session: the session to guest :raise: test.fail if required message is not found """ hpt_order_path = "/sys/kernel/debug/powerpc/hpt_order" hpt_order = session.cmd_output('cat %s' % hpt_order_path).strip() hpt_order = int(hpt_order) logging.info('Current hpt_order is %d', hpt_order) hpt_order += 1 cmd = 'echo %d > %s' % (hpt_order, hpt_order_path) cmd_result = session.cmd_status_output(cmd) result = process.CmdResult(stderr=cmd_result[1], stdout=cmd_result[1], exit_status=cmd_result[0]) libvirt.check_exit_status(result) dmesg = session.cmd('dmesg') dmesg_content = params.get('dmesg_content').split('|') for content in dmesg_content: if content % hpt_order not in dmesg: test.fail("'%s' is missing in dmesg" % (content % hpt_order)) else: logging.info("'%s' is found in dmesg", content % hpt_order) def check_vm_network_accessed(session=None): """ The operations to the VM need to be done before or after migration happens :param session: The session object to the host :raise: test.error when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") s_ping, _ = utils_test.ping(vm.get_address(), count=10, timeout=20, output_func=logging.debug, session=session) if s_ping != 0: if session: session.close() test.fail("%s did not respond after %d sec." % (vm.name, 20)) def check_virsh_command_and_option(command, option=None): """ Check if virsh command exists :param command: the command to be checked :param option: the command option to be checked """ msg = "This version of libvirt does not support " if not virsh.has_help_command(command): test.cancel(msg + "virsh command '%s'" % command) if option and not virsh.has_command_help_match(command, option): test.cancel(msg + "virsh command '%s' with option '%s'" % (command, option)) def add_ctrls(vm_xml, dev_type="pci", dev_index="0", dev_model="pci-root"): """ Add multiple devices :param dev_type: the type of the device to be added :param dev_index: the maximum index of the device to be added :param dev_model: the model of the device to be added """ for inx in range(0, int(dev_index) + 1): newcontroller = Controller("controller") newcontroller.type = dev_type newcontroller.index = inx newcontroller.model = dev_model logging.debug("New device is added:\n%s", newcontroller) vm_xml.add_device(newcontroller) vm_xml.sync() def do_migration(vm, dest_uri, options, extra): """ Execute the migration with given parameters :param vm: the guest to be migrated :param dest_uri: the destination uri for migration :param options: options next to 'migrate' command :param extra: options in the end of the migrate command line :return: CmdResult object """ logging.info("Sleeping 10 seconds before migration") time.sleep(10) # Migrate the guest. virsh_args.update({"ignore_status": True}) migration_res = vm.migrate(dest_uri, options, extra, **virsh_args) if int(migration_res.exit_status) != 0: logging.error("Migration failed for %s.", vm_name) return migration_res if vm.is_alive(): # vm.connect_uri was updated logging.info("VM is alive on destination %s.", dest_uri) else: test.fail("VM is not alive on destination %s" % dest_uri) # Throws exception if console shows panic message vm.verify_kernel_crash() return migration_res def cleanup_libvirtd_log(log_file): """ Remove existing libvirtd log file on source and target host. :param log_file: log file with absolute path """ if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) cmd = "rm -f %s" % log_file logging.debug("Delete remote libvirt log file '%s'", log_file) remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) def cleanup_dest(vm): """ Clean up the destination host environment when doing the uni-direction migration. :param vm: the guest to be cleaned up """ logging.info("Cleaning up VMs on %s", vm.connect_uri) try: if virsh.domain_exists(vm.name, uri=vm.connect_uri): vm_state = vm.state() if vm_state == "paused": vm.resume() elif vm_state == "shut off": vm.start() vm.destroy(gracefully=False) if vm.is_persistent(): vm.undefine() except Exception as detail: logging.error("Cleaning up destination failed.\n%s", detail) def run_stress_in_vm(): """ The function to load stress in VM """ stress_args = params.get("stress_args", "--cpu 8 --io 4 " "--vm 2 --vm-bytes 128M " "--timeout 20s") try: vm_session.cmd('stress %s' % stress_args) except Exception as detail: logging.debug(detail) def control_migrate_speed(to_speed=1, opts=""): """ Control migration duration :param to_speed: the speed value in Mbps to be set for migration :return int: the new migration speed after setting """ virsh_args.update({"ignore_status": False}) old_speed = virsh.migrate_getspeed(vm_name, extra=opts, **virsh_args) logging.debug("Current migration speed is %s MiB/s\n", old_speed.stdout.strip()) logging.debug("Set migration speed to %d MiB/s\n", to_speed) cmd_result = virsh.migrate_setspeed(vm_name, to_speed, extra=opts, **virsh_args) actual_speed = virsh.migrate_getspeed(vm_name, extra=opts, **virsh_args) logging.debug("New migration speed is %s MiB/s\n", actual_speed.stdout.strip()) return int(actual_speed.stdout.strip()) def check_setspeed(params): """ Set/get migration speed :param params: the parameters used :raise: test.fail if speed set does not take effect """ expected_value = int(params.get("migrate_speed", '41943040')) // (1024 * 1024) actual_value = control_migrate_speed(to_speed=expected_value) params.update({'compare_to_value': actual_value}) if actual_value != expected_value: test.fail("Migration speed is expected to be '%d MiB/s', but '%d MiB/s' " "found" % (expected_value, actual_value)) def check_domjobinfo(params, option=""): """ Check given item in domjobinfo of the guest is as expected :param params: the parameters used :param option: options for domjobinfo :raise: test.fail if the value of given item is unexpected """ def search_jobinfo(jobinfo): """ Find value of given item in domjobinfo :param jobinfo: cmdResult object :raise: test.fail if not found """ for item in jobinfo.stdout.splitlines(): if item.count(jobinfo_item): groups = re.findall(r'[0-9.]+', item.strip()) logging.debug("In '%s' search '%s'\n", item, groups[0]) if (math.fabs(float(groups[0]) - float(compare_to_value)) // float(compare_to_value) > diff_rate): test.fail("{} {} has too much difference from " "{}".format(jobinfo_item, groups[0], compare_to_value)) break jobinfo_item = params.get("jobinfo_item") compare_to_value = params.get("compare_to_value") logging.debug("compare_to_value:%s", compare_to_value) diff_rate = float(params.get("diff_rate", "0")) if not jobinfo_item or not compare_to_value: return vm_ref = '{}{}'.format(vm_name, option) jobinfo = virsh.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) check_domjobinfo_remote = params.get("check_domjobinfo_remote") if check_domjobinfo_remote: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) jobinfo = remote_virsh_session.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) remote_virsh_session.close_session() def search_jobinfo_output(jobinfo, items_to_check, postcopy_req=False): """ Check the results of domjobinfo :param jobinfo: cmdResult object :param items_to_check: expected value :param postcopy_req: True for postcopy migration and False for precopy :return: False if not found """ expected_value = copy.deepcopy(items_to_check) logging.debug("The items_to_check is %s", expected_value) for item in jobinfo.splitlines(): item_key = item.strip().split(':')[0] if "all_items" in expected_value and len(item_key) > 0: # "Time elapsed w/o network" and "Downtime w/o network" # have a chance to be missing, it is normal if item_key in ['Downtime w/o network', 'Time elapsed w/o network']: continue if expected_value["all_items"][0] == item_key: del expected_value["all_items"][0] else: test.fail("The item '%s' should be '%s'" % (item_key, expected_value["all_items"][0])) if item_key in expected_value: item_value = ':'.join(item.strip().split(':')[1:]).strip() if item_value != expected_value.get(item_key): test.fail("The value of {} is {} which is not " "expected".format(item_key, item_value)) else: del expected_value[item_key] if postcopy_req and item_key == "Postcopy requests": if int(item.strip().split(':')[1]) <= 0: test.fail("The value of Postcopy requests is incorrect") # Check if all the items in expect_dict checked or not if "all_items" in expected_value: if len(expected_value["all_items"]) > 0: test.fail("Missing item: {} from all_items" .format(expected_value["all_items"])) else: del expected_value["all_items"] if len(expected_value) != 0: test.fail("Missing item: {}".format(expected_value)) def set_migratepostcopy(): """ Switch to postcopy during migration """ res = virsh.migrate_postcopy(vm_name) logging.debug("Command output: %s", res) if not utils_misc.wait_for( lambda: migration_test.check_vm_state(vm_name, "paused", "post-copy"), 10): test.fail("vm status is expected to 'paused (post-copy)'") def check_domjobinfo_output(option="", is_mig_compelete=False): """ Check all items in domjobinfo of the guest on both remote and local :param option: options for domjobinfo :param is_mig_compelete: False for domjobinfo checking during migration, True for domjobinfo checking after migration :raise: test.fail if the value of given item is unexpected """ expected_list_during_mig = ["Job type", "Operation", "Time elapsed", "Data processed", "Data remaining", "Data total", "Memory processed", "Memory remaining", "Memory total", "Memory bandwidth", "Dirty rate", "Page size", "Iteration", "Constant pages", "Normal pages", "Normal data", "Expected downtime", "Setup time"] if libvirt_version.version_compare(4, 10, 0): expected_list_during_mig.insert(13, "Postcopy requests") expected_list_after_mig_src = copy.deepcopy(expected_list_during_mig) expected_list_after_mig_src[-2] = 'Total downtime' expected_list_after_mig_dest = copy.deepcopy(expected_list_after_mig_src) # Check version in remote if not expected_list_after_mig_dest.count("Postcopy requests"): remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, "#") if libvirt_version.version_compare(4, 10, 0, session=remote_session): expected_list_after_mig_dest.insert(14, "Postcopy requests") remote_session.close() expect_dict = {"src_notdone": {"Job type": "Unbounded", "Operation": "Outgoing migration", "all_items": expected_list_during_mig}, "dest_notdone": {"error": "Operation not supported: mig" "ration statistics are availab" "le only on the source host"}, "src_done": {"Job type": "Completed", "Operation": "Outgoing migration", "all_items": expected_list_after_mig_src}, "dest_done": {"Job type": "Completed", "Operation": "Incoming migration", "all_items": expected_list_after_mig_dest}} pc_opt = False if postcopy_options: pc_opt = True if is_mig_compelete: expect_dict["dest_done"].clear() expect_dict["dest_done"]["Job type"] = "None" else: set_migratepostcopy() vm_ref = '{}{}'.format(vm_name, option) src_jobinfo = virsh.domjobinfo(vm_ref, **virsh_args) cmd = "virsh domjobinfo {} {}".format(vm_name, option) dest_jobinfo = remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if not is_mig_compelete: search_jobinfo_output(src_jobinfo.stdout, expect_dict["src_notdone"], postcopy_req=pc_opt) search_jobinfo_output(dest_jobinfo.stderr, expect_dict["dest_notdone"]) else: search_jobinfo_output(src_jobinfo.stdout, expect_dict["src_done"]) search_jobinfo_output(dest_jobinfo.stdout, expect_dict["dest_done"]) def check_maxdowntime(params): """ Set/get migration maxdowntime :param params: the parameters used :raise: test.fail if maxdowntime set does not take effect """ expected_value = int(float(params.get("migrate_maxdowntime", '0.3')) * 1000) virsh_args.update({"ignore_status": False}) old_value = int(virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("Current migration maxdowntime is %d ms", old_value) logging.debug("Set migration maxdowntime to %d ms", expected_value) virsh.migrate_setmaxdowntime(vm_name, expected_value, **virsh_args) actual_value = int(virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("New migration maxdowntime is %d ms", actual_value) if actual_value != expected_value: test.fail("Migration maxdowntime is expected to be '%d ms', but '%d ms' " "found" % (expected_value, actual_value)) params.update({'compare_to_value': actual_value}) def run_time(init_time=2): """ Compare the duration of func to an expected one :param init_time: Expected run time :raise: test.fail if func takes more than init_time(second) """ def check_time(func): def wrapper(*args, **kwargs): start = time.time() result = func(*args, **kwargs) duration = time.time() - start if duration > init_time: test.fail("It takes too long to check {}. The duration is " "{}s which should be less than {}s" .format(func.__doc__, duration, init_time)) return result return wrapper return check_time def run_domstats(vm_name): """ Run domstats and domstate during migration in source and destination :param vm_name: VM name :raise: test.fail if domstats does not return in 2s or domstate is incorrect """ @run_time() def check_source_stats(vm_name): """domstats in source""" vm_stats = virsh.domstats(vm_name) logging.debug("domstats in source: {}".format(vm_stats)) @run_time() def check_dest_stats(vm_name): """domstats in target""" cmd = "virsh domstats {}".format(vm_name) dest_stats = remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) logging.debug("domstats in destination: {}".format(dest_stats)) expected_remote_state = "paused" expected_source_state = ["paused", "running"] if postcopy_options: set_migratepostcopy() expected_remote_state = "running" expected_source_state = ["paused"] check_source_stats(vm_name) vm_stat = virsh.domstate(vm_name, ignore_status=False) if ((not len(vm_stat.stdout.split())) or vm_stat.stdout.split()[0] not in expected_source_state): test.fail("Incorrect VM stat on source machine: {}" .format(vm_stat.stdout)) check_dest_stats(vm_name) cmd = "virsh domstate {}".format(vm_name) remote_vm_state = remote.run_remote_cmd(cmd, cmd_parms, runner_on_target, ignore_status=False) if ((not len(remote_vm_state.stdout.split())) or remote_vm_state.stdout.split()[0] != expected_remote_state): test.fail("Incorrect VM stat on destination machine: {}" .format(remote_vm_state.stdout)) else: logging.debug("vm stat on destination: {}".format(remote_vm_state)) if postcopy_options: vm_stat = virsh.domstate(vm_name, ignore_status=False) if ((not len(vm_stat.stdout.split())) or vm_stat.stdout.split()[0] != "paused"): test.fail("Incorrect VM stat on source machine: {}" .format(vm_stat.stdout)) def kill_qemu_target(): """ Kill qemu process on target host during Finish Phase of migration :raise: test.fail if domstate is not "post-copy failed" after qemu killed """ if not vm.is_qemu(): test.cancel("This case is qemu guest only.") set_migratepostcopy() emulator = new_xml.get_devices('emulator')[0] logging.debug("emulator is %s", emulator.path) cmd = 'pkill -9 {}'.format(os.path.basename(emulator.path)) runner_on_target.run(cmd) if not utils_misc.wait_for( lambda: migration_test.check_vm_state(vm_name, "paused", "post-copy failed"), 60): test.fail("vm status is expected to 'paused (post-copy failed)'") def do_actions_during_migrate(params): """ The entry point to execute action list during migration :param params: the parameters used """ actions_during_migration = params.get("actions_during_migration") if not actions_during_migration: return for action in actions_during_migration.split(","): if action == 'setspeed': check_setspeed(params) elif action == 'domjobinfo': check_domjobinfo(params) elif action == 'setmaxdowntime': check_maxdowntime(params) elif action == 'converge': check_converge(params) elif action == 'domjobinfo_output_all': check_domjobinfo_output() elif action == 'checkdomstats': run_domstats(vm_name) elif action == 'killqemutarget': kill_qemu_target() time.sleep(3) def attach_channel_xml(): """ Create channel xml and attach it to guest configuration """ # Check if pty channel exists already for elem in new_xml.devices.by_device_tag('channel'): if elem.type_name == channel_type_name: logging.debug("{0} channel already exists in guest. " "No need to add new one".format(channel_type_name)) return params = {'channel_type_name': channel_type_name, 'target_type': target_type, 'target_name': target_name} channel_xml = libvirt.create_channel_xml(params) virsh.attach_device(domain_opt=vm_name, file_opt=channel_xml.xml, flagstr="--config", ignore_status=False) logging.debug("New VMXML with channel:\n%s", virsh.dumpxml(vm_name)) def check_timeout_postcopy(params): """ Check the vm state on target host after timeout when --postcopy and --timeout-postcopy are used. The vm state is expected as running. :param params: the parameters used """ timeout = int(params.get("timeout_postcopy", 10)) time.sleep(timeout + 1) remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) vm_state = results_stdout_52lts(remote_virsh_session.domstate(vm_name)).strip() if vm_state != "running": remote_virsh_session.close_session() test.fail("After timeout '%s' seconds, " "the vm state on target host should " "be 'running', but '%s' found" % (timeout, vm_state)) remote_virsh_session.close_session() def check_converge(params): """ Handle option '--auto-converge --auto-converge-initial --auto-converge-increment '. 'Auto converge throttle' in domjobinfo should start with the initial value and increase with correct increment and max value is 99. :param params: The parameters used :raise: exceptions.TestFail when unexpected or no throttle is found """ initial = int(params.get("initial", 20)) increment = int(params.get("increment", 10)) max_converge = int(params.get("max_converge", 99)) allow_throttle_list = [initial + count * increment for count in range(0, (100 - initial) // increment + 1) if (initial + count * increment) < 100] allow_throttle_list.append(max_converge) logging.debug("The allowed 'Auto converge throttle' value " "is %s", allow_throttle_list) throttle = 0 jobtype = "None" while throttle < 100: cmd_result = virsh.domjobinfo(vm_name, debug=True, ignore_status=True) if cmd_result.exit_status: logging.debug(cmd_result.stderr) # Check if migration is completed if "domain is not running" in cmd_result.stderr: args = vm_name + " --completed" cmd_result = virsh.domjobinfo(args, debug=True, ignore_status=True) if cmd_result.exit_status: test.error("Failed to get domjobinfo and domjobinfo " "--completed: %s" % cmd_result.stderr) else: test.error("Failed to get domjobinfo: %s" % cmd_result.stderr) jobinfo = cmd_result.stdout for line in jobinfo.splitlines(): key = line.split(':')[0] if key.count("Job type"): jobtype = line.split(':')[-1].strip() elif key.count("Auto converge throttle"): throttle = int(line.split(':')[-1].strip()) logging.debug("Auto converge throttle:%s", str(throttle)) if throttle and throttle not in allow_throttle_list: test.fail("Invalid auto converge throttle " "value '%s'" % throttle) if throttle == 99: logging.debug("'Auto converge throttle' reaches maximum " "allowed value 99") break if jobtype == "None" or jobtype == "Completed": logging.debug("Jobtype:%s", jobtype) if not throttle: test.fail("'Auto converge throttle' is " "not found in the domjobinfo") break time.sleep(5) def get_usable_compress_cache(pagesize): """ Get a number which is bigger than pagesize and is power of two. :param pagesize: the given integer :return: an integer satisfying the criteria """ def calculate(num): result = num & (num - 1) return (result == 0) item = pagesize found = False while (not found): item += 1 found = calculate(item) logging.debug("%d is smallest one that is bigger than '%s' and " "is power of 2", item, pagesize) return item def update_config_file(config_type, new_conf, remote_host=True, params=None): """ Update the specified configuration file with dict :param config_type: Like libvirtd, qemu :param new_conf: The str including new configuration :param remote_host: True to also update in remote host :param params: The dict including parameters to connect remote host :return: utils_config.LibvirtConfigCommon object """ logging.debug("Update configuration file") cleanup_libvirtd_log(log_file) config_dict = eval(new_conf) updated_conf = libvirt.customize_libvirt_config(config_dict, config_type=config_type, remote_host=remote_host, extra_params=params) return updated_conf def check_migration_res(result): """ Check if the migration result is as expected :param result: the output of migration :raise: test.fail if test is failed """ logging.info("Migration out: %s", results_stdout_52lts(result).strip()) logging.info("Migration error: %s", results_stderr_52lts(result).strip()) if status_error: # Migration should fail if err_msg: # Special error messages are expected if not re.search(err_msg, results_stderr_52lts(result).strip()): test.fail("Can not find the expected patterns '%s' in " "output '%s'" % (err_msg, results_stderr_52lts(result).strip())) else: logging.debug("It is the expected error message") else: if int(result.exit_status) != 0: logging.debug("Migration failure is expected result") else: test.fail("Migration success is unexpected result") else: if int(result.exit_status) != 0: test.fail(results_stderr_52lts(result).strip()) check_parameters(test, params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} virsh_opt = params.get("virsh_opt", "") server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") log_file = params.get("libvirt_log", "/var/log/libvirt/libvirtd.log") check_complete_job = "yes" == params.get("check_complete_job", "no") check_domjobinfo_results = "yes" == params.get("check_domjobinfo_results") contrl_index = params.get("new_contrl_index", None) asynch_migration = "yes" == params.get("asynch_migrate", "no") grep_str_remote_log = params.get("grep_str_remote_log", "") grep_str_local_log = params.get("grep_str_local_log", "") disable_verify_peer = "yes" == params.get("disable_verify_peer", "no") status_error = "yes" == params.get("status_error", "no") stress_in_vm = "yes" == params.get("stress_in_vm", "no") low_speed = params.get("low_speed", None) remote_virsh_dargs = {'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True} cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} hpt_resize = params.get("hpt_resize", None) htm_state = params.get("htm_state", None) # For pty channel test add_channel = "yes" == params.get("add_channel", "no") channel_type_name = params.get("channel_type_name", None) target_type = params.get("target_type", None) target_name = params.get("target_name", None) cmd_run_in_remote_guest = params.get("cmd_run_in_remote_guest", None) cmd_run_in_remote_guest_1 = params.get("cmd_run_in_remote_guest_1", None) cmd_run_in_remote_host = params.get("cmd_run_in_remote_host", None) cmd_run_in_remote_host_1 = params.get("cmd_run_in_remote_host_1", None) cmd_run_in_remote_host_2 = params.get("cmd_run_in_remote_host_2", None) # For qemu command line checking qemu_check = params.get("qemu_check", None) xml_check_after_mig = params.get("guest_xml_check_after_mig", None) # params for cache matrix test cache = params.get("cache") remove_cache = "yes" == params.get("remove_cache", "no") err_msg = params.get("err_msg") arch = platform.machine() if any([hpt_resize, contrl_index, htm_state]) and 'ppc64' not in arch: test.cancel("The case is PPC only.") # For TLS tls_recovery = params.get("tls_auto_recovery", "yes") # qemu config qemu_conf_dict = None # libvirtd config libvirtd_conf_dict = None # remote shell session remote_session = None remote_virsh_session = None vm = None vm_session = None libvirtd_conf = None qemu_conf = None mig_result = None test_exception = None is_TestError = False is_TestFail = False is_TestSkip = False # Objects to be cleaned up in the end objs_list = [] tls_obj = None # Local variables vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() if not orig_config_xml: test.error("Backing up xmlfile failed.") try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Change the configuration files if needed before starting guest # For qemu.conf if extra.count("--tls"): # Setup TLS tls_obj = TLSConnection(params) if tls_recovery == "yes": objs_list.append(tls_obj) tls_obj.auto_recover = True tls_obj.conn_setup() # Setup qemu.conf qemu_conf_dict = params.get("qemu_conf_dict") if qemu_conf_dict: qemu_conf = update_config_file('qemu', qemu_conf_dict, params=params) # Setup libvirtd libvirtd_conf_dict = params.get("libvirtd_conf_dict") if libvirtd_conf_dict: libvirtd_conf = update_config_file('libvirtd', libvirtd_conf_dict, params=params) # Prepare required guest xml before starting guest if contrl_index: new_xml.remove_all_device_by_type('controller') logging.debug("After removing controllers, current XML:\n%s\n", new_xml) add_ctrls(new_xml, dev_index=contrl_index) if add_channel: attach_channel_xml() if hpt_resize: set_feature(new_xml, 'hpt', hpt_resize) if htm_state: set_feature(new_xml, 'htm', htm_state) if cache: params["driver_cache"] = cache if remove_cache: params["enable_cache"] = "no" # Change the disk of the vm to shared disk and then start VM libvirt.set_vm_disk(vm, params) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check qemu command line after guest is started if qemu_check: check_content = qemu_check if hpt_resize: check_content = "%s%s" % (qemu_check, hpt_resize) if htm_state: check_content = "%s%s" % (qemu_check, htm_state) libvirt.check_qemu_cmd_line(check_content) # Check local guest network connection before migration vm_session = vm.wait_for_login() check_vm_network_accessed() # Preparation for the running guest before migration if hpt_resize and hpt_resize != 'disabled': trigger_hpt_resize(vm_session) if stress_in_vm: pkg_name = 'stress' logging.debug("Check if stress tool is installed") pkg_mgr = utils_package.package_manager(vm_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("Stress tool will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) stress_thread = threading.Thread(target=run_stress_in_vm, args=()) stress_thread.start() if extra.count("timeout-postcopy"): func_name = check_timeout_postcopy if params.get("actions_during_migration"): func_name = do_actions_during_migrate if extra.count("comp-xbzrle-cache"): cache = get_usable_compress_cache(memory.get_page_size()) extra = "%s %s" % (extra, cache) # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (extra, postcopy_options) if low_speed: control_migrate_speed(int(low_speed)) if postcopy_options and libvirt_version.version_compare(5, 0, 0): control_migrate_speed(int(low_speed), opts=postcopy_options) # Execute migration process if not asynch_migration: mig_result = do_migration(vm, dest_uri, options, extra) else: migration_test = libvirt.MigrationTest() logging.debug("vm.connect_uri=%s", vm.connect_uri) vms = [vm] try: migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_opt, func=func_name, extra_opts=extra, func_params=params) mig_result = migration_test.ret except exceptions.TestFail as fail_detail: test.fail(fail_detail) except exceptions.TestSkipError as skip_detail: test.cancel(skip_detail) except exceptions.TestError as error_detail: test.error(error_detail) except Exception as details: mig_result = migration_test.ret logging.error(details) check_migration_res(mig_result) if add_channel: # Get the channel device source path of remote guest if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) file_path = tempfile.mktemp(dir=data_dir.get_tmp_dir()) remote_virsh_session.dumpxml(vm_name, to_file=file_path, debug=True, ignore_status=True) local_vmxml = vm_xml.VMXML.new_from_dumpxml(vm_name) local_vmxml.xmltreefile = xml_utils.XMLTreeFile(file_path) for elem in local_vmxml.devices.by_device_tag('channel'): logging.debug("Found channel device {}".format(elem)) if elem.type_name == channel_type_name: host_source = elem.source.get('path') logging.debug("Remote guest uses {} for channel device".format(host_source)) break remote_virsh_session.close_session() if not host_source: test.fail("Can not find source for %s channel on remote host" % channel_type_name) # Prepare to wait for message on remote host from the channel cmd_result = remote.run_remote_cmd(cmd_run_in_remote_host % host_source, cmd_parms, runner_on_target) # Send message from remote guest to the channel file remote_vm_obj = utils_test.RemoteVMManager(cmd_parms) vm_ip = vm.get_address() vm_pwd = params.get("password") remote_vm_obj.setup_ssh_auth(vm_ip, vm_pwd) cmd_result = remote_vm_obj.run_command(vm_ip, cmd_run_in_remote_guest_1) remote_vm_obj.run_command(vm_ip, cmd_run_in_remote_guest % results_stdout_52lts(cmd_result).strip()) logging.debug("Sending message is done") # Check message on remote host from the channel remote.run_remote_cmd(cmd_run_in_remote_host_1, cmd_parms, runner_on_target) logging.debug("Receiving message is done") remote.run_remote_cmd(cmd_run_in_remote_host_2, cmd_parms, runner_on_target) if check_complete_job: opts = " --completed" check_virsh_command_and_option("domjobinfo", opts) if extra.count("comp-xbzrle-cache"): params.update({'compare_to_value': cache // 1024}) check_domjobinfo(params, option=opts) if check_domjobinfo_results: check_domjobinfo_output(option=opts, is_mig_compelete=True) if grep_str_local_log: cmd = "grep -E '%s' %s" % (grep_str_local_log, log_file) cmdRes = process.run(cmd, shell=True, ignore_status=True) if cmdRes.exit_status: test.fail(results_stderr_52lts(cmdRes).strip()) if grep_str_remote_log: cmd = "grep -E '%s' %s" % (grep_str_remote_log, log_file) remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if xml_check_after_mig: if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) target_guest_dumpxml = results_stdout_52lts( remote_virsh_session.dumpxml(vm_name, debug=True, ignore_status=True)).strip() if hpt_resize: check_str = hpt_resize elif htm_state: check_str = htm_state if hpt_resize or htm_state: xml_check_after_mig = "%s'%s'" % (xml_check_after_mig, check_str) if not re.search(xml_check_after_mig, target_guest_dumpxml): remote_virsh_session.close_session() test.fail("Fail to search '%s' in target guest XML:\n%s" % (xml_check_after_mig, target_guest_dumpxml)) if contrl_index: all_ctrls = re.findall(xml_check_after_mig, target_guest_dumpxml) if len(all_ctrls) != int(contrl_index) + 1: remote_virsh_session.close_session() test.fail("%s pci-root controllers are expected in guest XML, " "but found %s" % (int(contrl_index) + 1, len(all_ctrls))) remote_virsh_session.close_session() if int(mig_result.exit_status) == 0: server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") check_vm_network_accessed(server_session) server_session.close() except exceptions.TestFail as details: is_TestFail = True test_exception = details except exceptions.TestSkipError as details: is_TestSkip = True test_exception = details except exceptions.TestError as details: is_TestError = True test_exception = details except Exception as details: test_exception = details finally: logging.debug("Recover test environment") try: # Clean VM on destination vm.connect_uri = dest_uri cleanup_dest(vm) vm.connect_uri = src_uri logging.info("Recovery VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) if remote_virsh_session: remote_virsh_session.close_session() if remote_session: remote_session.close() # Delete files on target # Killing qemu process on target may lead a problem like # qemu process becomes a zombie process whose ppid is 1. # As a workaround, have to remove the files under # /var/run/libvirt/qemu to make libvirt work. if vm.is_qemu(): dest_pid_files = os.path.join("/var/run/libvirt/qemu", vm_name + '*') cmd = "rm -f %s" % dest_pid_files logging.debug("Delete remote pid files '%s'", dest_pid_files) remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if extra.count("--tls") and not disable_verify_peer: logging.debug("Recover the qemu configuration") libvirt.customize_libvirt_config(None, config_type="qemu", remote_host=True, extra_params=params, is_recover=True, config_object=qemu_conf) for update_conf in [libvirtd_conf, qemu_conf]: if update_conf: logging.debug("Recover the configurations") libvirt.customize_libvirt_config(None, remote_host=True, extra_params=params, is_recover=True, config_object=update_conf) logging.info("Remove local NFS image") source_file = params.get("source_file") libvirt.delete_local_disk("file", path=source_file) if objs_list: for obj in objs_list: logging.debug("Clean up local objs") del obj except Exception as exception_detail: if (not test_exception and not is_TestError and not is_TestFail and not is_TestSkip): raise exception_detail else: # if any of above exceptions has been raised, only print # error log here to avoid of hiding the original issue logging.error(exception_detail) # Check result if is_TestFail: test.fail(test_exception) if is_TestSkip: test.cancel(test_exception) if is_TestError: test.error(test_exception) if not test_exception: logging.info("Case execution is done.") else: test.error(test_exception)
def run(test, params, env): """ Test virsh migrate command. """ def set_feature(vmxml, feature, value): """ Set guest features for PPC :param state: the htm status :param vmxml: guest xml """ features_xml = vm_xml.VMFeaturesXML() if feature == 'hpt': features_xml.hpt_resizing = value elif feature == 'htm': features_xml.htm = value vmxml.features = features_xml vmxml.sync() def trigger_hpt_resize(session): """ Check the HPT order file and dmesg :param session: the session to guest :raise: test.fail if required message is not found """ hpt_order_path = "/sys/kernel/debug/powerpc/hpt_order" hpt_order = session.cmd_output('cat %s' % hpt_order_path).strip() hpt_order = int(hpt_order) logging.info('Current hpt_order is %d', hpt_order) hpt_order += 1 cmd = 'echo %d > %s' % (hpt_order, hpt_order_path) cmd_result = session.cmd_status_output(cmd) result = process.CmdResult(stderr=cmd_result[1], stdout=cmd_result[1], exit_status=cmd_result[0]) libvirt.check_exit_status(result) dmesg = session.cmd('dmesg') dmesg_content = params.get('dmesg_content').split('|') for content in dmesg_content: if content % hpt_order not in dmesg: test.fail("'%s' is missing in dmesg" % (content % hpt_order)) else: logging.info("'%s' is found in dmesg", content % hpt_order) def check_vm_network_accessed(session=None): """ The operations to the VM need to be done before or after migration happens :param session: The session object to the host :raise: test.error when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") s_ping, _ = utils_test.ping(vm.get_address(), count=10, timeout=20, output_func=logging.debug, session=session) if s_ping != 0: if session: session.close() test.fail("%s did not respond after %d sec." % (vm.name, 20)) def check_virsh_command_and_option(command, option=None): """ Check if virsh command exists :param command: the command to be checked :param option: the command option to be checked """ msg = "This version of libvirt does not support " if not virsh.has_help_command(command): test.cancel(msg + "virsh command '%s'" % command) if option and not virsh.has_command_help_match(command, option): test.cancel(msg + "virsh command '%s' with option '%s'" % (command, option)) def add_ctrls(vm_xml, dev_type="pci", dev_index="0", dev_model="pci-root"): """ Add multiple devices :param dev_type: the type of the device to be added :param dev_index: the maximum index of the device to be added :param dev_model: the model of the device to be added """ for inx in range(0, int(dev_index) + 1): newcontroller = Controller("controller") newcontroller.type = dev_type newcontroller.index = inx newcontroller.model = dev_model logging.debug("New device is added:\n%s", newcontroller) vm_xml.add_device(newcontroller) vm_xml.sync() def do_migration(vm, dest_uri, options, extra): """ Execute the migration with given parameters :param vm: the guest to be migrated :param dest_uri: the destination uri for migration :param options: options next to 'migrate' command :param extra: options in the end of the migrate command line :return: CmdResult object """ logging.info("Sleeping 10 seconds before migration") time.sleep(10) # Migrate the guest. virsh_args.update({"ignore_status": True}) migration_res = vm.migrate(dest_uri, options, extra, **virsh_args) if int(migration_res.exit_status) != 0: logging.error("Migration failed for %s.", vm_name) return migration_res if vm.is_alive(): # vm.connect_uri was updated logging.info("VM is alive on destination %s.", dest_uri) else: test.fail("VM is not alive on destination %s" % dest_uri) # Throws exception if console shows panic message vm.verify_kernel_crash() return migration_res def cleanup_libvirtd_log(log_file): """ Remove existing libvirtd log file on source and target host. :param log_file: log file with absolute path """ if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) cmd = "rm -f %s" % log_file logging.debug("Delete remote libvirt log file '%s'", log_file) cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) def cleanup_dest(vm): """ Clean up the destination host environment when doing the uni-direction migration. :param vm: the guest to be cleaned up """ logging.info("Cleaning up VMs on %s", vm.connect_uri) try: if virsh.domain_exists(vm.name, uri=vm.connect_uri): vm_state = vm.state() if vm_state == "paused": vm.resume() elif vm_state == "shut off": vm.start() vm.destroy(gracefully=False) if vm.is_persistent(): vm.undefine() except Exception as detail: logging.error("Cleaning up destination failed.\n%s", detail) def run_stress_in_vm(): """ The function to load stress in VM """ stress_args = params.get("stress_args", "--cpu 8 --io 4 " "--vm 2 --vm-bytes 128M " "--timeout 20s") try: vm_session.cmd('stress %s' % stress_args) except Exception as detail: logging.debug(detail) def control_migrate_speed(to_speed=1): """ Control migration duration :param to_speed: the speed value in Mbps to be set for migration :return int: the new migration speed after setting """ virsh_args.update({"ignore_status": False}) old_speed = virsh.migrate_getspeed(vm_name, **virsh_args) logging.debug("Current migration speed is %s MiB/s\n", old_speed.stdout.strip()) logging.debug("Set migration speed to %d MiB/s\n", to_speed) cmd_result = virsh.migrate_setspeed(vm_name, to_speed, "", **virsh_args) actual_speed = virsh.migrate_getspeed(vm_name, **virsh_args) logging.debug("New migration speed is %s MiB/s\n", actual_speed.stdout.strip()) return int(actual_speed.stdout.strip()) def check_setspeed(params): """ Set/get migration speed :param params: the parameters used :raise: test.fail if speed set does not take effect """ expected_value = int(params.get("migrate_speed", '41943040')) // (1024 * 1024) actual_value = control_migrate_speed(to_speed=expected_value) params.update({'compare_to_value': actual_value}) if actual_value != expected_value: test.fail("Migration speed is expected to be '%d MiB/s', but '%d MiB/s' " "found" % (expected_value, actual_value)) def check_domjobinfo(params, option=""): """ Check given item in domjobinfo of the guest is as expected :param params: the parameters used :param option: options for domjobinfo :raise: test.fail if the value of given item is unexpected """ def search_jobinfo(jobinfo): """ Find value of given item in domjobinfo :param jobinfo: cmdResult object :raise: test.fail if not found """ for item in jobinfo.stdout.splitlines(): if item.count(jobinfo_item): groups = re.findall(r'[0-9.]+', item.strip()) logging.debug("In '%s' search '%s'\n", item, groups[0]) if (math.fabs(float(groups[0]) - float(compare_to_value)) // float(compare_to_value) > diff_rate): test.fail("{} {} has too much difference from " "{}".format(jobinfo_item, groups[0], compare_to_value)) break jobinfo_item = params.get("jobinfo_item") compare_to_value = params.get("compare_to_value") logging.debug("compare_to_value:%s", compare_to_value) diff_rate = float(params.get("diff_rate", "0")) if not jobinfo_item or not compare_to_value: return vm_ref = '{}{}'.format(vm_name, option) jobinfo = virsh.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) check_domjobinfo_remote = params.get("check_domjobinfo_remote") if check_domjobinfo_remote: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) jobinfo = remote_virsh_session.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) remote_virsh_session.close_session() def check_maxdowntime(params): """ Set/get migration maxdowntime :param params: the parameters used :raise: test.fail if maxdowntime set does not take effect """ expected_value = int(float(params.get("migrate_maxdowntime", '0.3')) * 1000) virsh_args.update({"ignore_status": False}) old_value = int(virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("Current migration maxdowntime is %d ms", old_value) logging.debug("Set migration maxdowntime to %d ms", expected_value) virsh.migrate_setmaxdowntime(vm_name, expected_value, **virsh_args) actual_value = int(virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("New migration maxdowntime is %d ms", actual_value) if actual_value != expected_value: test.fail("Migration maxdowntime is expected to be '%d ms', but '%d ms' " "found" % (expected_value, actual_value)) params.update({'compare_to_value': actual_value}) def do_actions_during_migrate(params): """ The entry point to execute action list during migration :param params: the parameters used """ actions_during_migration = params.get("actions_during_migration") if not actions_during_migration: return for action in actions_during_migration.split(","): if action == 'setspeed': check_setspeed(params) elif action == 'domjobinfo': check_domjobinfo(params) elif action == 'setmaxdowntime': check_maxdowntime(params) time.sleep(3) def attach_channel_xml(): """ Create channel xml and attach it to guest configuration """ # Check if pty channel exists already for elem in new_xml.devices.by_device_tag('channel'): if elem.type_name == channel_type_name: logging.debug("{0} channel already exists in guest. " "No need to add new one".format(channel_type_name)) return params = {'channel_type_name': channel_type_name, 'target_type': target_type, 'target_name': target_name} channel_xml = libvirt.create_channel_xml(params) virsh.attach_device(domain_opt=vm_name, file_opt=channel_xml.xml, flagstr="--config", ignore_status=False) logging.debug("New VMXML with channel:\n%s", virsh.dumpxml(vm_name)) def check_timeout_postcopy(params): """ Check the vm state on target host after timeout when --postcopy and --timeout-postcopy are used. The vm state is expected as running. :param params: the parameters used """ timeout = int(params.get("timeout_postcopy", 10)) time.sleep(timeout + 1) remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) vm_state = results_stdout_52lts(remote_virsh_session.domstate(vm_name)).strip() if vm_state != "running": remote_virsh_session.close_session() test.fail("After timeout '%s' seconds, " "the vm state on target host should " "be 'running', but '%s' found", timeout, vm_state) remote_virsh_session.close_session() def get_usable_compress_cache(pagesize): """ Get a number which is bigger than pagesize and is power of two. :param pagesize: the given integer :return: an integer satisfying the criteria """ def calculate(num): result = num & (num - 1) return (result == 0) item = pagesize found = False while (not found): item += 1 found = calculate(item) logging.debug("%d is smallest one that is bigger than '%s' and " "is power of 2", item, pagesize) return item def check_migration_res(result): """ Check if the migration result is as expected :param result: the output of migration :raise: test.fail if test is failed """ logging.info("Migration out: %s", results_stdout_52lts(result).strip()) logging.info("Migration error: %s", results_stderr_52lts(result).strip()) if status_error: # Migration should fail if err_msg: # Special error messages are expected if not re.search(err_msg, results_stderr_52lts(result).strip()): test.fail("Can not find the expected patterns '%s' in " "output '%s'" % (err_msg, results_stderr_52lts(result).strip())) else: logging.debug("It is the expected error message") else: if int(result.exit_status) != 0: logging.debug("Migration failure is expected result") else: test.fail("Migration success is unexpected result") else: if int(result.exit_status) != 0: test.fail(results_stderr_52lts(result).strip()) check_parameters(test, params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} virsh_opt = params.get("virsh_opt", "") server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") log_file = params.get("libvirt_log", "/var/log/libvirt/libvirtd.log") check_complete_job = "yes" == params.get("check_complete_job", "no") config_libvirtd = "yes" == params.get("config_libvirtd", "no") contrl_index = params.get("new_contrl_index", None) asynch_migration = "yes" == params.get("asynch_migrate", "no") grep_str_remote_log = params.get("grep_str_remote_log", "") grep_str_local_log = params.get("grep_str_local_log", "") disable_verify_peer = "yes" == params.get("disable_verify_peer", "no") status_error = "yes" == params.get("status_error", "no") stress_in_vm = "yes" == params.get("stress_in_vm", "no") low_speed = params.get("low_speed", None) remote_virsh_dargs = {'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True} hpt_resize = params.get("hpt_resize", None) htm_state = params.get("htm_state", None) # For pty channel test add_channel = "yes" == params.get("add_channel", "no") channel_type_name = params.get("channel_type_name", None) target_type = params.get("target_type", None) target_name = params.get("target_name", None) cmd_run_in_remote_guest = params.get("cmd_run_in_remote_guest", None) cmd_run_in_remote_guest_1 = params.get("cmd_run_in_remote_guest_1", None) cmd_run_in_remote_host = params.get("cmd_run_in_remote_host", None) cmd_run_in_remote_host_1 = params.get("cmd_run_in_remote_host_1", None) cmd_run_in_remote_host_2 = params.get("cmd_run_in_remote_host_2", None) # For qemu command line checking qemu_check = params.get("qemu_check", None) xml_check_after_mig = params.get("guest_xml_check_after_mig", None) # params for cache matrix test cache = params.get("cache") remove_cache = "yes" == params.get("remove_cache", "no") err_msg = params.get("err_msg") arch = platform.machine() if any([hpt_resize, contrl_index, htm_state]) and 'ppc64' not in arch: test.cancel("The case is PPC only.") # For TLS tls_recovery = params.get("tls_auto_recovery", "yes") # qemu config qemu_conf_dict = None # libvirtd config libvirtd_conf_dict = None remote_virsh_session = None vm = None vm_session = None libvirtd_conf = None qemu_conf = None mig_result = None test_exception = None is_TestError = False is_TestFail = False is_TestSkip = False # Objects to be cleaned up in the end objs_list = [] tls_obj = None # Local variables vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() if not orig_config_xml: test.error("Backing up xmlfile failed.") try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Change the configuration files if needed before starting guest # For qemu.conf if extra.count("--tls"): # Setup TLS tls_obj = TLSConnection(params) if tls_recovery == "yes": objs_list.append(tls_obj) tls_obj.auto_recover = True tls_obj.conn_setup() if not disable_verify_peer: qemu_conf_dict = {"migrate_tls_x509_verify": "1"} # Setup qemu configure logging.debug("Configure the qemu") cleanup_libvirtd_log(log_file) qemu_conf = libvirt.customize_libvirt_config(qemu_conf_dict, config_type="qemu", remote_host=True, extra_params=params) # Setup libvirtd if config_libvirtd: logging.debug("Configure the libvirtd") cleanup_libvirtd_log(log_file) libvirtd_conf_dict = setup_libvirtd_conf_dict(params) libvirtd_conf = libvirt.customize_libvirt_config(libvirtd_conf_dict, remote_host=True, extra_params=params) # Prepare required guest xml before starting guest if contrl_index: new_xml.remove_all_device_by_type('controller') logging.debug("After removing controllers, current XML:\n%s\n", new_xml) add_ctrls(new_xml, dev_index=contrl_index) if add_channel: attach_channel_xml() if hpt_resize: set_feature(new_xml, 'hpt', hpt_resize) if htm_state: set_feature(new_xml, 'htm', htm_state) if cache: params["driver_cache"] = cache if remove_cache: params["enable_cache"] = "no" # Change the disk of the vm to shared disk and then start VM libvirt.set_vm_disk(vm, params) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check qemu command line after guest is started if qemu_check: check_content = qemu_check if hpt_resize: check_content = "%s%s" % (qemu_check, hpt_resize) if htm_state: check_content = "%s%s" % (qemu_check, htm_state) libvirt.check_qemu_cmd_line(check_content) # Check local guest network connection before migration vm_session = vm.wait_for_login() check_vm_network_accessed() # Preparation for the running guest before migration if hpt_resize and hpt_resize != 'disabled': trigger_hpt_resize(vm_session) if low_speed: control_migrate_speed(int(low_speed)) if stress_in_vm: pkg_name = 'stress' logging.debug("Check if stress tool is installed") pkg_mgr = utils_package.package_manager(vm_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("Stress tool will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) stress_thread = threading.Thread(target=run_stress_in_vm, args=()) stress_thread.start() if extra.count("timeout-postcopy"): func_name = check_timeout_postcopy if params.get("actions_during_migration"): func_name = do_actions_during_migrate if extra.count("comp-xbzrle-cache"): cache = get_usable_compress_cache(memory.get_page_size()) extra = "%s %s" % (extra, cache) # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (extra, postcopy_options) # Execute migration process if not asynch_migration: mig_result = do_migration(vm, dest_uri, options, extra) else: migration_test = libvirt.MigrationTest() logging.debug("vm.connect_uri=%s", vm.connect_uri) vms = [vm] try: migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_opt, func=func_name, extra_opts=extra, func_params=params) mig_result = migration_test.ret except exceptions.TestFail as fail_detail: test.fail(fail_detail) except exceptions.TestSkipError as skip_detail: test.cancel(skip_detail) except exceptions.TestError as error_detail: test.error(error_detail) except Exception as details: mig_result = migration_test.ret logging.error(details) check_migration_res(mig_result) if add_channel: # Get the channel device source path of remote guest if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) file_path = tempfile.mktemp(dir=data_dir.get_tmp_dir()) remote_virsh_session.dumpxml(vm_name, to_file=file_path, debug=True, ignore_status=True) local_vmxml = vm_xml.VMXML.new_from_dumpxml(vm_name) local_vmxml.xmltreefile = xml_utils.XMLTreeFile(file_path) for elem in local_vmxml.devices.by_device_tag('channel'): logging.debug("Found channel device {}".format(elem)) if elem.type_name == channel_type_name: host_source = elem.source.get('path') logging.debug("Remote guest uses {} for channel device".format(host_source)) break remote_virsh_session.close_session() if not host_source: test.fail("Can not find source for %s channel on remote host" % channel_type_name) # Prepare to wait for message on remote host from the channel cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} cmd_result = remote.run_remote_cmd(cmd_run_in_remote_host % host_source, cmd_parms, runner_on_target) # Send message from remote guest to the channel file remote_vm_obj = utils_test.RemoteVMManager(cmd_parms) vm_ip = vm.get_address() vm_pwd = params.get("password") remote_vm_obj.setup_ssh_auth(vm_ip, vm_pwd) cmd_result = remote_vm_obj.run_command(vm_ip, cmd_run_in_remote_guest_1) remote_vm_obj.run_command(vm_ip, cmd_run_in_remote_guest % results_stdout_52lts(cmd_result).strip()) logging.debug("Sending message is done") # Check message on remote host from the channel remote.run_remote_cmd(cmd_run_in_remote_host_1, cmd_parms, runner_on_target) logging.debug("Receiving message is done") remote.run_remote_cmd(cmd_run_in_remote_host_2, cmd_parms, runner_on_target) if check_complete_job: opts = " --completed" check_virsh_command_and_option("domjobinfo", opts) if extra.count("comp-xbzrle-cache"): params.update({'compare_to_value': cache // 1024}) check_domjobinfo(params, option=opts) if grep_str_local_log: cmd = "grep -E '%s' %s" % (grep_str_local_log, log_file) cmdRes = process.run(cmd, shell=True, ignore_status=True) if cmdRes.exit_status: test.fail(results_stderr_52lts(cmdRes).strip()) if grep_str_remote_log: cmd = "grep -E '%s' %s" % (grep_str_remote_log, log_file) cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if xml_check_after_mig: if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) target_guest_dumpxml = results_stdout_52lts( remote_virsh_session.dumpxml(vm_name, debug=True, ignore_status=True)).strip() if hpt_resize: check_str = hpt_resize elif htm_state: check_str = htm_state if hpt_resize or htm_state: xml_check_after_mig = "%s'%s'" % (xml_check_after_mig, check_str) if not re.search(xml_check_after_mig, target_guest_dumpxml): remote_virsh_session.close_session() test.fail("Fail to search '%s' in target guest XML:\n%s" % (xml_check_after_mig, target_guest_dumpxml)) if contrl_index: all_ctrls = re.findall(xml_check_after_mig, target_guest_dumpxml) if len(all_ctrls) != int(contrl_index) + 1: remote_virsh_session.close_session() test.fail("%s pci-root controllers are expected in guest XML, " "but found %s" % (int(contrl_index) + 1, len(all_ctrls))) remote_virsh_session.close_session() if int(mig_result.exit_status) == 0: server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") check_vm_network_accessed(server_session) server_session.close() except exceptions.TestFail as details: is_TestFail = True test_exception = details except exceptions.TestSkipError as details: is_TestSkip = True test_exception = details except exceptions.TestError as details: is_TestError = True test_exception = details except Exception as details: test_exception = details finally: logging.debug("Recover test environment") try: # Clean VM on destination vm.connect_uri = dest_uri cleanup_dest(vm) vm.connect_uri = src_uri logging.info("Recovery VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) if remote_virsh_session: remote_virsh_session.close_session() if extra.count("--tls") and not disable_verify_peer: logging.debug("Recover the qemu configuration") libvirt.customize_libvirt_config(None, config_type="qemu", remote_host=True, extra_params=params, is_recover=True, config_object=qemu_conf) if config_libvirtd: logging.debug("Recover the libvirtd configuration") libvirt.customize_libvirt_config(None, remote_host=True, extra_params=params, is_recover=True, config_object=libvirtd_conf) logging.info("Remove local NFS image") source_file = params.get("source_file") libvirt.delete_local_disk("file", path=source_file) if objs_list: for obj in objs_list: logging.debug("Clean up local objs") del obj except Exception as exception_detail: if (not test_exception and not is_TestError and not is_TestFail and not is_TestSkip): raise exception_detail else: # if any of above exceptions has been raised, only print # error log here to avoid of hiding the original issue logging.error(exception_detail) # Check result if is_TestFail: test.fail(test_exception) if is_TestSkip: test.cancel(test_exception) if is_TestError: test.error(test_exception) if not test_exception: logging.info("Case execution is done.") else: test.error(test_exception)