def create_or_del_network(net_dict, is_del=False, remote_args=None): """ Create or delete network on local or remote :param net_dict: Dictionary with the network parameters :param is_del: Whether the networks should be deleted :param remote_args: The parameters for remote """ remote_virsh_session = None if remote_args: remote_virsh_session = virsh.VirshPersistent(**remote_args) if not is_del: net_dev = libvirt.create_net_xml(net_dict.get("net_name"), net_dict) if not remote_virsh_session: if net_dev.get_active(): net_dev.undefine() net_dev.define() net_dev.start() else: remote_ip = remote_args.get("remote_ip") remote_user = remote_args.get("remote_user") remote_pwd = remote_args.get("remote_pwd") if not all([remote_ip, remote_user, remote_pwd]): raise exceptions.TestError( "remote_[ip|user|pwd] are necessary!") remote.scp_to_remote(remote_ip, '22', remote_user, remote_pwd, net_dev.xml, net_dev.xml, limit="", log_filename=None, timeout=600, interface=None) remote_virsh_session.net_define(net_dev.xml, debug=True) remote_virsh_session.net_start(net_dict.get("net_name"), debug=True) remote.run_remote_cmd("rm -rf %s" % net_dev.xml, remote_args) else: virsh_session = virsh if remote_virsh_session: virsh_session = remote_virsh_session if net_dict.get("net_name") in virsh_session.net_state_dict(): virsh_session.net_destroy(net_dict.get("net_name"), debug=True, ignore_status=True) virsh_session.net_undefine(net_dict.get("net_name"), debug=True, ignore_status=True) if remote_virsh_session: remote_virsh_session.close_session()
def cleanup_libvirtd_log(log_file): """ Remove existing libvirtd log file on source and target host. :param log_file: log file with absolute path """ if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) cmd = "rm -f %s" % log_file logging.debug("Delete remote libvirt log file '%s'", log_file) remote.run_remote_cmd(cmd, cmd_parms, runner_on_target)
def cleanup_libvirtd_log(log_file): """ Remove existing libvirtd log file on source and target host. :param log_file: log file with absolute path """ if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) cmd = "rm -f %s" % log_file logging.debug("Delete remote libvirt log file '%s'", log_file) cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} remote.run_remote_cmd(cmd, cmd_parms, runner_on_target)
def run_migration_back(params, test): """ Execute migration back from target host to source host :param params: dict, test parameters :param test: test object """ migrate_vm_back = "yes" == params.get("migrate_vm_back", "no") vm_name = params.get("migrate_main_vm") options = params.get("virsh_migrate_options", "--live --verbose") if migrate_vm_back: ssh_connection = utils_conn.SSHConnection( server_ip=params.get("client_ip"), server_pwd=params.get("local_pwd"), client_ip=params.get("server_ip"), client_pwd=params.get("server_pwd")) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine src_full_uri = libvirt_vm.complete_uri( params.get("migrate_source_host")) migration_test = migration.MigrationTest() migration_test.migrate_pre_setup(src_full_uri, params) cmd = "virsh migrate %s %s %s" % (vm_name, options, src_full_uri) test.log.debug("Start migration: %s", cmd) runner_on_target = remote.RemoteRunner( host=params.get("remote_ip"), username=params.get("remote_user"), password=params.get("remote_pwd")) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) test.log.info(cmd_result) if cmd_result.exit_status: destroy_cmd = "virsh destroy %s" % vm_name remote.run_remote_cmd(destroy_cmd, params, runner_on_target, ignore_status=False) test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) else: test.log.debug("No need to migrate back")
def run_domstats(vm_name): """ Run domstats and domstate during migration in source and destination :param vm_name: VM name :raise: test.fail if domstats does not return in 2s or domstate is incorrect """ @run_time() def check_source_stats(vm_name): """domstats in source""" vm_stats = virsh.domstats(vm_name) logging.debug("domstats in source: {}".format(vm_stats)) @run_time() def check_dest_stats(vm_name): """domstats in target""" cmd = "virsh domstats {}".format(vm_name) dest_stats = remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) logging.debug("domstats in destination: {}".format(dest_stats)) expected_remote_state = "paused" expected_source_state = ["paused", "running"] if postcopy_options: set_migratepostcopy() expected_remote_state = "running" expected_source_state = ["paused"] check_source_stats(vm_name) vm_stat = virsh.domstate(vm_name, ignore_status=False) if ((not len(vm_stat.stdout.split())) or vm_stat.stdout.split()[0] not in expected_source_state): test.fail("Incorrect VM stat on source machine: {}" .format(vm_stat.stdout)) check_dest_stats(vm_name) cmd = "virsh domstate {}".format(vm_name) remote_vm_state = remote.run_remote_cmd(cmd, cmd_parms, runner_on_target, ignore_status=False) if ((not len(remote_vm_state.stdout.split())) or remote_vm_state.stdout.split()[0] != expected_remote_state): test.fail("Incorrect VM stat on destination machine: {}" .format(remote_vm_state.stdout)) else: logging.debug("vm stat on destination: {}".format(remote_vm_state)) if postcopy_options: vm_stat = virsh.domstate(vm_name, ignore_status=False) if ((not len(vm_stat.stdout.split())) or vm_stat.stdout.split()[0] != "paused"): test.fail("Incorrect VM stat on source machine: {}" .format(vm_stat.stdout))
def setup_loop_dev(block_device, remote_params=None): """ Setup a loop device for test :param block_device: block device name :param remote_params: Dict of parameters, which should include: server_ip, server_user, server_pwd """ loop_dev = None # Find a free loop device cmd = "losetup --find" if remote_params: loop_dev = remote_old.run_remote_cmd( cmd, remote_params, ignore_status=False).stdout_text.strip() else: loop_dev = process.run(cmd, shell=True).stdout_text.strip() logging.debug("loop dev: %s", loop_dev) # Setup a loop device cmd = 'losetup %s %s' % (loop_dev, block_device) if remote_params: remote_old.run_remote_cmd(cmd, remote_params, ignore_status=False) else: process.run(cmd, shell=True) return loop_dev
def set_hostname(hostname, test, remote_params=None): """ Set hostname for source or dest host :param hostname: string, hostname :param test: test object :param remote_params: Dict of remote host parameters, which should include: server_ip, server_user, server_pwd """ cmd = "hostnamectl set-hostname %s" % hostname if remote_params: ret = remote_old.run_remote_cmd(cmd, remote_params, ignore_status=False) else: ret = process.run(cmd, ignore_status=False, shell=True) output = ret.stdout_text.strip() if ret.exit_status: test.fail("Failed to run '%s': %s" % (cmd, output)) logging.info("Set hostname: %s" % hostname)
def check_established(params): """ Parses netstat output for established connection on remote or local :param params: the parameters used :return: str, the port used :raises: exceptions.TestFail if no match """ port_to_check = params.get("port_to_check", "4915") check_local = 'yes' == params.get("check_local_port", "no") ipv6_config = "yes" == params.get("ipv6_config", "no") if ipv6_config: server_ip = params.get("ipv6_addr_des", "")[:17] else: server_ip = params.get("server_ip", params.get("remote_ip")) cmd = "netstat -tunap|grep %s" % port_to_check if check_local: cmdRes = process.run(cmd, shell=True) else: cmdRes = remote.run_remote_cmd(cmd, params) if port_to_check != '4915': pat_str = r'.*%s:%s.*ESTABLISHED.*qemu-kvm.*' % (server_ip, port_to_check) search = re.search(pat_str, cmdRes.stdout_text.strip()) if not search: raise exceptions.TestFail("Pattern '%s' is not matched in " "'%s'" % (pat_str, cmdRes.stdout_text.strip())) else: return port_to_check else: pat_str = r'.*%s:(\d*).*ESTABLISHED.*qemu-kvm.*' % server_ip search = re.search(pat_str, cmdRes.stdout_text.strip()) if search: logging.debug("Get the port used:%s", search.group(1)) return search.group(1) else: raise exceptions.TestFail("Pattern '%s' is not matched in " "'%s'" % (pat_str, cmdRes.stdout_text.strip()))
def check_established(params): """ Parses netstat output for established connection on remote :param params: the parameters used """ port_to_check = params.get("port_to_check", "4915") ipv6_config = "yes" == params.get("ipv6_config", "no") if ipv6_config: server_ip = params.get("ipv6_addr_des", "")[:17] client_ip = params.get("ipv6_addr_src", "")[:17] else: server_ip = params.get("server_ip") client_ip = params.get("client_ip") cmd = "netstat -tunap|grep %s" % port_to_check exp_msg = r".*%s:%s.*%s.*ESTABLISHED.*qemu-kvm.*" % ( server_ip, port_to_check, client_ip) cmdRes = remote.run_remote_cmd(cmd, params) libvirt.check_result(cmdRes, expected_match=exp_msg)
def run(test, params, env): """ Test virsh migrate command. """ def set_feature(vmxml, feature, value): """ Set guest features for PPC :param state: the htm status :param vmxml: guest xml """ features_xml = vm_xml.VMFeaturesXML() if feature == 'hpt': hpt_xml = vm_xml.VMFeaturesHptXML() hpt_xml.resizing = value features_xml.hpt = hpt_xml elif feature == 'htm': features_xml.htm = value vmxml.features = features_xml vmxml.sync() def trigger_hpt_resize(session): """ Check the HPT order file and dmesg :param session: the session to guest :raise: test.fail if required message is not found """ hpt_order_path = "/sys/kernel/debug/powerpc/hpt_order" hpt_order = session.cmd_output('cat %s' % hpt_order_path).strip() hpt_order = int(hpt_order) logging.info('Current hpt_order is %d', hpt_order) hpt_order += 1 cmd = 'echo %d > %s' % (hpt_order, hpt_order_path) cmd_result = session.cmd_status_output(cmd) result = process.CmdResult(stderr=cmd_result[1], stdout=cmd_result[1], exit_status=cmd_result[0]) libvirt.check_exit_status(result) dmesg = session.cmd('dmesg') dmesg_content = params.get('dmesg_content').split('|') for content in dmesg_content: if content % hpt_order not in dmesg: test.fail("'%s' is missing in dmesg" % (content % hpt_order)) else: logging.info("'%s' is found in dmesg", content % hpt_order) def check_vm_network_accessed(session=None): """ The operations to the VM need to be done before or after migration happens :param session: The session object to the host :raise: test.error when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") s_ping, _ = utils_test.ping(vm.get_address(), count=10, timeout=20, output_func=logging.debug, session=session) if s_ping != 0: if session: session.close() test.fail("%s did not respond after %d sec." % (vm.name, 20)) def check_virsh_command_and_option(command, option=None): """ Check if virsh command exists :param command: the command to be checked :param option: the command option to be checked """ msg = "This version of libvirt does not support " if not virsh.has_help_command(command): test.cancel(msg + "virsh command '%s'" % command) if option and not virsh.has_command_help_match(command, option): test.cancel(msg + "virsh command '%s' with option '%s'" % (command, option)) def add_ctrls(vm_xml, dev_type="pci", dev_index="0", dev_model="pci-root"): """ Add multiple devices :param dev_type: the type of the device to be added :param dev_index: the maximum index of the device to be added :param dev_model: the model of the device to be added """ for inx in range(0, int(dev_index) + 1): newcontroller = Controller("controller") newcontroller.type = dev_type newcontroller.index = inx newcontroller.model = dev_model logging.debug("New device is added:\n%s", newcontroller) vm_xml.add_device(newcontroller) vm_xml.sync() def do_migration(vm, dest_uri, options, extra): """ Execute the migration with given parameters :param vm: the guest to be migrated :param dest_uri: the destination uri for migration :param options: options next to 'migrate' command :param extra: options in the end of the migrate command line :return: CmdResult object """ logging.info("Sleeping 10 seconds before migration") time.sleep(10) # Migrate the guest. virsh_args.update({"ignore_status": True}) migration_res = vm.migrate(dest_uri, options, extra, **virsh_args) if int(migration_res.exit_status) != 0: logging.error("Migration failed for %s.", vm_name) return migration_res if vm.is_alive(): # vm.connect_uri was updated logging.info("VM is alive on destination %s.", dest_uri) else: test.fail("VM is not alive on destination %s" % dest_uri) # Throws exception if console shows panic message vm.verify_kernel_crash() return migration_res def cleanup_libvirtd_log(log_file): """ Remove existing libvirtd log file on source and target host. :param log_file: log file with absolute path """ if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) cmd = "rm -f %s" % log_file logging.debug("Delete remote libvirt log file '%s'", log_file) remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) def cleanup_dest(vm): """ Clean up the destination host environment when doing the uni-direction migration. :param vm: the guest to be cleaned up """ logging.info("Cleaning up VMs on %s", vm.connect_uri) try: if virsh.domain_exists(vm.name, uri=vm.connect_uri): vm_state = vm.state() if vm_state == "paused": vm.resume() elif vm_state == "shut off": vm.start() vm.destroy(gracefully=False) if vm.is_persistent(): vm.undefine() except Exception as detail: logging.error("Cleaning up destination failed.\n%s", detail) def run_stress_in_vm(): """ The function to load stress in VM """ stress_args = params.get("stress_args", "--cpu 8 --io 4 " "--vm 2 --vm-bytes 128M " "--timeout 20s") try: vm_session.cmd('stress %s' % stress_args) except Exception as detail: logging.debug(detail) def control_migrate_speed(to_speed=1, opts=""): """ Control migration duration :param to_speed: the speed value in Mbps to be set for migration :return int: the new migration speed after setting """ virsh_args.update({"ignore_status": False}) old_speed = virsh.migrate_getspeed(vm_name, extra=opts, **virsh_args) logging.debug("Current migration speed is %s MiB/s\n", old_speed.stdout.strip()) logging.debug("Set migration speed to %d MiB/s\n", to_speed) cmd_result = virsh.migrate_setspeed(vm_name, to_speed, extra=opts, **virsh_args) actual_speed = virsh.migrate_getspeed(vm_name, extra=opts, **virsh_args) logging.debug("New migration speed is %s MiB/s\n", actual_speed.stdout.strip()) return int(actual_speed.stdout.strip()) def check_setspeed(params): """ Set/get migration speed :param params: the parameters used :raise: test.fail if speed set does not take effect """ expected_value = int(params.get("migrate_speed", '41943040')) // (1024 * 1024) actual_value = control_migrate_speed(to_speed=expected_value) params.update({'compare_to_value': actual_value}) if actual_value != expected_value: test.fail("Migration speed is expected to be '%d MiB/s', but '%d MiB/s' " "found" % (expected_value, actual_value)) def check_domjobinfo(params, option=""): """ Check given item in domjobinfo of the guest is as expected :param params: the parameters used :param option: options for domjobinfo :raise: test.fail if the value of given item is unexpected """ def search_jobinfo(jobinfo): """ Find value of given item in domjobinfo :param jobinfo: cmdResult object :raise: test.fail if not found """ for item in jobinfo.stdout.splitlines(): if item.count(jobinfo_item): groups = re.findall(r'[0-9.]+', item.strip()) logging.debug("In '%s' search '%s'\n", item, groups[0]) if (math.fabs(float(groups[0]) - float(compare_to_value)) // float(compare_to_value) > diff_rate): test.fail("{} {} has too much difference from " "{}".format(jobinfo_item, groups[0], compare_to_value)) break jobinfo_item = params.get("jobinfo_item") compare_to_value = params.get("compare_to_value") logging.debug("compare_to_value:%s", compare_to_value) diff_rate = float(params.get("diff_rate", "0")) if not jobinfo_item or not compare_to_value: return vm_ref = '{}{}'.format(vm_name, option) jobinfo = virsh.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) check_domjobinfo_remote = params.get("check_domjobinfo_remote") if check_domjobinfo_remote: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) jobinfo = remote_virsh_session.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) remote_virsh_session.close_session() def search_jobinfo_output(jobinfo, items_to_check, postcopy_req=False): """ Check the results of domjobinfo :param jobinfo: cmdResult object :param items_to_check: expected value :param postcopy_req: True for postcopy migration and False for precopy :return: False if not found """ expected_value = copy.deepcopy(items_to_check) logging.debug("The items_to_check is %s", expected_value) for item in jobinfo.splitlines(): item_key = item.strip().split(':')[0] if "all_items" in expected_value and len(item_key) > 0: # "Time elapsed w/o network" and "Downtime w/o network" # have a chance to be missing, it is normal if item_key in ['Downtime w/o network', 'Time elapsed w/o network']: continue if expected_value["all_items"][0] == item_key: del expected_value["all_items"][0] else: test.fail("The item '%s' should be '%s'" % (item_key, expected_value["all_items"][0])) if item_key in expected_value: item_value = ':'.join(item.strip().split(':')[1:]).strip() if item_value != expected_value.get(item_key): test.fail("The value of {} is {} which is not " "expected".format(item_key, item_value)) else: del expected_value[item_key] if postcopy_req and item_key == "Postcopy requests": if int(item.strip().split(':')[1]) <= 0: test.fail("The value of Postcopy requests is incorrect") # Check if all the items in expect_dict checked or not if "all_items" in expected_value: if len(expected_value["all_items"]) > 0: test.fail("Missing item: {} from all_items" .format(expected_value["all_items"])) else: del expected_value["all_items"] if len(expected_value) != 0: test.fail("Missing item: {}".format(expected_value)) def set_migratepostcopy(): """ Switch to postcopy during migration """ res = virsh.migrate_postcopy(vm_name) logging.debug("Command output: %s", res) if not utils_misc.wait_for( lambda: migration_test.check_vm_state(vm_name, "paused", "post-copy"), 10): test.fail("vm status is expected to 'paused (post-copy)'") def check_domjobinfo_output(option="", is_mig_compelete=False): """ Check all items in domjobinfo of the guest on both remote and local :param option: options for domjobinfo :param is_mig_compelete: False for domjobinfo checking during migration, True for domjobinfo checking after migration :raise: test.fail if the value of given item is unexpected """ expected_list_during_mig = ["Job type", "Operation", "Time elapsed", "Data processed", "Data remaining", "Data total", "Memory processed", "Memory remaining", "Memory total", "Memory bandwidth", "Dirty rate", "Page size", "Iteration", "Constant pages", "Normal pages", "Normal data", "Expected downtime", "Setup time"] if libvirt_version.version_compare(4, 10, 0): expected_list_during_mig.insert(13, "Postcopy requests") expected_list_after_mig_src = copy.deepcopy(expected_list_during_mig) expected_list_after_mig_src[-2] = 'Total downtime' expected_list_after_mig_dest = copy.deepcopy(expected_list_after_mig_src) # Check version in remote if not expected_list_after_mig_dest.count("Postcopy requests"): remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, "#") if libvirt_version.version_compare(4, 10, 0, session=remote_session): expected_list_after_mig_dest.insert(14, "Postcopy requests") remote_session.close() expect_dict = {"src_notdone": {"Job type": "Unbounded", "Operation": "Outgoing migration", "all_items": expected_list_during_mig}, "dest_notdone": {"error": "Operation not supported: mig" "ration statistics are availab" "le only on the source host"}, "src_done": {"Job type": "Completed", "Operation": "Outgoing migration", "all_items": expected_list_after_mig_src}, "dest_done": {"Job type": "Completed", "Operation": "Incoming migration", "all_items": expected_list_after_mig_dest}} pc_opt = False if postcopy_options: pc_opt = True if is_mig_compelete: expect_dict["dest_done"].clear() expect_dict["dest_done"]["Job type"] = "None" else: set_migratepostcopy() vm_ref = '{}{}'.format(vm_name, option) src_jobinfo = virsh.domjobinfo(vm_ref, **virsh_args) cmd = "virsh domjobinfo {} {}".format(vm_name, option) dest_jobinfo = remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if not is_mig_compelete: search_jobinfo_output(src_jobinfo.stdout, expect_dict["src_notdone"], postcopy_req=pc_opt) search_jobinfo_output(dest_jobinfo.stderr, expect_dict["dest_notdone"]) else: search_jobinfo_output(src_jobinfo.stdout, expect_dict["src_done"]) search_jobinfo_output(dest_jobinfo.stdout, expect_dict["dest_done"]) def check_maxdowntime(params): """ Set/get migration maxdowntime :param params: the parameters used :raise: test.fail if maxdowntime set does not take effect """ expected_value = int(float(params.get("migrate_maxdowntime", '0.3')) * 1000) virsh_args.update({"ignore_status": False}) old_value = int(virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("Current migration maxdowntime is %d ms", old_value) logging.debug("Set migration maxdowntime to %d ms", expected_value) virsh.migrate_setmaxdowntime(vm_name, expected_value, **virsh_args) actual_value = int(virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("New migration maxdowntime is %d ms", actual_value) if actual_value != expected_value: test.fail("Migration maxdowntime is expected to be '%d ms', but '%d ms' " "found" % (expected_value, actual_value)) params.update({'compare_to_value': actual_value}) def run_time(init_time=2): """ Compare the duration of func to an expected one :param init_time: Expected run time :raise: test.fail if func takes more than init_time(second) """ def check_time(func): def wrapper(*args, **kwargs): start = time.time() result = func(*args, **kwargs) duration = time.time() - start if duration > init_time: test.fail("It takes too long to check {}. The duration is " "{}s which should be less than {}s" .format(func.__doc__, duration, init_time)) return result return wrapper return check_time def run_domstats(vm_name): """ Run domstats and domstate during migration in source and destination :param vm_name: VM name :raise: test.fail if domstats does not return in 2s or domstate is incorrect """ @run_time() def check_source_stats(vm_name): """domstats in source""" vm_stats = virsh.domstats(vm_name) logging.debug("domstats in source: {}".format(vm_stats)) @run_time() def check_dest_stats(vm_name): """domstats in target""" cmd = "virsh domstats {}".format(vm_name) dest_stats = remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) logging.debug("domstats in destination: {}".format(dest_stats)) expected_remote_state = "paused" expected_source_state = ["paused", "running"] if postcopy_options: set_migratepostcopy() expected_remote_state = "running" expected_source_state = ["paused"] check_source_stats(vm_name) vm_stat = virsh.domstate(vm_name, ignore_status=False) if ((not len(vm_stat.stdout.split())) or vm_stat.stdout.split()[0] not in expected_source_state): test.fail("Incorrect VM stat on source machine: {}" .format(vm_stat.stdout)) check_dest_stats(vm_name) cmd = "virsh domstate {}".format(vm_name) remote_vm_state = remote.run_remote_cmd(cmd, cmd_parms, runner_on_target, ignore_status=False) if ((not len(remote_vm_state.stdout.split())) or remote_vm_state.stdout.split()[0] != expected_remote_state): test.fail("Incorrect VM stat on destination machine: {}" .format(remote_vm_state.stdout)) else: logging.debug("vm stat on destination: {}".format(remote_vm_state)) if postcopy_options: vm_stat = virsh.domstate(vm_name, ignore_status=False) if ((not len(vm_stat.stdout.split())) or vm_stat.stdout.split()[0] != "paused"): test.fail("Incorrect VM stat on source machine: {}" .format(vm_stat.stdout)) def kill_qemu_target(): """ Kill qemu process on target host during Finish Phase of migration :raise: test.fail if domstate is not "post-copy failed" after qemu killed """ if not vm.is_qemu(): test.cancel("This case is qemu guest only.") set_migratepostcopy() emulator = new_xml.get_devices('emulator')[0] logging.debug("emulator is %s", emulator.path) cmd = 'pkill -9 {}'.format(os.path.basename(emulator.path)) runner_on_target.run(cmd) if not utils_misc.wait_for( lambda: migration_test.check_vm_state(vm_name, "paused", "post-copy failed"), 60): test.fail("vm status is expected to 'paused (post-copy failed)'") def do_actions_during_migrate(params): """ The entry point to execute action list during migration :param params: the parameters used """ actions_during_migration = params.get("actions_during_migration") if not actions_during_migration: return for action in actions_during_migration.split(","): if action == 'setspeed': check_setspeed(params) elif action == 'domjobinfo': check_domjobinfo(params) elif action == 'setmaxdowntime': check_maxdowntime(params) elif action == 'converge': check_converge(params) elif action == 'domjobinfo_output_all': check_domjobinfo_output() elif action == 'checkdomstats': run_domstats(vm_name) elif action == 'killqemutarget': kill_qemu_target() time.sleep(3) def attach_channel_xml(): """ Create channel xml and attach it to guest configuration """ # Check if pty channel exists already for elem in new_xml.devices.by_device_tag('channel'): if elem.type_name == channel_type_name: logging.debug("{0} channel already exists in guest. " "No need to add new one".format(channel_type_name)) return params = {'channel_type_name': channel_type_name, 'target_type': target_type, 'target_name': target_name} channel_xml = libvirt.create_channel_xml(params) virsh.attach_device(domain_opt=vm_name, file_opt=channel_xml.xml, flagstr="--config", ignore_status=False) logging.debug("New VMXML with channel:\n%s", virsh.dumpxml(vm_name)) def check_timeout_postcopy(params): """ Check the vm state on target host after timeout when --postcopy and --timeout-postcopy are used. The vm state is expected as running. :param params: the parameters used """ timeout = int(params.get("timeout_postcopy", 10)) time.sleep(timeout + 1) remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) vm_state = results_stdout_52lts(remote_virsh_session.domstate(vm_name)).strip() if vm_state != "running": remote_virsh_session.close_session() test.fail("After timeout '%s' seconds, " "the vm state on target host should " "be 'running', but '%s' found" % (timeout, vm_state)) remote_virsh_session.close_session() def check_converge(params): """ Handle option '--auto-converge --auto-converge-initial --auto-converge-increment '. 'Auto converge throttle' in domjobinfo should start with the initial value and increase with correct increment and max value is 99. :param params: The parameters used :raise: exceptions.TestFail when unexpected or no throttle is found """ initial = int(params.get("initial", 20)) increment = int(params.get("increment", 10)) max_converge = int(params.get("max_converge", 99)) allow_throttle_list = [initial + count * increment for count in range(0, (100 - initial) // increment + 1) if (initial + count * increment) < 100] allow_throttle_list.append(max_converge) logging.debug("The allowed 'Auto converge throttle' value " "is %s", allow_throttle_list) throttle = 0 jobtype = "None" while throttle < 100: cmd_result = virsh.domjobinfo(vm_name, debug=True, ignore_status=True) if cmd_result.exit_status: logging.debug(cmd_result.stderr) # Check if migration is completed if "domain is not running" in cmd_result.stderr: args = vm_name + " --completed" cmd_result = virsh.domjobinfo(args, debug=True, ignore_status=True) if cmd_result.exit_status: test.error("Failed to get domjobinfo and domjobinfo " "--completed: %s" % cmd_result.stderr) else: test.error("Failed to get domjobinfo: %s" % cmd_result.stderr) jobinfo = cmd_result.stdout for line in jobinfo.splitlines(): key = line.split(':')[0] if key.count("Job type"): jobtype = line.split(':')[-1].strip() elif key.count("Auto converge throttle"): throttle = int(line.split(':')[-1].strip()) logging.debug("Auto converge throttle:%s", str(throttle)) if throttle and throttle not in allow_throttle_list: test.fail("Invalid auto converge throttle " "value '%s'" % throttle) if throttle == 99: logging.debug("'Auto converge throttle' reaches maximum " "allowed value 99") break if jobtype == "None" or jobtype == "Completed": logging.debug("Jobtype:%s", jobtype) if not throttle: test.fail("'Auto converge throttle' is " "not found in the domjobinfo") break time.sleep(5) def get_usable_compress_cache(pagesize): """ Get a number which is bigger than pagesize and is power of two. :param pagesize: the given integer :return: an integer satisfying the criteria """ def calculate(num): result = num & (num - 1) return (result == 0) item = pagesize found = False while (not found): item += 1 found = calculate(item) logging.debug("%d is smallest one that is bigger than '%s' and " "is power of 2", item, pagesize) return item def update_config_file(config_type, new_conf, remote_host=True, params=None): """ Update the specified configuration file with dict :param config_type: Like libvirtd, qemu :param new_conf: The str including new configuration :param remote_host: True to also update in remote host :param params: The dict including parameters to connect remote host :return: utils_config.LibvirtConfigCommon object """ logging.debug("Update configuration file") cleanup_libvirtd_log(log_file) config_dict = eval(new_conf) updated_conf = libvirt.customize_libvirt_config(config_dict, config_type=config_type, remote_host=remote_host, extra_params=params) return updated_conf def check_migration_res(result): """ Check if the migration result is as expected :param result: the output of migration :raise: test.fail if test is failed """ logging.info("Migration out: %s", results_stdout_52lts(result).strip()) logging.info("Migration error: %s", results_stderr_52lts(result).strip()) if status_error: # Migration should fail if err_msg: # Special error messages are expected if not re.search(err_msg, results_stderr_52lts(result).strip()): test.fail("Can not find the expected patterns '%s' in " "output '%s'" % (err_msg, results_stderr_52lts(result).strip())) else: logging.debug("It is the expected error message") else: if int(result.exit_status) != 0: logging.debug("Migration failure is expected result") else: test.fail("Migration success is unexpected result") else: if int(result.exit_status) != 0: test.fail(results_stderr_52lts(result).strip()) check_parameters(test, params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} virsh_opt = params.get("virsh_opt", "") server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") log_file = params.get("libvirt_log", "/var/log/libvirt/libvirtd.log") check_complete_job = "yes" == params.get("check_complete_job", "no") check_domjobinfo_results = "yes" == params.get("check_domjobinfo_results") contrl_index = params.get("new_contrl_index", None) asynch_migration = "yes" == params.get("asynch_migrate", "no") grep_str_remote_log = params.get("grep_str_remote_log", "") grep_str_local_log = params.get("grep_str_local_log", "") disable_verify_peer = "yes" == params.get("disable_verify_peer", "no") status_error = "yes" == params.get("status_error", "no") stress_in_vm = "yes" == params.get("stress_in_vm", "no") low_speed = params.get("low_speed", None) remote_virsh_dargs = {'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True} cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} hpt_resize = params.get("hpt_resize", None) htm_state = params.get("htm_state", None) # For pty channel test add_channel = "yes" == params.get("add_channel", "no") channel_type_name = params.get("channel_type_name", None) target_type = params.get("target_type", None) target_name = params.get("target_name", None) cmd_run_in_remote_guest = params.get("cmd_run_in_remote_guest", None) cmd_run_in_remote_guest_1 = params.get("cmd_run_in_remote_guest_1", None) cmd_run_in_remote_host = params.get("cmd_run_in_remote_host", None) cmd_run_in_remote_host_1 = params.get("cmd_run_in_remote_host_1", None) cmd_run_in_remote_host_2 = params.get("cmd_run_in_remote_host_2", None) # For qemu command line checking qemu_check = params.get("qemu_check", None) xml_check_after_mig = params.get("guest_xml_check_after_mig", None) # params for cache matrix test cache = params.get("cache") remove_cache = "yes" == params.get("remove_cache", "no") err_msg = params.get("err_msg") arch = platform.machine() if any([hpt_resize, contrl_index, htm_state]) and 'ppc64' not in arch: test.cancel("The case is PPC only.") # For TLS tls_recovery = params.get("tls_auto_recovery", "yes") # qemu config qemu_conf_dict = None # libvirtd config libvirtd_conf_dict = None # remote shell session remote_session = None remote_virsh_session = None vm = None vm_session = None libvirtd_conf = None qemu_conf = None mig_result = None test_exception = None is_TestError = False is_TestFail = False is_TestSkip = False # Objects to be cleaned up in the end objs_list = [] tls_obj = None # Local variables vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() if not orig_config_xml: test.error("Backing up xmlfile failed.") try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Change the configuration files if needed before starting guest # For qemu.conf if extra.count("--tls"): # Setup TLS tls_obj = TLSConnection(params) if tls_recovery == "yes": objs_list.append(tls_obj) tls_obj.auto_recover = True tls_obj.conn_setup() # Setup qemu.conf qemu_conf_dict = params.get("qemu_conf_dict") if qemu_conf_dict: qemu_conf = update_config_file('qemu', qemu_conf_dict, params=params) # Setup libvirtd libvirtd_conf_dict = params.get("libvirtd_conf_dict") if libvirtd_conf_dict: libvirtd_conf = update_config_file('libvirtd', libvirtd_conf_dict, params=params) # Prepare required guest xml before starting guest if contrl_index: new_xml.remove_all_device_by_type('controller') logging.debug("After removing controllers, current XML:\n%s\n", new_xml) add_ctrls(new_xml, dev_index=contrl_index) if add_channel: attach_channel_xml() if hpt_resize: set_feature(new_xml, 'hpt', hpt_resize) if htm_state: set_feature(new_xml, 'htm', htm_state) if cache: params["driver_cache"] = cache if remove_cache: params["enable_cache"] = "no" # Change the disk of the vm to shared disk and then start VM libvirt.set_vm_disk(vm, params) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check qemu command line after guest is started if qemu_check: check_content = qemu_check if hpt_resize: check_content = "%s%s" % (qemu_check, hpt_resize) if htm_state: check_content = "%s%s" % (qemu_check, htm_state) libvirt.check_qemu_cmd_line(check_content) # Check local guest network connection before migration vm_session = vm.wait_for_login() check_vm_network_accessed() # Preparation for the running guest before migration if hpt_resize and hpt_resize != 'disabled': trigger_hpt_resize(vm_session) if stress_in_vm: pkg_name = 'stress' logging.debug("Check if stress tool is installed") pkg_mgr = utils_package.package_manager(vm_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("Stress tool will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) stress_thread = threading.Thread(target=run_stress_in_vm, args=()) stress_thread.start() if extra.count("timeout-postcopy"): func_name = check_timeout_postcopy if params.get("actions_during_migration"): func_name = do_actions_during_migrate if extra.count("comp-xbzrle-cache"): cache = get_usable_compress_cache(memory.get_page_size()) extra = "%s %s" % (extra, cache) # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (extra, postcopy_options) if low_speed: control_migrate_speed(int(low_speed)) if postcopy_options and libvirt_version.version_compare(5, 0, 0): control_migrate_speed(int(low_speed), opts=postcopy_options) # Execute migration process if not asynch_migration: mig_result = do_migration(vm, dest_uri, options, extra) else: migration_test = libvirt.MigrationTest() logging.debug("vm.connect_uri=%s", vm.connect_uri) vms = [vm] try: migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_opt, func=func_name, extra_opts=extra, func_params=params) mig_result = migration_test.ret except exceptions.TestFail as fail_detail: test.fail(fail_detail) except exceptions.TestSkipError as skip_detail: test.cancel(skip_detail) except exceptions.TestError as error_detail: test.error(error_detail) except Exception as details: mig_result = migration_test.ret logging.error(details) check_migration_res(mig_result) if add_channel: # Get the channel device source path of remote guest if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) file_path = tempfile.mktemp(dir=data_dir.get_tmp_dir()) remote_virsh_session.dumpxml(vm_name, to_file=file_path, debug=True, ignore_status=True) local_vmxml = vm_xml.VMXML.new_from_dumpxml(vm_name) local_vmxml.xmltreefile = xml_utils.XMLTreeFile(file_path) for elem in local_vmxml.devices.by_device_tag('channel'): logging.debug("Found channel device {}".format(elem)) if elem.type_name == channel_type_name: host_source = elem.source.get('path') logging.debug("Remote guest uses {} for channel device".format(host_source)) break remote_virsh_session.close_session() if not host_source: test.fail("Can not find source for %s channel on remote host" % channel_type_name) # Prepare to wait for message on remote host from the channel cmd_result = remote.run_remote_cmd(cmd_run_in_remote_host % host_source, cmd_parms, runner_on_target) # Send message from remote guest to the channel file remote_vm_obj = utils_test.RemoteVMManager(cmd_parms) vm_ip = vm.get_address() vm_pwd = params.get("password") remote_vm_obj.setup_ssh_auth(vm_ip, vm_pwd) cmd_result = remote_vm_obj.run_command(vm_ip, cmd_run_in_remote_guest_1) remote_vm_obj.run_command(vm_ip, cmd_run_in_remote_guest % results_stdout_52lts(cmd_result).strip()) logging.debug("Sending message is done") # Check message on remote host from the channel remote.run_remote_cmd(cmd_run_in_remote_host_1, cmd_parms, runner_on_target) logging.debug("Receiving message is done") remote.run_remote_cmd(cmd_run_in_remote_host_2, cmd_parms, runner_on_target) if check_complete_job: opts = " --completed" check_virsh_command_and_option("domjobinfo", opts) if extra.count("comp-xbzrle-cache"): params.update({'compare_to_value': cache // 1024}) check_domjobinfo(params, option=opts) if check_domjobinfo_results: check_domjobinfo_output(option=opts, is_mig_compelete=True) if grep_str_local_log: cmd = "grep -E '%s' %s" % (grep_str_local_log, log_file) cmdRes = process.run(cmd, shell=True, ignore_status=True) if cmdRes.exit_status: test.fail(results_stderr_52lts(cmdRes).strip()) if grep_str_remote_log: cmd = "grep -E '%s' %s" % (grep_str_remote_log, log_file) remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if xml_check_after_mig: if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) target_guest_dumpxml = results_stdout_52lts( remote_virsh_session.dumpxml(vm_name, debug=True, ignore_status=True)).strip() if hpt_resize: check_str = hpt_resize elif htm_state: check_str = htm_state if hpt_resize or htm_state: xml_check_after_mig = "%s'%s'" % (xml_check_after_mig, check_str) if not re.search(xml_check_after_mig, target_guest_dumpxml): remote_virsh_session.close_session() test.fail("Fail to search '%s' in target guest XML:\n%s" % (xml_check_after_mig, target_guest_dumpxml)) if contrl_index: all_ctrls = re.findall(xml_check_after_mig, target_guest_dumpxml) if len(all_ctrls) != int(contrl_index) + 1: remote_virsh_session.close_session() test.fail("%s pci-root controllers are expected in guest XML, " "but found %s" % (int(contrl_index) + 1, len(all_ctrls))) remote_virsh_session.close_session() if int(mig_result.exit_status) == 0: server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") check_vm_network_accessed(server_session) server_session.close() except exceptions.TestFail as details: is_TestFail = True test_exception = details except exceptions.TestSkipError as details: is_TestSkip = True test_exception = details except exceptions.TestError as details: is_TestError = True test_exception = details except Exception as details: test_exception = details finally: logging.debug("Recover test environment") try: # Clean VM on destination vm.connect_uri = dest_uri cleanup_dest(vm) vm.connect_uri = src_uri logging.info("Recovery VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) if remote_virsh_session: remote_virsh_session.close_session() if remote_session: remote_session.close() # Delete files on target # Killing qemu process on target may lead a problem like # qemu process becomes a zombie process whose ppid is 1. # As a workaround, have to remove the files under # /var/run/libvirt/qemu to make libvirt work. if vm.is_qemu(): dest_pid_files = os.path.join("/var/run/libvirt/qemu", vm_name + '*') cmd = "rm -f %s" % dest_pid_files logging.debug("Delete remote pid files '%s'", dest_pid_files) remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if extra.count("--tls") and not disable_verify_peer: logging.debug("Recover the qemu configuration") libvirt.customize_libvirt_config(None, config_type="qemu", remote_host=True, extra_params=params, is_recover=True, config_object=qemu_conf) for update_conf in [libvirtd_conf, qemu_conf]: if update_conf: logging.debug("Recover the configurations") libvirt.customize_libvirt_config(None, remote_host=True, extra_params=params, is_recover=True, config_object=update_conf) logging.info("Remove local NFS image") source_file = params.get("source_file") libvirt.delete_local_disk("file", path=source_file) if objs_list: for obj in objs_list: logging.debug("Clean up local objs") del obj except Exception as exception_detail: if (not test_exception and not is_TestError and not is_TestFail and not is_TestSkip): raise exception_detail else: # if any of above exceptions has been raised, only print # error log here to avoid of hiding the original issue logging.error(exception_detail) # Check result if is_TestFail: test.fail(test_exception) if is_TestSkip: test.cancel(test_exception) if is_TestError: test.error(test_exception) if not test_exception: logging.info("Case execution is done.") else: test.error(test_exception)
def run(test, params, env): """ Test virsh migrate command. """ def set_feature(vmxml, feature, value): """ Set guest features for PPC :param state: the htm status :param vmxml: guest xml """ features_xml = vm_xml.VMFeaturesXML() if feature == 'hpt': features_xml.hpt_resizing = value elif feature == 'htm': features_xml.htm = value vmxml.features = features_xml vmxml.sync() def trigger_hpt_resize(session): """ Check the HPT order file and dmesg :param session: the session to guest :raise: test.fail if required message is not found """ hpt_order_path = "/sys/kernel/debug/powerpc/hpt_order" hpt_order = session.cmd_output('cat %s' % hpt_order_path).strip() hpt_order = int(hpt_order) logging.info('Current hpt_order is %d', hpt_order) hpt_order += 1 cmd = 'echo %d > %s' % (hpt_order, hpt_order_path) cmd_result = session.cmd_status_output(cmd) result = process.CmdResult(stderr=cmd_result[1], stdout=cmd_result[1], exit_status=cmd_result[0]) libvirt.check_exit_status(result) dmesg = session.cmd('dmesg') dmesg_content = params.get('dmesg_content').split('|') for content in dmesg_content: if content % hpt_order not in dmesg: test.fail("'%s' is missing in dmesg" % (content % hpt_order)) else: logging.info("'%s' is found in dmesg", content % hpt_order) def check_vm_network_accessed(session=None): """ The operations to the VM need to be done before or after migration happens :param session: The session object to the host :raise: test.error when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") s_ping, _ = utils_test.ping(vm.get_address(), count=10, timeout=20, output_func=logging.debug, session=session) if s_ping != 0: if session: session.close() test.fail("%s did not respond after %d sec." % (vm.name, 20)) def check_virsh_command_and_option(command, option=None): """ Check if virsh command exists :param command: the command to be checked :param option: the command option to be checked """ msg = "This version of libvirt does not support " if not virsh.has_help_command(command): test.cancel(msg + "virsh command '%s'" % command) if option and not virsh.has_command_help_match(command, option): test.cancel(msg + "virsh command '%s' with option '%s'" % (command, option)) def add_ctrls(vm_xml, dev_type="pci", dev_index="0", dev_model="pci-root"): """ Add multiple devices :param dev_type: the type of the device to be added :param dev_index: the maximum index of the device to be added :param dev_model: the model of the device to be added """ for inx in range(0, int(dev_index) + 1): newcontroller = Controller("controller") newcontroller.type = dev_type newcontroller.index = inx newcontroller.model = dev_model logging.debug("New device is added:\n%s", newcontroller) vm_xml.add_device(newcontroller) vm_xml.sync() def do_migration(vm, dest_uri, options, extra): """ Execute the migration with given parameters :param vm: the guest to be migrated :param dest_uri: the destination uri for migration :param options: options next to 'migrate' command :param extra: options in the end of the migrate command line :return: CmdResult object """ logging.info("Sleeping 10 seconds before migration") time.sleep(10) # Migrate the guest. virsh_args.update({"ignore_status": True}) migration_res = vm.migrate(dest_uri, options, extra, **virsh_args) if int(migration_res.exit_status) != 0: logging.error("Migration failed for %s.", vm_name) return migration_res if vm.is_alive(): # vm.connect_uri was updated logging.info("VM is alive on destination %s.", dest_uri) else: test.fail("VM is not alive on destination %s" % dest_uri) # Throws exception if console shows panic message vm.verify_kernel_crash() return migration_res def cleanup_libvirtd_log(log_file): """ Remove existing libvirtd log file on source and target host. :param log_file: log file with absolute path """ if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) cmd = "rm -f %s" % log_file logging.debug("Delete remote libvirt log file '%s'", log_file) cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) def cleanup_dest(vm): """ Clean up the destination host environment when doing the uni-direction migration. :param vm: the guest to be cleaned up """ logging.info("Cleaning up VMs on %s", vm.connect_uri) try: if virsh.domain_exists(vm.name, uri=vm.connect_uri): vm_state = vm.state() if vm_state == "paused": vm.resume() elif vm_state == "shut off": vm.start() vm.destroy(gracefully=False) if vm.is_persistent(): vm.undefine() except Exception as detail: logging.error("Cleaning up destination failed.\n%s", detail) def run_stress_in_vm(): """ The function to load stress in VM """ stress_args = params.get("stress_args", "--cpu 8 --io 4 " "--vm 2 --vm-bytes 128M " "--timeout 20s") try: vm_session.cmd('stress %s' % stress_args) except Exception as detail: logging.debug(detail) def control_migrate_speed(to_speed=1): """ Control migration duration :param to_speed: the speed value in Mbps to be set for migration :return int: the new migration speed after setting """ virsh_args.update({"ignore_status": False}) old_speed = virsh.migrate_getspeed(vm_name, **virsh_args) logging.debug("Current migration speed is %s MiB/s\n", old_speed.stdout.strip()) logging.debug("Set migration speed to %d MiB/s\n", to_speed) cmd_result = virsh.migrate_setspeed(vm_name, to_speed, "", **virsh_args) actual_speed = virsh.migrate_getspeed(vm_name, **virsh_args) logging.debug("New migration speed is %s MiB/s\n", actual_speed.stdout.strip()) return int(actual_speed.stdout.strip()) def check_setspeed(params): """ Set/get migration speed :param params: the parameters used :raise: test.fail if speed set does not take effect """ expected_value = int(params.get("migrate_speed", '41943040')) // (1024 * 1024) actual_value = control_migrate_speed(to_speed=expected_value) params.update({'compare_to_value': actual_value}) if actual_value != expected_value: test.fail("Migration speed is expected to be '%d MiB/s', but '%d MiB/s' " "found" % (expected_value, actual_value)) def check_domjobinfo(params, option=""): """ Check given item in domjobinfo of the guest is as expected :param params: the parameters used :param option: options for domjobinfo :raise: test.fail if the value of given item is unexpected """ def search_jobinfo(jobinfo): """ Find value of given item in domjobinfo :param jobinfo: cmdResult object :raise: test.fail if not found """ for item in jobinfo.stdout.splitlines(): if item.count(jobinfo_item): groups = re.findall(r'[0-9.]+', item.strip()) logging.debug("In '%s' search '%s'\n", item, groups[0]) if (math.fabs(float(groups[0]) - float(compare_to_value)) // float(compare_to_value) > diff_rate): test.fail("{} {} has too much difference from " "{}".format(jobinfo_item, groups[0], compare_to_value)) break jobinfo_item = params.get("jobinfo_item") compare_to_value = params.get("compare_to_value") logging.debug("compare_to_value:%s", compare_to_value) diff_rate = float(params.get("diff_rate", "0")) if not jobinfo_item or not compare_to_value: return vm_ref = '{}{}'.format(vm_name, option) jobinfo = virsh.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) check_domjobinfo_remote = params.get("check_domjobinfo_remote") if check_domjobinfo_remote: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) jobinfo = remote_virsh_session.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) remote_virsh_session.close_session() def check_maxdowntime(params): """ Set/get migration maxdowntime :param params: the parameters used :raise: test.fail if maxdowntime set does not take effect """ expected_value = int(float(params.get("migrate_maxdowntime", '0.3')) * 1000) virsh_args.update({"ignore_status": False}) old_value = int(virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("Current migration maxdowntime is %d ms", old_value) logging.debug("Set migration maxdowntime to %d ms", expected_value) virsh.migrate_setmaxdowntime(vm_name, expected_value, **virsh_args) actual_value = int(virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("New migration maxdowntime is %d ms", actual_value) if actual_value != expected_value: test.fail("Migration maxdowntime is expected to be '%d ms', but '%d ms' " "found" % (expected_value, actual_value)) params.update({'compare_to_value': actual_value}) def do_actions_during_migrate(params): """ The entry point to execute action list during migration :param params: the parameters used """ actions_during_migration = params.get("actions_during_migration") if not actions_during_migration: return for action in actions_during_migration.split(","): if action == 'setspeed': check_setspeed(params) elif action == 'domjobinfo': check_domjobinfo(params) elif action == 'setmaxdowntime': check_maxdowntime(params) time.sleep(3) def attach_channel_xml(): """ Create channel xml and attach it to guest configuration """ # Check if pty channel exists already for elem in new_xml.devices.by_device_tag('channel'): if elem.type_name == channel_type_name: logging.debug("{0} channel already exists in guest. " "No need to add new one".format(channel_type_name)) return params = {'channel_type_name': channel_type_name, 'target_type': target_type, 'target_name': target_name} channel_xml = libvirt.create_channel_xml(params) virsh.attach_device(domain_opt=vm_name, file_opt=channel_xml.xml, flagstr="--config", ignore_status=False) logging.debug("New VMXML with channel:\n%s", virsh.dumpxml(vm_name)) def check_timeout_postcopy(params): """ Check the vm state on target host after timeout when --postcopy and --timeout-postcopy are used. The vm state is expected as running. :param params: the parameters used """ timeout = int(params.get("timeout_postcopy", 10)) time.sleep(timeout + 1) remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) vm_state = results_stdout_52lts(remote_virsh_session.domstate(vm_name)).strip() if vm_state != "running": remote_virsh_session.close_session() test.fail("After timeout '%s' seconds, " "the vm state on target host should " "be 'running', but '%s' found", timeout, vm_state) remote_virsh_session.close_session() def get_usable_compress_cache(pagesize): """ Get a number which is bigger than pagesize and is power of two. :param pagesize: the given integer :return: an integer satisfying the criteria """ def calculate(num): result = num & (num - 1) return (result == 0) item = pagesize found = False while (not found): item += 1 found = calculate(item) logging.debug("%d is smallest one that is bigger than '%s' and " "is power of 2", item, pagesize) return item def check_migration_res(result): """ Check if the migration result is as expected :param result: the output of migration :raise: test.fail if test is failed """ logging.info("Migration out: %s", results_stdout_52lts(result).strip()) logging.info("Migration error: %s", results_stderr_52lts(result).strip()) if status_error: # Migration should fail if err_msg: # Special error messages are expected if not re.search(err_msg, results_stderr_52lts(result).strip()): test.fail("Can not find the expected patterns '%s' in " "output '%s'" % (err_msg, results_stderr_52lts(result).strip())) else: logging.debug("It is the expected error message") else: if int(result.exit_status) != 0: logging.debug("Migration failure is expected result") else: test.fail("Migration success is unexpected result") else: if int(result.exit_status) != 0: test.fail(results_stderr_52lts(result).strip()) check_parameters(test, params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} virsh_opt = params.get("virsh_opt", "") server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") log_file = params.get("libvirt_log", "/var/log/libvirt/libvirtd.log") check_complete_job = "yes" == params.get("check_complete_job", "no") config_libvirtd = "yes" == params.get("config_libvirtd", "no") contrl_index = params.get("new_contrl_index", None) asynch_migration = "yes" == params.get("asynch_migrate", "no") grep_str_remote_log = params.get("grep_str_remote_log", "") grep_str_local_log = params.get("grep_str_local_log", "") disable_verify_peer = "yes" == params.get("disable_verify_peer", "no") status_error = "yes" == params.get("status_error", "no") stress_in_vm = "yes" == params.get("stress_in_vm", "no") low_speed = params.get("low_speed", None) remote_virsh_dargs = {'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True} hpt_resize = params.get("hpt_resize", None) htm_state = params.get("htm_state", None) # For pty channel test add_channel = "yes" == params.get("add_channel", "no") channel_type_name = params.get("channel_type_name", None) target_type = params.get("target_type", None) target_name = params.get("target_name", None) cmd_run_in_remote_guest = params.get("cmd_run_in_remote_guest", None) cmd_run_in_remote_guest_1 = params.get("cmd_run_in_remote_guest_1", None) cmd_run_in_remote_host = params.get("cmd_run_in_remote_host", None) cmd_run_in_remote_host_1 = params.get("cmd_run_in_remote_host_1", None) cmd_run_in_remote_host_2 = params.get("cmd_run_in_remote_host_2", None) # For qemu command line checking qemu_check = params.get("qemu_check", None) xml_check_after_mig = params.get("guest_xml_check_after_mig", None) # params for cache matrix test cache = params.get("cache") remove_cache = "yes" == params.get("remove_cache", "no") err_msg = params.get("err_msg") arch = platform.machine() if any([hpt_resize, contrl_index, htm_state]) and 'ppc64' not in arch: test.cancel("The case is PPC only.") # For TLS tls_recovery = params.get("tls_auto_recovery", "yes") # qemu config qemu_conf_dict = None # libvirtd config libvirtd_conf_dict = None remote_virsh_session = None vm = None vm_session = None libvirtd_conf = None qemu_conf = None mig_result = None test_exception = None is_TestError = False is_TestFail = False is_TestSkip = False # Objects to be cleaned up in the end objs_list = [] tls_obj = None # Local variables vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() if not orig_config_xml: test.error("Backing up xmlfile failed.") try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Change the configuration files if needed before starting guest # For qemu.conf if extra.count("--tls"): # Setup TLS tls_obj = TLSConnection(params) if tls_recovery == "yes": objs_list.append(tls_obj) tls_obj.auto_recover = True tls_obj.conn_setup() if not disable_verify_peer: qemu_conf_dict = {"migrate_tls_x509_verify": "1"} # Setup qemu configure logging.debug("Configure the qemu") cleanup_libvirtd_log(log_file) qemu_conf = libvirt.customize_libvirt_config(qemu_conf_dict, config_type="qemu", remote_host=True, extra_params=params) # Setup libvirtd if config_libvirtd: logging.debug("Configure the libvirtd") cleanup_libvirtd_log(log_file) libvirtd_conf_dict = setup_libvirtd_conf_dict(params) libvirtd_conf = libvirt.customize_libvirt_config(libvirtd_conf_dict, remote_host=True, extra_params=params) # Prepare required guest xml before starting guest if contrl_index: new_xml.remove_all_device_by_type('controller') logging.debug("After removing controllers, current XML:\n%s\n", new_xml) add_ctrls(new_xml, dev_index=contrl_index) if add_channel: attach_channel_xml() if hpt_resize: set_feature(new_xml, 'hpt', hpt_resize) if htm_state: set_feature(new_xml, 'htm', htm_state) if cache: params["driver_cache"] = cache if remove_cache: params["enable_cache"] = "no" # Change the disk of the vm to shared disk and then start VM libvirt.set_vm_disk(vm, params) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check qemu command line after guest is started if qemu_check: check_content = qemu_check if hpt_resize: check_content = "%s%s" % (qemu_check, hpt_resize) if htm_state: check_content = "%s%s" % (qemu_check, htm_state) libvirt.check_qemu_cmd_line(check_content) # Check local guest network connection before migration vm_session = vm.wait_for_login() check_vm_network_accessed() # Preparation for the running guest before migration if hpt_resize and hpt_resize != 'disabled': trigger_hpt_resize(vm_session) if low_speed: control_migrate_speed(int(low_speed)) if stress_in_vm: pkg_name = 'stress' logging.debug("Check if stress tool is installed") pkg_mgr = utils_package.package_manager(vm_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("Stress tool will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) stress_thread = threading.Thread(target=run_stress_in_vm, args=()) stress_thread.start() if extra.count("timeout-postcopy"): func_name = check_timeout_postcopy if params.get("actions_during_migration"): func_name = do_actions_during_migrate if extra.count("comp-xbzrle-cache"): cache = get_usable_compress_cache(memory.get_page_size()) extra = "%s %s" % (extra, cache) # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (extra, postcopy_options) # Execute migration process if not asynch_migration: mig_result = do_migration(vm, dest_uri, options, extra) else: migration_test = libvirt.MigrationTest() logging.debug("vm.connect_uri=%s", vm.connect_uri) vms = [vm] try: migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_opt, func=func_name, extra_opts=extra, func_params=params) mig_result = migration_test.ret except exceptions.TestFail as fail_detail: test.fail(fail_detail) except exceptions.TestSkipError as skip_detail: test.cancel(skip_detail) except exceptions.TestError as error_detail: test.error(error_detail) except Exception as details: mig_result = migration_test.ret logging.error(details) check_migration_res(mig_result) if add_channel: # Get the channel device source path of remote guest if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) file_path = tempfile.mktemp(dir=data_dir.get_tmp_dir()) remote_virsh_session.dumpxml(vm_name, to_file=file_path, debug=True, ignore_status=True) local_vmxml = vm_xml.VMXML.new_from_dumpxml(vm_name) local_vmxml.xmltreefile = xml_utils.XMLTreeFile(file_path) for elem in local_vmxml.devices.by_device_tag('channel'): logging.debug("Found channel device {}".format(elem)) if elem.type_name == channel_type_name: host_source = elem.source.get('path') logging.debug("Remote guest uses {} for channel device".format(host_source)) break remote_virsh_session.close_session() if not host_source: test.fail("Can not find source for %s channel on remote host" % channel_type_name) # Prepare to wait for message on remote host from the channel cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} cmd_result = remote.run_remote_cmd(cmd_run_in_remote_host % host_source, cmd_parms, runner_on_target) # Send message from remote guest to the channel file remote_vm_obj = utils_test.RemoteVMManager(cmd_parms) vm_ip = vm.get_address() vm_pwd = params.get("password") remote_vm_obj.setup_ssh_auth(vm_ip, vm_pwd) cmd_result = remote_vm_obj.run_command(vm_ip, cmd_run_in_remote_guest_1) remote_vm_obj.run_command(vm_ip, cmd_run_in_remote_guest % results_stdout_52lts(cmd_result).strip()) logging.debug("Sending message is done") # Check message on remote host from the channel remote.run_remote_cmd(cmd_run_in_remote_host_1, cmd_parms, runner_on_target) logging.debug("Receiving message is done") remote.run_remote_cmd(cmd_run_in_remote_host_2, cmd_parms, runner_on_target) if check_complete_job: opts = " --completed" check_virsh_command_and_option("domjobinfo", opts) if extra.count("comp-xbzrle-cache"): params.update({'compare_to_value': cache // 1024}) check_domjobinfo(params, option=opts) if grep_str_local_log: cmd = "grep -E '%s' %s" % (grep_str_local_log, log_file) cmdRes = process.run(cmd, shell=True, ignore_status=True) if cmdRes.exit_status: test.fail(results_stderr_52lts(cmdRes).strip()) if grep_str_remote_log: cmd = "grep -E '%s' %s" % (grep_str_remote_log, log_file) cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if xml_check_after_mig: if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) target_guest_dumpxml = results_stdout_52lts( remote_virsh_session.dumpxml(vm_name, debug=True, ignore_status=True)).strip() if hpt_resize: check_str = hpt_resize elif htm_state: check_str = htm_state if hpt_resize or htm_state: xml_check_after_mig = "%s'%s'" % (xml_check_after_mig, check_str) if not re.search(xml_check_after_mig, target_guest_dumpxml): remote_virsh_session.close_session() test.fail("Fail to search '%s' in target guest XML:\n%s" % (xml_check_after_mig, target_guest_dumpxml)) if contrl_index: all_ctrls = re.findall(xml_check_after_mig, target_guest_dumpxml) if len(all_ctrls) != int(contrl_index) + 1: remote_virsh_session.close_session() test.fail("%s pci-root controllers are expected in guest XML, " "but found %s" % (int(contrl_index) + 1, len(all_ctrls))) remote_virsh_session.close_session() if int(mig_result.exit_status) == 0: server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") check_vm_network_accessed(server_session) server_session.close() except exceptions.TestFail as details: is_TestFail = True test_exception = details except exceptions.TestSkipError as details: is_TestSkip = True test_exception = details except exceptions.TestError as details: is_TestError = True test_exception = details except Exception as details: test_exception = details finally: logging.debug("Recover test environment") try: # Clean VM on destination vm.connect_uri = dest_uri cleanup_dest(vm) vm.connect_uri = src_uri logging.info("Recovery VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) if remote_virsh_session: remote_virsh_session.close_session() if extra.count("--tls") and not disable_verify_peer: logging.debug("Recover the qemu configuration") libvirt.customize_libvirt_config(None, config_type="qemu", remote_host=True, extra_params=params, is_recover=True, config_object=qemu_conf) if config_libvirtd: logging.debug("Recover the libvirtd configuration") libvirt.customize_libvirt_config(None, remote_host=True, extra_params=params, is_recover=True, config_object=libvirtd_conf) logging.info("Remove local NFS image") source_file = params.get("source_file") libvirt.delete_local_disk("file", path=source_file) if objs_list: for obj in objs_list: logging.debug("Clean up local objs") del obj except Exception as exception_detail: if (not test_exception and not is_TestError and not is_TestFail and not is_TestSkip): raise exception_detail else: # if any of above exceptions has been raised, only print # error log here to avoid of hiding the original issue logging.error(exception_detail) # Check result if is_TestFail: test.fail(test_exception) if is_TestSkip: test.cancel(test_exception) if is_TestError: test.error(test_exception) if not test_exception: logging.info("Case execution is done.") else: test.error(test_exception)
def run(test, params, env): """ Test remote access with TCP, TLS connection """ test_dict = dict(params) vm_name = test_dict.get("main_vm") vm = env.get_vm(vm_name) start_vm = test_dict.get("start_vm", "no") # Server and client parameters server_ip = test_dict.get("server_ip") server_user = test_dict.get("server_user") server_pwd = test_dict.get("server_pwd") client_ip = test_dict.get("client_ip") client_user = test_dict.get("client_user") client_pwd = test_dict.get("client_pwd") server_cn = test_dict.get("server_cn") client_cn = test_dict.get("client_cn") target_ip = test_dict.get("target_ip", "") # generate remote IP if target_ip == "": if server_cn: target_ip = server_cn elif server_ip: target_ip = server_ip else: target_ip = target_ip remote_virsh_dargs = { 'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True } # Ceph disk parameters driver = test_dict.get("test_driver", "qemu") transport = test_dict.get("transport") plus = test_dict.get("conn_plus", "+") source_type = test_dict.get("vm_disk_source_type", "file") virsh_options = test_dict.get("virsh_options", "--verbose --live") vol_name = test_dict.get("vol_name") disk_src_protocol = params.get("disk_source_protocol") source_file = test_dict.get("disk_source_file") disk_format = test_dict.get("disk_format", "qcow2") mon_host = params.get("mon_host") ceph_key_opt = "" attach_disk = False # Disk XML file disk_xml = None # Define ceph_disk conditional variable ceph_disk = "yes" == test_dict.get("ceph_disk") # For --postcopy enable postcopy_options = test_dict.get("postcopy_options") if postcopy_options and not virsh_options.count(postcopy_options): virsh_options = "%s %s" % (virsh_options, postcopy_options) test_dict['virsh_options'] = virsh_options # For bi-directional and tls reverse test uri_port = test_dict.get("uri_port", ":22") uri_path = test_dict.get("uri_path", "/system") src_uri = test_dict.get("migration_source_uri", "qemu:///system") uri = "%s%s%s://%s%s%s" % (driver, plus, transport, target_ip, uri_port, uri_path) test_dict["desuri"] = uri # Make sure all of parameters are assigned a valid value check_parameters(test, test_dict) # Set up SSH key #ssh_key.setup_ssh_key(server_ip, server_user, server_pwd, port=22) remote_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") remote_session.close() #ssh_key.setup_ssh_key(server_ip, server_user, server_pwd, port=22) # Set up remote ssh key and remote /etc/hosts file for bi-direction migration migrate_vm_back = "yes" == test_dict.get("migrate_vm_back", "no") if migrate_vm_back: ssh_key.setup_remote_ssh_key(server_ip, server_user, server_pwd) ssh_key.setup_remote_known_hosts_file(client_ip, server_ip, server_user, server_pwd) # Reset Vm state if needed if vm.is_alive() and start_vm == "no": vm.destroy(gracefully=False) # Back up xml file. vmxml_backup = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) # Setup migration context migrate_setup = migration.MigrationTest() migrate_setup.migrate_pre_setup(test_dict["desuri"], params) # Install ceph-common on remote host machine. remote_ssh_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, r"[\#\$]\s*$") if not utils_package.package_install(["ceph-common"], remote_ssh_session): test.error("Failed to install required packages on remote host") remote_ssh_session.close() try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Get initial Selinux config flex bit LOCAL_SELINUX_ENFORCING_STATUS = utils_selinux.get_status() logging.info("previous local enforce :%s", LOCAL_SELINUX_ENFORCING_STATUS) cmd_result = remote.run_remote_cmd('getenforce', params, runner_on_target) REMOTE_SELINUX_ENFORCING_STATUS = cmd_result.stdout_text logging.info("previous remote enforce :%s", REMOTE_SELINUX_ENFORCING_STATUS) if ceph_disk: logging.info( "Put local SELinux in permissive mode when test ceph migrating" ) utils_selinux.set_status("enforcing") logging.info("Put remote SELinux in permissive mode") cmd = "setenforce enforcing" cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) status, output = cmd_result.exit_status, cmd_result.stdout_text.strip( ) if status: test.Error("Failed to set SELinux " "in permissive mode") # Prepare ceph disk. key_file = os.path.join(data_dir.get_tmp_dir(), "ceph.key") test_dict['key_file'] = key_file test_dict['first_disk'] = vm.get_first_disk_devices() ceph_key_opt, secret_uuid = prepare_ceph_disk( test_dict, remote_virsh_dargs, test, runner_on_target) host_ip = test_dict.get('mon_host') disk_image = test_dict.get('disk_img') # Build auth information. auth_attrs = {} auth_attrs['auth_user'] = params.get("auth_user") auth_attrs['secret_type'] = params.get("secret_type") auth_attrs['secret_uuid'] = secret_uuid build_disk_xml(vm_name, disk_format, host_ip, disk_src_protocol, vol_name, disk_image, auth=auth_attrs) vm_xml_cxt = process.run("virsh dumpxml %s" % vm_name, shell=True).stdout_text logging.debug("The VM XML with ceph disk source: \n%s", vm_xml_cxt) try: if vm.is_dead(): vm.start() except virt_vm.VMStartError as e: logging.info("Failed to start VM") test.fail("Failed to start VM: %s" % vm_name) # Ensure the same VM name doesn't exist on remote host before migrating. destroy_vm_cmd = "virsh destroy %s" % vm_name remote.run_remote_cmd(cmd, params, runner_on_target) # Trigger migration migrate_vm(test, test_dict) if migrate_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=client_ip, server_pwd=client_pwd, client_ip=server_ip, client_pwd=server_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine migrate_setup.migrate_pre_setup(src_uri, params) cmd = "virsh migrate %s %s %s" % (vm_name, virsh_options, src_uri) logging.debug("Start migrating: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) status, output = cmd_result.exit_status, cmd_result.stdout_text.strip( ) logging.info(output) if status: destroy_cmd = "virsh destroy %s" % vm_name remote.run_remote_cmd(destroy_cmd, params, runner_on_target) test.fail("Failed to run '%s' on remote: %s" % (cmd, output)) finally: logging.info("Recovery test environment") # Clean up of pre migration setup for local machine if migrate_vm_back: migrate_setup.migrate_pre_setup(src_uri, params, cleanup=True) # Ensure VM can be cleaned up on remote host even migrating fail. destroy_vm_cmd = "virsh destroy %s" % vm_name remote.run_remote_cmd(destroy_vm_cmd, params, runner_on_target) logging.info("Recovery VM XML configuration") vmxml_backup.sync() logging.debug("The current VM XML:\n%s", vmxml_backup.xmltreefile) # Clean up ceph environment. if disk_src_protocol == "rbd": # Clean up secret secret_list = get_secret_list() if secret_list: for secret_uuid in secret_list: virsh.secret_undefine(secret_uuid) # Clean up dirty secrets on remote host if testing involve in ceph auth. client_name = test_dict.get('client_name') client_key = test_dict.get("client_key") if client_name and client_key: try: remote_virsh = virsh.VirshPersistent(**remote_virsh_dargs) remote_dirty_secret_list = get_secret_list(remote_virsh) for dirty_secret_uuid in remote_dirty_secret_list: remote_virsh.secret_undefine(dirty_secret_uuid) except (process.CmdError, remote.SCPError) as detail: test.Error(detail) finally: remote_virsh.close_session() # Delete the disk if it exists. disk_src_name = "%s/%s" % (vol_name, test_dict.get('disk_img')) cmd = ("rbd -m {0} {1} info {2} && rbd -m {0} {1} rm " "{2}".format(mon_host, ceph_key_opt, disk_src_name)) process.run(cmd, ignore_status=True, shell=True) if LOCAL_SELINUX_ENFORCING_STATUS: logging.info("Restore SELinux in original mode") utils_selinux.set_status(LOCAL_SELINUX_ENFORCING_STATUS) if REMOTE_SELINUX_ENFORCING_STATUS: logging.info("Put remote SELinux in original mode") cmd = "yes yes | setenforce %s" % REMOTE_SELINUX_ENFORCING_STATUS remote.run_remote_cmd(cmd, params, runner_on_target) # Remove known hosts on local host cmd = "ssh-keygen -R %s" % server_ip process.run(cmd, ignore_status=True, shell=True) # Remove known hosts on remote host cmd = "ssh-keygen -R %s" % client_ip remote.run_remote_cmd(cmd, params, runner_on_target)
def prepare_ceph_disk(ceph_params, remote_virsh_dargs, test, runner_on_target): """ Prepare one image on remote ceph server with enabled or disabled auth And expose it to VM by network access :param ceph_params: parameter to setup ceph. :param remote_virsh_dargs: parameter to remote virsh. :param test: test itself. """ # Ceph server config parameters virsh_dargs = {'debug': True, 'ignore_status': True} prompt = ceph_params.get("prompt", r"[\#\$]\s*$") ceph_disk = "yes" == ceph_params.get("ceph_disk") mon_host = ceph_params.get('mon_host') client_name = ceph_params.get('client_name') client_key = ceph_params.get("client_key") vol_name = ceph_params.get("vol_name") disk_img = ceph_params.get("disk_img") key_file = ceph_params.get("key_file") disk_format = ceph_params.get("disk_format") key_opt = "" # Auth and secret config parameters. auth_user = ceph_params.get("auth_user") auth_key = ceph_params.get("auth_key") auth_type = ceph_params.get("auth_type") auth_usage = ceph_params.get("secret_usage") secret_uuid = ceph_params.get("secret_uuid") # Remote host parameters. remote_ip = ceph_params.get("server_ip") remote_user = ceph_params.get("server_user", "root") remote_pwd = ceph_params.get("server_pwd") # Clean up dirty secrets in test environments if there are. dirty_secret_list = get_secret_list() if dirty_secret_list: for dirty_secret_uuid in dirty_secret_list: virsh.secret_undefine(dirty_secret_uuid) # Install ceph-common package which include rbd command if utils_package.package_install(["ceph-common"]): if client_name and client_key: # Clean up dirty secrets on remote host. try: remote_virsh = virsh.VirshPersistent(**remote_virsh_dargs) remote_dirty_secret_list = get_secret_list(remote_virsh) for dirty_secret_uuid in remote_dirty_secret_list: remote_virsh.secret_undefine(dirty_secret_uuid) except (process.CmdError, remote.SCPError) as detail: raise exceptions.TestError(detail) finally: logging.debug('clean up secret on remote host') remote_virsh.close_session() with open(key_file, 'w') as f: f.write("[%s]\n\tkey = %s\n" % (client_name, client_key)) key_opt = "--keyring %s" % key_file # Create secret xml sec_xml = secret_xml.SecretXML("no", "no") sec_xml.usage = auth_type sec_xml.usage_name = auth_usage sec_xml.uuid = secret_uuid sec_xml.xmltreefile.write() logging.debug("Secret xml: %s", sec_xml) ret = virsh.secret_define(sec_xml.xml) libvirt.check_exit_status(ret) secret_uuid = re.findall(r".+\S+(\ +\S+)\ +.+\S+", ret.stdout.strip())[0].lstrip() logging.debug("Secret uuid %s", secret_uuid) if secret_uuid is None: test.fail("Failed to get secret uuid") # Set secret value ret = virsh.secret_set_value(secret_uuid, auth_key, **virsh_dargs) libvirt.check_exit_status(ret) # Create secret on remote host. local_path = sec_xml.xml remote_path = '/var/lib/libvirt/images/new_secret.xml' remote_folder = '/var/lib/libvirt/images' cmd = 'mkdir -p %s && chmod 777 %s && touch %s' % ( remote_folder, remote_folder, remote_path) cmd_result = remote.run_remote_cmd(cmd, ceph_params, runner_on_target) status, output = cmd_result.exit_status, cmd_result.stdout_text.strip( ) if status: test.fail("Failed to run '%s' on the remote: %s" % (cmd, output)) remote.scp_to_remote(remote_ip, '22', remote_user, remote_pwd, local_path, remote_path, limit="", log_filename=None, timeout=600, interface=None) cmd = "/usr/bin/virsh secret-define --file %s" % remote_path cmd_result = remote.run_remote_cmd(cmd, ceph_params, runner_on_target) status, output = cmd_result.exit_status, cmd_result.stdout_text.strip( ) if status: test.fail("Failed to run '%s' on the remote: %s" % (cmd, output)) # Set secret value on remote host. cmd = "/usr/bin/virsh secret-set-value --secret %s --base64 %s" % ( secret_uuid, auth_key) cmd_result = remote.run_remote_cmd(cmd, ceph_params, runner_on_target) status, output = cmd_result.exit_status, cmd_result.stdout_text.strip( ) if status: test.fail("Failed to run '%s' on the remote: %s" % (cmd, output)) # Delete the disk if it exists disk_src_name = "%s/%s" % (vol_name, disk_img) cmd = ("rbd -m {0} {1} info {2} && rbd -m {0} {1} rm " "{2}".format(mon_host, key_opt, disk_src_name)) process.run(cmd, ignore_status=True, shell=True) # Convert the disk format first_disk_device = ceph_params.get('first_disk') blk_source = first_disk_device['source'] disk_path = ("rbd:%s:mon_host=%s" % (disk_src_name, mon_host)) if auth_user and auth_key: disk_path += (":id=%s:key=%s" % (auth_user, auth_key)) disk_cmd = ("rbd -m %s %s info %s || qemu-img convert" " -O %s %s %s" % (mon_host, key_opt, disk_src_name, disk_format, blk_source, disk_path)) process.run(disk_cmd, ignore_status=False, shell=True) return (key_opt, secret_uuid)
def run(test, params, env): """ Test virsh migrate command. """ def set_feature(vmxml, feature, value): """ Set guest features for PPC :param state: the htm status :param vmxml: guest xml """ features_xml = vm_xml.VMFeaturesXML() if feature == 'hpt': features_xml.hpt_resizing = value elif feature == 'htm': features_xml.htm = value vmxml.features = features_xml vmxml.sync() def trigger_hpt_resize(session): """ Check the HPT order file and dmesg :param session: the session to guest :raise: test.fail if required message is not found """ hpt_order_path = "/sys/kernel/debug/powerpc/hpt_order" hpt_order = session.cmd_output('cat %s' % hpt_order_path).strip() hpt_order = int(hpt_order) logging.info('Current hpt_order is %d', hpt_order) hpt_order += 1 cmd = 'echo %d > %s' % (hpt_order, hpt_order_path) cmd_result = session.cmd_status_output(cmd) result = process.CmdResult(stderr=cmd_result[1], stdout=cmd_result[1], exit_status=cmd_result[0]) libvirt.check_exit_status(result) dmesg = session.cmd('dmesg') dmesg_content = params.get('dmesg_content').split('|') for content in dmesg_content: if content % hpt_order not in dmesg: test.fail("'%s' is missing in dmesg" % (content % hpt_order)) else: logging.info("'%s' is found in dmesg", content % hpt_order) def check_vm_network_accessed(session=None): """ The operations to the VM need to be done before or after migration happens :param session: The session object to the host :raise: test.error when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") s_ping, _ = utils_test.ping(vm.get_address(), count=10, timeout=20, output_func=logging.debug, session=session) if s_ping != 0: if session: session.close() test.fail("%s did not respond after %d sec." % (vm.name, 20)) def check_virsh_command_and_option(command, option=None): """ Check if virsh command exists :param command: the command to be checked :param option: the command option to be checked """ msg = "This version of libvirt does not support " if not virsh.has_help_command(command): test.cancel(msg + "virsh command '%s'" % command) if option and not virsh.has_command_help_match(command, option): test.cancel(msg + "virsh command '%s' with option '%s'" % (command, option)) def add_ctrls(vm_xml, dev_type="pci", dev_index="0", dev_model="pci-root"): """ Add multiple devices :param dev_type: the type of the device to be added :param dev_index: the maximum index of the device to be added :param dev_model: the model of the device to be added """ for inx in range(0, int(dev_index) + 1): newcontroller = Controller("controller") newcontroller.type = dev_type newcontroller.index = inx newcontroller.model = dev_model logging.debug("New device is added:\n%s", newcontroller) vm_xml.add_device(newcontroller) vm_xml.sync() def do_migration(vm, dest_uri, options, extra): """ Execute the migration with given parameters :param vm: the guest to be migrated :param dest_uri: the destination uri for migration :param options: options next to 'migrate' command :param extra: options in the end of the migrate command line :return: CmdResult object """ logging.info("Sleeping 10 seconds before migration") time.sleep(10) # Migrate the guest. virsh_args.update({"ignore_status": True}) migration_res = vm.migrate(dest_uri, options, extra, **virsh_args) if int(migration_res.exit_status) != 0: logging.error("Migration failed for %s.", vm_name) return migration_res if vm.is_alive(): # vm.connect_uri was updated logging.info("VM is alive on destination %s.", dest_uri) else: test.fail("VM is not alive on destination %s" % dest_uri) # Throws exception if console shows panic message vm.verify_kernel_crash() return migration_res def cleanup_libvirtd_log(log_file): """ Remove existing libvirtd log file on source and target host. :param log_file: log file with absolute path """ if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) cmd = "rm -f %s" % log_file logging.debug("Delete remote libvirt log file '%s'", log_file) cmd_parms = { 'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd } remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) def cleanup_dest(vm): """ Clean up the destination host environment when doing the uni-direction migration. :param vm: the guest to be cleaned up """ logging.info("Cleaning up VMs on %s", vm.connect_uri) try: if virsh.domain_exists(vm.name, uri=vm.connect_uri): vm_state = vm.state() if vm_state == "paused": vm.resume() elif vm_state == "shut off": vm.start() vm.destroy(gracefully=False) if vm.is_persistent(): vm.undefine() except Exception as detail: logging.error("Cleaning up destination failed.\n%s", detail) def run_stress_in_vm(): """ The function to load stress in VM """ stress_args = params.get( "stress_args", "--cpu 8 --io 4 " "--vm 2 --vm-bytes 128M " "--timeout 20s") try: vm_session.cmd('stress %s' % stress_args) except Exception as detail: logging.debug(detail) def control_migrate_speed(to_speed=1): """ Control migration duration :param to_speed: the speed value in Mbps to be set for migration :return int: the new migration speed after setting """ virsh_args.update({"ignore_status": False}) old_speed = virsh.migrate_getspeed(vm_name, **virsh_args) logging.debug("Current migration speed is %s MiB/s\n", old_speed.stdout.strip()) logging.debug("Set migration speed to %d MiB/s\n", to_speed) cmd_result = virsh.migrate_setspeed(vm_name, to_speed, "", **virsh_args) actual_speed = virsh.migrate_getspeed(vm_name, **virsh_args) logging.debug("New migration speed is %s MiB/s\n", actual_speed.stdout.strip()) return int(actual_speed.stdout.strip()) def check_setspeed(params): """ Set/get migration speed :param params: the parameters used :raise: test.fail if speed set does not take effect """ expected_value = int(params.get("migrate_speed", '41943040')) // (1024 * 1024) actual_value = control_migrate_speed(to_speed=expected_value) params.update({'compare_to_value': actual_value}) if actual_value != expected_value: test.fail( "Migration speed is expected to be '%d MiB/s', but '%d MiB/s' " "found" % (expected_value, actual_value)) def check_domjobinfo(params, option=""): """ Check given item in domjobinfo of the guest is as expected :param params: the parameters used :param option: options for domjobinfo :raise: test.fail if the value of given item is unexpected """ def search_jobinfo(jobinfo): """ Find value of given item in domjobinfo :param jobinfo: cmdResult object :raise: test.fail if not found """ for item in jobinfo.stdout.splitlines(): if item.count(jobinfo_item): groups = re.findall(r'[0-9.]+', item.strip()) logging.debug("In '%s' search '%s'\n", item, groups[0]) if (math.fabs(float(groups[0]) - float(compare_to_value)) // float(compare_to_value) > diff_rate): test.fail("{} {} has too much difference from " "{}".format(jobinfo_item, groups[0], compare_to_value)) break jobinfo_item = params.get("jobinfo_item") compare_to_value = params.get("compare_to_value") logging.debug("compare_to_value:%s", compare_to_value) diff_rate = float(params.get("diff_rate", "0")) if not jobinfo_item or not compare_to_value: return vm_ref = '{}{}'.format(vm_name, option) jobinfo = virsh.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) check_domjobinfo_remote = params.get("check_domjobinfo_remote") if check_domjobinfo_remote: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) jobinfo = remote_virsh_session.domjobinfo(vm_ref, **virsh_args) search_jobinfo(jobinfo) remote_virsh_session.close_session() def check_maxdowntime(params): """ Set/get migration maxdowntime :param params: the parameters used :raise: test.fail if maxdowntime set does not take effect """ expected_value = int( float(params.get("migrate_maxdowntime", '0.3')) * 1000) virsh_args.update({"ignore_status": False}) old_value = int(virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("Current migration maxdowntime is %d ms", old_value) logging.debug("Set migration maxdowntime to %d ms", expected_value) virsh.migrate_setmaxdowntime(vm_name, expected_value, **virsh_args) actual_value = int( virsh.migrate_getmaxdowntime(vm_name).stdout.strip()) logging.debug("New migration maxdowntime is %d ms", actual_value) if actual_value != expected_value: test.fail( "Migration maxdowntime is expected to be '%d ms', but '%d ms' " "found" % (expected_value, actual_value)) params.update({'compare_to_value': actual_value}) def do_actions_during_migrate(params): """ The entry point to execute action list during migration :param params: the parameters used """ actions_during_migration = params.get("actions_during_migration") if not actions_during_migration: return for action in actions_during_migration.split(","): if action == 'setspeed': check_setspeed(params) elif action == 'domjobinfo': check_domjobinfo(params) elif action == 'setmaxdowntime': check_maxdowntime(params) time.sleep(3) def attach_channel_xml(): """ Create channel xml and attach it to guest configuration """ # Check if pty channel exists already for elem in new_xml.devices.by_device_tag('channel'): if elem.type_name == channel_type_name: logging.debug( "{0} channel already exists in guest. " "No need to add new one".format(channel_type_name)) return params = { 'channel_type_name': channel_type_name, 'target_type': target_type, 'target_name': target_name } channel_xml = libvirt.create_channel_xml(params) virsh.attach_device(domain_opt=vm_name, file_opt=channel_xml.xml, flagstr="--config", ignore_status=False) logging.debug("New VMXML with channel:\n%s", virsh.dumpxml(vm_name)) def check_timeout_postcopy(params): """ Check the vm state on target host after timeout when --postcopy and --timeout-postcopy are used. The vm state is expected as running. :param params: the parameters used """ timeout = int(params.get("timeout_postcopy", 10)) time.sleep(timeout + 1) remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) vm_state = results_stdout_52lts( remote_virsh_session.domstate(vm_name)).strip() if vm_state != "running": remote_virsh_session.close_session() test.fail( "After timeout '%s' seconds, " "the vm state on target host should " "be 'running', but '%s' found", timeout, vm_state) remote_virsh_session.close_session() def get_usable_compress_cache(pagesize): """ Get a number which is bigger than pagesize and is power of two. :param pagesize: the given integer :return: an integer satisfying the criteria """ def calculate(num): result = num & (num - 1) return (result == 0) item = pagesize found = False while (not found): item += 1 found = calculate(item) logging.debug( "%d is smallest one that is bigger than '%s' and " "is power of 2", item, pagesize) return item def check_migration_res(result): """ Check if the migration result is as expected :param result: the output of migration :raise: test.fail if test is failed """ logging.info("Migration out: %s", results_stdout_52lts(result).strip()) logging.info("Migration error: %s", results_stderr_52lts(result).strip()) if status_error: # Migration should fail if err_msg: # Special error messages are expected if not re.search(err_msg, results_stderr_52lts(result).strip()): test.fail("Can not find the expected patterns '%s' in " "output '%s'" % (err_msg, results_stderr_52lts(result).strip())) else: logging.debug("It is the expected error message") else: if int(result.exit_status) != 0: logging.debug("Migration failure is expected result") else: test.fail("Migration success is unexpected result") else: if int(result.exit_status) != 0: test.fail(results_stderr_52lts(result).strip()) check_parameters(test, params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} virsh_opt = params.get("virsh_opt", "") server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") log_file = params.get("libvirt_log", "/var/log/libvirt/libvirtd.log") check_complete_job = "yes" == params.get("check_complete_job", "no") config_libvirtd = "yes" == params.get("config_libvirtd", "no") contrl_index = params.get("new_contrl_index", None) asynch_migration = "yes" == params.get("asynch_migrate", "no") grep_str_remote_log = params.get("grep_str_remote_log", "") grep_str_local_log = params.get("grep_str_local_log", "") disable_verify_peer = "yes" == params.get("disable_verify_peer", "no") status_error = "yes" == params.get("status_error", "no") stress_in_vm = "yes" == params.get("stress_in_vm", "no") low_speed = params.get("low_speed", None) remote_virsh_dargs = { 'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True } hpt_resize = params.get("hpt_resize", None) htm_state = params.get("htm_state", None) # For pty channel test add_channel = "yes" == params.get("add_channel", "no") channel_type_name = params.get("channel_type_name", None) target_type = params.get("target_type", None) target_name = params.get("target_name", None) cmd_run_in_remote_guest = params.get("cmd_run_in_remote_guest", None) cmd_run_in_remote_guest_1 = params.get("cmd_run_in_remote_guest_1", None) cmd_run_in_remote_host = params.get("cmd_run_in_remote_host", None) cmd_run_in_remote_host_1 = params.get("cmd_run_in_remote_host_1", None) cmd_run_in_remote_host_2 = params.get("cmd_run_in_remote_host_2", None) # For qemu command line checking qemu_check = params.get("qemu_check", None) xml_check_after_mig = params.get("guest_xml_check_after_mig", None) # params for cache matrix test cache = params.get("cache") remove_cache = "yes" == params.get("remove_cache", "no") err_msg = params.get("err_msg") arch = platform.machine() if any([hpt_resize, contrl_index, htm_state]) and 'ppc64' not in arch: test.cancel("The case is PPC only.") # For TLS tls_recovery = params.get("tls_auto_recovery", "yes") # qemu config qemu_conf_dict = None # libvirtd config libvirtd_conf_dict = None remote_virsh_session = None vm = None vm_session = None libvirtd_conf = None qemu_conf = None mig_result = None test_exception = None is_TestError = False is_TestFail = False is_TestSkip = False # Objects to be cleaned up in the end objs_list = [] tls_obj = None # Local variables vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() if not orig_config_xml: test.error("Backing up xmlfile failed.") try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Change the configuration files if needed before starting guest # For qemu.conf if extra.count("--tls"): # Setup TLS tls_obj = TLSConnection(params) if tls_recovery == "yes": objs_list.append(tls_obj) tls_obj.auto_recover = True tls_obj.conn_setup() if not disable_verify_peer: qemu_conf_dict = {"migrate_tls_x509_verify": "1"} # Setup qemu configure logging.debug("Configure the qemu") cleanup_libvirtd_log(log_file) qemu_conf = libvirt.customize_libvirt_config( qemu_conf_dict, config_type="qemu", remote_host=True, extra_params=params) # Setup libvirtd if config_libvirtd: logging.debug("Configure the libvirtd") cleanup_libvirtd_log(log_file) libvirtd_conf_dict = setup_libvirtd_conf_dict(params) libvirtd_conf = libvirt.customize_libvirt_config( libvirtd_conf_dict, remote_host=True, extra_params=params) # Prepare required guest xml before starting guest if contrl_index: new_xml.remove_all_device_by_type('controller') logging.debug("After removing controllers, current XML:\n%s\n", new_xml) add_ctrls(new_xml, dev_index=contrl_index) if add_channel: attach_channel_xml() if hpt_resize: set_feature(new_xml, 'hpt', hpt_resize) if htm_state: set_feature(new_xml, 'htm', htm_state) if cache: params["driver_cache"] = cache if remove_cache: params["enable_cache"] = "no" # Change the disk of the vm to shared disk and then start VM libvirt.set_vm_disk(vm, params) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check qemu command line after guest is started if qemu_check: check_content = qemu_check if hpt_resize: check_content = "%s%s" % (qemu_check, hpt_resize) if htm_state: check_content = "%s%s" % (qemu_check, htm_state) libvirt.check_qemu_cmd_line(check_content) # Check local guest network connection before migration vm_session = vm.wait_for_login() check_vm_network_accessed() # Preparation for the running guest before migration if hpt_resize and hpt_resize != 'disabled': trigger_hpt_resize(vm_session) if low_speed: control_migrate_speed(int(low_speed)) if stress_in_vm: pkg_name = 'stress' logging.debug("Check if stress tool is installed") pkg_mgr = utils_package.package_manager(vm_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("Stress tool will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) stress_thread = threading.Thread(target=run_stress_in_vm, args=()) stress_thread.start() if extra.count("timeout-postcopy"): func_name = check_timeout_postcopy if params.get("actions_during_migration"): func_name = do_actions_during_migrate if extra.count("comp-xbzrle-cache"): cache = get_usable_compress_cache(memory.get_page_size()) extra = "%s %s" % (extra, cache) # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (extra, postcopy_options) # Execute migration process if not asynch_migration: mig_result = do_migration(vm, dest_uri, options, extra) else: migration_test = libvirt.MigrationTest() logging.debug("vm.connect_uri=%s", vm.connect_uri) vms = [vm] try: migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_opt, func=func_name, extra_opts=extra, func_params=params) mig_result = migration_test.ret except exceptions.TestFail as fail_detail: test.fail(fail_detail) except exceptions.TestSkipError as skip_detail: test.cancel(skip_detail) except exceptions.TestError as error_detail: test.error(error_detail) except Exception as details: mig_result = migration_test.ret logging.error(details) check_migration_res(mig_result) if add_channel: # Get the channel device source path of remote guest if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent( **remote_virsh_dargs) file_path = tempfile.mktemp(dir=data_dir.get_tmp_dir()) remote_virsh_session.dumpxml(vm_name, to_file=file_path, debug=True, ignore_status=True) local_vmxml = vm_xml.VMXML.new_from_dumpxml(vm_name) local_vmxml.xmltreefile = xml_utils.XMLTreeFile(file_path) for elem in local_vmxml.devices.by_device_tag('channel'): logging.debug("Found channel device {}".format(elem)) if elem.type_name == channel_type_name: host_source = elem.source.get('path') logging.debug( "Remote guest uses {} for channel device".format( host_source)) break remote_virsh_session.close_session() if not host_source: test.fail("Can not find source for %s channel on remote host" % channel_type_name) # Prepare to wait for message on remote host from the channel cmd_parms = { 'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd } cmd_result = remote.run_remote_cmd( cmd_run_in_remote_host % host_source, cmd_parms, runner_on_target) # Send message from remote guest to the channel file remote_vm_obj = utils_test.RemoteVMManager(cmd_parms) vm_ip = vm.get_address() vm_pwd = params.get("password") remote_vm_obj.setup_ssh_auth(vm_ip, vm_pwd) cmd_result = remote_vm_obj.run_command(vm_ip, cmd_run_in_remote_guest_1) remote_vm_obj.run_command( vm_ip, cmd_run_in_remote_guest % results_stdout_52lts(cmd_result).strip()) logging.debug("Sending message is done") # Check message on remote host from the channel remote.run_remote_cmd(cmd_run_in_remote_host_1, cmd_parms, runner_on_target) logging.debug("Receiving message is done") remote.run_remote_cmd(cmd_run_in_remote_host_2, cmd_parms, runner_on_target) if check_complete_job: opts = " --completed" check_virsh_command_and_option("domjobinfo", opts) if extra.count("comp-xbzrle-cache"): params.update({'compare_to_value': cache // 1024}) check_domjobinfo(params, option=opts) if grep_str_local_log: cmd = "grep -E '%s' %s" % (grep_str_local_log, log_file) cmdRes = process.run(cmd, shell=True, ignore_status=True) if cmdRes.exit_status: test.fail(results_stderr_52lts(cmdRes).strip()) if grep_str_remote_log: cmd = "grep -E '%s' %s" % (grep_str_remote_log, log_file) cmd_parms = { 'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd } remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if xml_check_after_mig: if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent( **remote_virsh_dargs) target_guest_dumpxml = results_stdout_52lts( remote_virsh_session.dumpxml(vm_name, debug=True, ignore_status=True)).strip() if hpt_resize: check_str = hpt_resize elif htm_state: check_str = htm_state if hpt_resize or htm_state: xml_check_after_mig = "%s'%s'" % (xml_check_after_mig, check_str) if not re.search(xml_check_after_mig, target_guest_dumpxml): remote_virsh_session.close_session() test.fail("Fail to search '%s' in target guest XML:\n%s" % (xml_check_after_mig, target_guest_dumpxml)) if contrl_index: all_ctrls = re.findall(xml_check_after_mig, target_guest_dumpxml) if len(all_ctrls) != int(contrl_index) + 1: remote_virsh_session.close_session() test.fail( "%s pci-root controllers are expected in guest XML, " "but found %s" % (int(contrl_index) + 1, len(all_ctrls))) remote_virsh_session.close_session() if int(mig_result.exit_status) == 0: server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") check_vm_network_accessed(server_session) server_session.close() except exceptions.TestFail as details: is_TestFail = True test_exception = details except exceptions.TestSkipError as details: is_TestSkip = True test_exception = details except exceptions.TestError as details: is_TestError = True test_exception = details except Exception as details: test_exception = details finally: logging.debug("Recover test environment") try: # Clean VM on destination vm.connect_uri = dest_uri cleanup_dest(vm) vm.connect_uri = src_uri logging.info("Recovery VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) if remote_virsh_session: remote_virsh_session.close_session() if extra.count("--tls") and not disable_verify_peer: logging.debug("Recover the qemu configuration") libvirt.customize_libvirt_config(None, config_type="qemu", remote_host=True, extra_params=params, is_recover=True, config_object=qemu_conf) if config_libvirtd: logging.debug("Recover the libvirtd configuration") libvirt.customize_libvirt_config(None, remote_host=True, extra_params=params, is_recover=True, config_object=libvirtd_conf) logging.info("Remove local NFS image") source_file = params.get("source_file") libvirt.delete_local_disk("file", path=source_file) if objs_list: for obj in objs_list: logging.debug("Clean up local objs") del obj except Exception as exception_detail: if (not test_exception and not is_TestError and not is_TestFail and not is_TestSkip): raise exception_detail else: # if any of above exceptions has been raised, only print # error log here to avoid of hiding the original issue logging.error(exception_detail) # Check result if is_TestFail: test.fail(test_exception) if is_TestSkip: test.cancel(test_exception) if is_TestError: test.error(test_exception) if not test_exception: logging.info("Case execution is done.") else: test.error(test_exception)
def run(test, params, env): """ Test virsh migrate command. """ def check_vm_network_accessed(session=None): """ The operations to the VM need to be done before or after migration happens :param session: The session object to the host :raise: test.error when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") s_ping, _ = utils_test.ping(vm.get_address(), count=10, timeout=20, output_func=logging.debug, session=session) if s_ping != 0: if session: session.close() test.fail("%s did not respond after %d sec." % (vm.name, 20)) def check_migration_res(result): """ Check if the migration result is as expected :param result: the output of migration :raise: test.fail if test is failed """ if not result: test.error("No migration result is returned.") logging.info("Migration out: %s", result.stdout_text.strip()) logging.info("Migration error: %s", result.stderr_text.strip()) if status_error: # Migration should fail if err_msg: # Special error messages are expected if not re.search(err_msg, result.stderr_text.strip()): test.fail("Can not find the expected patterns '%s' in " "output '%s'" % (err_msg, result.stderr_text.strip())) else: logging.debug("It is the expected error message") else: if int(result.exit_status) != 0: logging.debug("Migration failure is expected result") else: test.fail("Migration success is unexpected result") else: if int(result.exit_status) != 0: test.fail(result.stderr_text.strip()) check_parameters(test, params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} virsh_options = params.get("virsh_options", "") server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") client_ip = params.get("client_ip") client_pwd = params.get("client_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") guest_src_url = params.get("guest_src_url") guest_src_path = params.get("guest_src_path", "/var/lib/libvirt/images/guest.img") check_disk = "yes" == params.get("check_disk") disk_model = params.get("disk_model") disk_target = params.get("disk_target", "vda") controller_model = params.get("controller_model") check_interface = "yes" == params.get("check_interface") iface_type = params.get("iface_type", "network") iface_model = params.get("iface_model", "virtio") iface_params = { 'type': iface_type, 'model': iface_model, 'del_addr': True, 'source': '{"network": "default"}' } check_memballoon = "yes" == params.get("check_memballoon") membal_model = params.get("membal_model") check_rng = "yes" == params.get("check_rng") rng_model = params.get("rng_model") migr_vm_back = "yes" == params.get("migrate_vm_back", "no") status_error = "yes" == params.get("status_error", "no") remote_virsh_dargs = { 'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True } xml_check_after_mig = params.get("guest_xml_check_after_mig") err_msg = params.get("err_msg") vm_session = None remote_virsh_session = None vm = None mig_result = None if not libvirt_version.version_compare(5, 0, 0): test.cancel("This libvirt version doesn't support " "virtio-transitional model.") # Make sure all of parameters are assigned a valid value check_parameters(test, params) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) params["virsh_migrate_connect_uri"] = libvirt_vm.complete_uri( params.get("migrate_source_host")) src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() migration_test = migration.MigrationTest() try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # download guest source and update interface model to keep guest up if guest_src_url: blk_source = download.get_file(guest_src_url, guest_src_path) if not blk_source: test.error("Fail to download image.") params["blk_source_name"] = blk_source if (not check_interface) and iface_model: iface_dict = {'model': iface_model} libvirt.modify_vm_iface(vm_name, "update_iface", iface_dict) if not check_disk: params["disk_model"] = "virtio-transitional" if check_interface: libvirt.modify_vm_iface(vm_name, "update_iface", iface_params) if check_memballoon: membal_dict = {'membal_model': membal_model} dom_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) libvirt.update_memballoon_xml(dom_xml, membal_dict) if check_rng: rng_dict = {'rng_model': rng_model} rng_xml = libvirt.create_rng_xml(rng_dict) dom_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) libvirt.add_vm_device(dom_xml, rng_xml) # Change the disk of the vm libvirt.set_vm_disk(vm, params) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check local guest network connection before migration vm_session = vm.wait_for_login(restart_network=True) check_vm_network_accessed() # Execute migration process vms = [vm] migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, extra_opts=extra) mig_result = migration_test.ret check_migration_res(mig_result) if int(mig_result.exit_status) == 0: server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") check_vm_network_accessed(server_session) server_session.close() if xml_check_after_mig: if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent( **remote_virsh_dargs) target_guest_dumpxml = (remote_virsh_session.dumpxml( vm_name, debug=True, ignore_status=True).stdout_text.strip()) if check_disk: check_str = disk_model if disk_model else controller_model if check_interface: check_str = iface_model if check_memballoon: check_str = membal_model if check_rng: check_str = rng_model xml_check_after_mig = "%s'%s'" % (xml_check_after_mig, check_str) if not re.search(xml_check_after_mig, target_guest_dumpxml): test.fail("Fail to search '%s' in target guest XML:\n%s" % (xml_check_after_mig, target_guest_dumpxml)) remote_virsh_session.close_session() # Execute migration from remote if migr_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=client_ip, server_pwd=client_pwd, client_ip=server_ip, client_pwd=server_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine migration_test.migrate_pre_setup(src_uri, params) cmd = "virsh migrate %s %s %s" % (vm_name, virsh_options, src_uri) logging.debug("Start migration: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) logging.info(cmd_result) if cmd_result.exit_status: test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) finally: logging.debug("Recover test environment") # Clean VM on destination vm.connect_uri = '' migration_test.cleanup_dest_vm(vm, src_uri, dest_uri) logging.info("Recovery VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) # Clean up of pre migration setup for local machine if migr_vm_back: if 'ssh_connection' in locals(): ssh_connection.auto_recover = True migration_test.migrate_pre_setup(src_uri, params, cleanup=True) if remote_virsh_session: remote_virsh_session.close_session() logging.info("Remove local NFS image") source_file = params.get("source_file") libvirt.delete_local_disk("file", path=source_file) if guest_src_url and blk_source: libvirt.delete_local_disk("file", path=blk_source)
def run(test, params, env): """ Test migration with special network settings 1) migrate guest with bridge type interface connected to ovs bridge :param test: test object :param params: Dictionary with the test parameters :param env: Dictionary with test environment. """ def check_vm_network_accessed(ping_dest, session=None): """ The operations to the VM need to be done before or after migration happens :param ping_dest: The destination to be ping :param session: The session object to the host :raise: test.fail when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") status, output = utils_net.ping(ping_dest, count=10, timeout=20, output_func=logging.debug, session=session) if status != 0: test.fail("Ping failed, status: %s, output: %s" % (status, output)) def update_iface_xml(vm_name, iface_dict): """ Update interfaces for guest :param vm_name: The name of VM :param iface_dict: The interface configurations params """ logging.debug("update iface xml") vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) vmxml.remove_all_device_by_type('interface') vmxml.sync() iface = interface.Interface('network') iface.xml = libvirt.modify_vm_iface(vm.name, "get_xml", iface_dict) libvirt.add_vm_device(vmxml, iface) migration_test = migration.MigrationTest() migration_test.check_parameters(params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") client_ip = params.get("client_ip") client_pwd = params.get("client_pwd") virsh_options = params.get("virsh_options", "") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options", "--live --p2p --verbose") func_params_exists = "yes" == params.get("func_params_exists", "no") migr_vm_back = "yes" == params.get("migr_vm_back", "no") ovs_bridge_name = params.get("ovs_bridge_name") network_dict = eval(params.get("network_dict", '{}')) iface_dict = eval(params.get("iface_dict", '{}')) remote_virsh_dargs = { 'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True } func_name = None libvirtd_conf = None mig_result = None # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) params["virsh_migrate_connect_uri"] = libvirt_vm.complete_uri( params.get("migrate_source_host")) src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() extra_args = {} if func_params_exists: extra_args.update({'func_params': params}) postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (extra, postcopy_options) func_name = virsh.migrate_postcopy # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, r'[$#%]') if ovs_bridge_name: status, stdout = utils_net.create_ovs_bridge(ovs_bridge_name) if status: test.fail("Failed to create ovs bridge on local. Status: %s" "Stdout: %s" % (status, stdout)) status, stdout = utils_net.create_ovs_bridge( ovs_bridge_name, session=remote_session) if status: test.fail("Failed to create ovs bridge on remote. Status: %s" "Stdout: %s" % (status, stdout)) if network_dict: libvirt_network.create_or_del_network( network_dict, remote_args=remote_virsh_dargs) libvirt_network.create_or_del_network(network_dict) remote_session.close() # Change domain network xml if iface_dict: if "mac" not in iface_dict: mac = utils_net.generate_mac_address_simple() iface_dict.update({'mac': mac}) else: mac = iface_dict["mac"] update_iface_xml(vm_name, iface_dict) # Change the disk of the vm libvirt.set_vm_disk(vm, params) if not vm.is_alive(): try: vm.start() except virt_vm.VMStartError as err: test.fail("Failed to start VM: %s" % err) logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check local guest network connection before migration if vm.serial_console is not None: vm.cleanup_serial_console() vm.create_serial_console() vm_session = vm.wait_for_serial_login(timeout=240) utils_net.restart_guest_network(vm_session) vm_ip = utils_net.get_guest_ip_addr(vm_session, mac) logging.debug("VM IP Addr: %s", vm_ip) check_vm_network_accessed(vm_ip) # Execute migration process vms = [vm] migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, func=func_name, extra_opts=extra, **extra_args) mig_result = migration_test.ret migration_test.check_result(mig_result, params) if int(mig_result.exit_status) == 0: remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, r'[$#%]') check_vm_network_accessed(vm_ip, session=remote_session) remote_session.close() # Execute migration from remote if migr_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=client_ip, server_pwd=client_pwd, client_ip=server_ip, client_pwd=server_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine migration_test.migrate_pre_setup(src_uri, params) cmd = "virsh migrate %s %s %s" % (vm_name, options, src_uri) logging.debug("Start migration: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) logging.info(cmd_result) if cmd_result.exit_status: test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) logging.debug("VM is migrated back.") check_vm_network_accessed(vm_ip) finally: logging.debug("Recover test environment") # Clean VM on destination and source try: migration_test.cleanup_dest_vm(vm, vm.connect_uri, dest_uri) except Exception as err: logging.error(err) if vm.is_alive(): vm.destroy(gracefully=False) logging.info("Recovery VM XML configration") orig_config_xml.sync() remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, r'[$#%]') if network_dict: libvirt_network.create_or_del_network( network_dict, is_del=True, remote_args=remote_virsh_dargs) libvirt_network.create_or_del_network(network_dict, is_del=True) if ovs_bridge_name: utils_net.delete_ovs_bridge(ovs_bridge_name) utils_net.delete_ovs_bridge(ovs_bridge_name, session=remote_session) remote_session.close() if migr_vm_back: if 'ssh_connection' in locals(): ssh_connection.auto_recover = True migration_test.migrate_pre_setup(src_uri, params, cleanup=True) logging.info("Remove local NFS image") source_file = params.get("source_file") if source_file: libvirt.delete_local_disk("file", path=source_file)
def run(test, params, env): """ Test migration with glusterfs. """ def create_or_clean_backend_dir(g_uri, params, session=None, is_clean=False): """ Create/cleanup backend directory :params g_uri: glusterfs uri :params params: the parameters to be checked :params session: VM/remote session object :params is_cleanup: True for cleanup backend directory; False for create one. :return: gluster_img if is_clean is equal to True """ mount_point = params.get("gluster_mount_dir") is_symlink = params.get("gluster_create_symlink") == "yes" symlink_name = params.get("gluster_symlink") gluster_img = None if not is_clean: if not utils_misc.check_exists(mount_point, session): utils_misc.make_dirs(mount_point, session) if gluster.glusterfs_is_mounted(mount_point, session): gluster.glusterfs_umount(g_uri, mount_point, session) gluster.glusterfs_mount(g_uri, mount_point, session) gluster_img = os.path.join(mount_point, disk_img) if is_symlink: utils_misc.make_symlink(mount_point, symlink_name) utils_misc.make_symlink(mount_point, symlink_name, remote_session) gluster_img = os.path.join(symlink_name, disk_img) return gluster_img else: if is_symlink: utils_misc.rm_link(symlink_name, session) gluster.glusterfs_umount(g_uri, mount_point, session) if utils_misc.check_exists(mount_point, session): utils_misc.safe_rmdir(gluster_mount_dir, session=session) def do_migration(vm, dest_uri, options, extra): """ Execute the migration with given parameters :param vm: the guest to be migrated :param dest_uri: the destination uri for migration :param options: options next to 'migrate' command :param extra: options in the end of the migrate command line :return: CmdResult object """ # Migrate the guest. virsh_args.update({"ignore_status": True}) migration_res = vm.migrate(dest_uri, options, extra, **virsh_args) if int(migration_res.exit_status) != 0: logging.error("Migration failed for %s.", vm_name) return migration_res if vm.is_alive(): logging.info("VM is alive on destination %s.", dest_uri) else: test.fail("VM is not alive on destination %s" % dest_uri) # Throws exception if console shows panic message vm.verify_kernel_crash() return migration_res def check_migration_res(result): """ Check if the migration result is as expected :param result: the output of migration :raise: test.fail if test is failed """ if not result: test.error("No migration result is returned.") logging.info("Migration out: %s", result.stdout_text.strip()) logging.info("Migration error: %s", result.stderr_text.strip()) if status_error: # Migration should fail if err_msg: # Special error messages are expected if not re.search(err_msg, result.stderr_text.strip()): test.fail("Can not find the expected patterns '%s' in " "output '%s'" % (err_msg, result.stderr_text.strip())) else: logging.debug("It is the expected error message") else: if int(result.exit_status) != 0: logging.debug("Migration failure is expected result") else: test.fail("Migration success is unexpected result") else: if int(result.exit_status) != 0: test.fail(result.stderr_text.strip()) # Local variables virsh_args = {"debug": True} server_ip = params["server_ip"] = params.get("remote_ip") server_user = params["server_user"] = params.get("remote_user", "root") server_pwd = params["server_pwd"] = params.get("remote_pwd") client_ip = params["client_ip"] = params.get("local_ip") client_pwd = params["client_pwd"] = params.get("local_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") virsh_options = params.get("virsh_options", "--verbose --live") vol_name = params.get("vol_name") disk_format = params.get("disk_format", "qcow2") gluster_mount_dir = params.get("gluster_mount_dir") status_error = "yes" == params.get("status_error", "no") err_msg = params.get("err_msg") host_ip = params.get("gluster_server_ip", "") migr_vm_back = params.get("migrate_vm_back", "no") == "yes" selinux_local = params.get('set_sebool_local', 'yes') == "yes" selinux_remote = params.get('set_sebool_remote', 'no') == "yes" sebool_fusefs_local = params.get('set_sebool_fusefs_local', 'yes') sebool_fusefs_remote = params.get('set_sebool_fusefs_remote', 'yes') test_dict = dict(params) test_dict["local_boolean_varible"] = "virt_use_fusefs" test_dict["remote_boolean_varible"] = "virt_use_fusefs" remove_pkg = False seLinuxBool = None seLinuxfusefs = None gluster_uri = None mig_result = None # Make sure all of parameters are assigned a valid value check_parameters(test, params) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) params["virsh_migrate_connect_uri"] = libvirt_vm.complete_uri( params.get("migrate_source_host")) src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options: virsh_options = "%s %s" % (virsh_options, postcopy_options) params['virsh_options'] = virsh_options vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # Back up xml file. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() migrate_setup = libvirt.MigrationTest() try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Configure selinux if selinux_local or selinux_remote: seLinuxBool = utils_misc.SELinuxBoolean(params) seLinuxBool.setup() if sebool_fusefs_local or sebool_fusefs_remote: seLinuxfusefs = utils_misc.SELinuxBoolean(test_dict) seLinuxfusefs.setup() # Setup glusterfs and disk xml. disk_img = "gluster.%s" % disk_format params['disk_img'] = disk_img libvirt.set_vm_disk(vm, params) vm_xml_cxt = virsh.dumpxml(vm_name).stdout_text.strip() logging.debug("The VM XML with gluster disk source: \n%s", vm_xml_cxt) # Check if gluster server is deployed locally if not host_ip: logging.debug("Enable port 24007 and 49152:49216") migrate_setup.migrate_pre_setup(src_uri, params, ports="24007") migrate_setup.migrate_pre_setup(src_uri, params) gluster_uri = "{}:{}".format(client_ip, vol_name) else: gluster_uri = "{}:{}".format(host_ip, vol_name) remote_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") if gluster_mount_dir: # The package 'glusterfs-fuse' is not installed on target # which makes issue when trying to 'mount -t glusterfs' pkg_name = 'glusterfs-fuse' logging.debug("Check if glusterfs-fuse is installed") pkg_mgr = utils_package.package_manager(remote_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("glusterfs-fuse will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) else: remove_pkg = True gluster_img = create_or_clean_backend_dir(gluster_uri, params) create_or_clean_backend_dir(gluster_uri, params, remote_session) logging.debug("Gluster Image is %s", gluster_img) gluster_backend_disk = {'disk_source_name': gluster_img} # Update disk xml with gluster image in backend dir libvirt.set_vm_disk(vm, gluster_backend_disk) remote_session.close() mig_result = do_migration(vm, dest_uri, options, extra) check_migration_res(mig_result) if migr_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=client_ip, server_pwd=client_pwd, client_ip=server_ip, client_pwd=server_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine migrate_setup.migrate_pre_setup(src_uri, params) cmd = "virsh migrate %s %s %s" % (vm_name, virsh_options, src_uri) logging.debug("Start migrating: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) logging.info(cmd_result) if cmd_result.exit_status: destroy_cmd = "virsh destroy %s" % vm_name remote.run_remote_cmd(destroy_cmd, params, runner_on_target, ignore_status=False) test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) finally: logging.info("Recovery test environment") orig_config_xml.sync() # Clean up of pre migration setup for local machine if migr_vm_back: if 'ssh_connection' in locals(): ssh_connection.auto_recover = True migrate_setup.migrate_pre_setup(src_uri, params, cleanup=True) # Cleanup selinu configuration if seLinuxBool: seLinuxBool.cleanup() if seLinuxfusefs: seLinuxfusefs.cleanup() # Disable ports 24007 and 49152:49216 if not host_ip: logging.debug("Disable 24007 and 49152:49216 in Firewall") migrate_setup.migrate_pre_setup(src_uri, params, cleanup=True, ports="24007") migrate_setup.migrate_pre_setup(src_uri, params, cleanup=True) gluster.setup_or_cleanup_gluster(False, **params) # Cleanup backend directory/symlink if gluster_mount_dir and gluster_uri: remote_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") create_or_clean_backend_dir(gluster_uri, params, is_clean=True) create_or_clean_backend_dir(gluster_uri, params, remote_session, True) if remove_pkg: pkg_mgr = utils_package.package_manager(remote_session, pkg_name) if pkg_mgr.is_installed(pkg_name): logging.debug("glusterfs-fuse will be uninstalled") if not pkg_mgr.remove(): logging.error("Package '%s' un-installation fails", pkg_name) remote_session.close()
def run(test, params, env): """ Test virsh migrate command. """ migration_test = migration.MigrationTest() migration_test.check_parameters(params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} virsh_options = params.get("virsh_options", "") server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") client_ip = params.get("client_ip") client_pwd = params.get("client_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") guest_src_url = params.get("guest_src_url") guest_src_path = params.get("guest_src_path", "/var/lib/libvirt/images/guest.img") check_disk = "yes" == params.get("check_disk") disk_model = params.get("disk_model") disk_target = params.get("disk_target", "vda") controller_model = params.get("controller_model") check_interface = "yes" == params.get("check_interface") iface_type = params.get("iface_type", "network") iface_model = params.get("iface_model", "virtio") iface_params = {'type': iface_type, 'model': iface_model, 'del_addr': True, 'source': '{"network": "default"}'} check_memballoon = "yes" == params.get("check_memballoon") membal_model = params.get("membal_model") check_rng = "yes" == params.get("check_rng") rng_model = params.get("rng_model") migrate_vm_back = "yes" == params.get("migrate_vm_back", "no") status_error = "yes" == params.get("status_error", "no") remote_virsh_dargs = {'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True} remote_dargs = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd, 'file_path': "/etc/libvirt/libvirt.conf"} xml_check_after_mig = params.get("guest_xml_check_after_mig") err_msg = params.get("err_msg") vm_session = None remote_virsh_session = None vm = None mig_result = None remove_dict = {} remote_libvirt_file = None src_libvirt_file = None if not libvirt_version.version_compare(5, 0, 0): test.cancel("This libvirt version doesn't support " "virtio-transitional model.") # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) params["virsh_migrate_connect_uri"] = libvirt_vm.complete_uri( params.get("migrate_source_host")) src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") extra_args = migration_test.update_virsh_migrate_extra_args(params) vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # download guest source and update interface model to keep guest up if guest_src_url: blk_source = download.get_file(guest_src_url, guest_src_path) if not blk_source: test.error("Fail to download image.") params["blk_source_name"] = blk_source if (not check_interface) and iface_model: iface_dict = {'model': iface_model} libvirt.modify_vm_iface(vm_name, "update_iface", iface_dict) if not check_disk: params["disk_model"] = "virtio-transitional" if check_interface: libvirt.modify_vm_iface(vm_name, "update_iface", iface_params) if check_memballoon: membal_dict = {'membal_model': membal_model} dom_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) libvirt.update_memballoon_xml(dom_xml, membal_dict) if check_rng: rng_dict = {'rng_model': rng_model} rng_xml = libvirt.create_rng_xml(rng_dict) dom_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) libvirt.add_vm_device(dom_xml, rng_xml) # Change the disk of the vm libvirt.set_vm_disk(vm, params) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check local guest network connection before migration vm_session = vm.wait_for_login(restart_network=True) migration_test.ping_vm(vm, params) remove_dict = {"do_search": '{"%s": "ssh:/"}' % dest_uri} src_libvirt_file = libvirt_config.remove_key_for_modular_daemon( remove_dict) # Execute migration process vms = [vm] migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, extra_opts=extra, **extra_args) mig_result = migration_test.ret if int(mig_result.exit_status) == 0: migration_test.ping_vm(vm, params, dest_uri) if xml_check_after_mig: if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) target_guest_dumpxml = (remote_virsh_session.dumpxml(vm_name, debug=True, ignore_status=True) .stdout_text.strip()) if check_disk: check_str = disk_model if disk_model else controller_model if check_interface: check_str = iface_model if check_memballoon: check_str = membal_model if check_rng: check_str = rng_model xml_check_after_mig = "%s'%s'" % (xml_check_after_mig, check_str) if not re.search(xml_check_after_mig, target_guest_dumpxml): test.fail("Fail to search '%s' in target guest XML:\n%s" % (xml_check_after_mig, target_guest_dumpxml)) remote_virsh_session.close_session() # Execute migration from remote if migrate_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=client_ip, server_pwd=client_pwd, client_ip=server_ip, client_pwd=server_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine migration_test.migrate_pre_setup(src_uri, params) remove_dict = {"do_search": ('{"%s": "ssh:/"}' % src_uri)} remote_libvirt_file = libvirt_config\ .remove_key_for_modular_daemon(remove_dict, remote_dargs) cmd = "virsh migrate %s %s %s" % (vm_name, options, src_uri) logging.debug("Start migration: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) logging.info(cmd_result) if cmd_result.exit_status: test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) finally: logging.debug("Recover test environment") # Clean VM on destination migration_test.cleanup_vm(vm, dest_uri) logging.info("Recover VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) if src_libvirt_file: src_libvirt_file.restore() if remote_libvirt_file: del remote_libvirt_file # Clean up of pre migration setup for local machine if migrate_vm_back: if 'ssh_connection' in locals(): ssh_connection.auto_recover = True migration_test.migrate_pre_setup(src_uri, params, cleanup=True) if remote_virsh_session: remote_virsh_session.close_session() logging.info("Remove local NFS image") source_file = params.get("source_file") libvirt.delete_local_disk("file", path=source_file) if guest_src_url and blk_source: libvirt.delete_local_disk("file", path=blk_source)
def run(test, params, env): """ Test storage migration 1) Do storage migration(copy-storage-all/copy-storage-inc) with TLS encryption - NBD transport 2) Cancel storage migration with TLS encryption 3) Copy only the top image for storage migration with backing chain :param test: test object :param params: Dictionary with the test parameters :param env: Dictionary with test environment. """ def prepare_nfs_backingfile(vm, params): """ Create an image using nfs type backing_file :param vm: The guest :param params: the parameters used """ mnt_path_name = params.get("nfs_mount_dir", "nfs-mount") exp_opt = params.get("export_options", "rw,no_root_squash,fsid=0") exp_dir = params.get("export_dir", "nfs-export") backingfile_img = params.get("source_dist_img", "nfs-img") disk_format = params.get("disk_format", "qcow2") img_name = params.get("img_name", "test.img") precreation = "yes" == params.get("precreation", "yes") vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) disk_xml = vmxml.devices.by_device_tag('disk')[0] src_disk_format = disk_xml.xmltreefile.find('driver').get('type') first_disk = vm.get_first_disk_devices() blk_source = first_disk['source'] disk_img = os.path.join(os.path.dirname(blk_source), img_name) res = libvirt.setup_or_cleanup_nfs(True, mnt_path_name, is_mount=True, export_options=exp_opt, export_dir=exp_dir) mnt_path = res["mount_dir"] params["selinux_status_bak"] = res["selinux_status_bak"] if vm.is_alive(): vm.destroy(gracefully=False) disk_cmd = ("qemu-img convert -f %s -O %s %s %s/%s" % (src_disk_format, disk_format, blk_source, mnt_path, backingfile_img)) process.run(disk_cmd, ignore_status=False, verbose=True) local_image_list.append("%s/%s" % (mnt_path, backingfile_img)) logging.debug("Create a local image backing on NFS.") disk_cmd = ("qemu-img create -f %s -b %s/%s %s" % (disk_format, mnt_path, backingfile_img, disk_img)) process.run(disk_cmd, ignore_status=False, verbose=True) local_image_list.append(disk_img) if precreation: logging.debug("Create an image backing on NFS on remote host.") remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, r'[$#%]') utils_misc.make_dirs(os.path.dirname(blk_source), remote_session) status, stdout = utils_misc.cmd_status_output( disk_cmd, session=remote_session) logging.debug("status: {}, stdout: {}".format(status, stdout)) remote_image_list.append("%s/%s" % (mnt_path, backingfile_img)) remote_image_list.append(disk_img) remote_session.close() params.update({'disk_source_name': disk_img, 'disk_type': 'file', 'disk_source_protocol': 'file'}) libvirt.set_vm_disk(vm, params) migration_test = migration.MigrationTest() migration_test.check_parameters(params) # Local variables server_ip = params["server_ip"] = params.get("remote_ip") server_user = params["server_user"] = params.get("remote_user", "root") server_pwd = params["server_pwd"] = params.get("remote_pwd") client_ip = params["client_ip"] = params.get("local_ip") client_pwd = params["client_pwd"] = params.get("local_pwd") virsh_options = params.get("virsh_options", "") copy_storage_option = params.get("copy_storage_option") extra = params.get("virsh_migrate_extra", "") options = params.get("virsh_migrate_options", "--live --verbose") backingfile_type = params.get("backingfile_type") check_str_local_log = params.get("check_str_local_log", "") disk_format = params.get("disk_format", "qcow2") log_file = params.get("log_outputs", "/var/log/libvirt/libvirtd.log") daemon_conf_dict = eval(params.get("daemon_conf_dict", '{}')) cancel_migration = "yes" == params.get("cancel_migration", "no") migrate_again = "yes" == params.get("migrate_again", "no") precreation = "yes" == params.get("precreation", "yes") tls_recovery = "yes" == params.get("tls_auto_recovery", "yes") func_params_exists = "yes" == params.get("func_params_exists", "no") status_error = "yes" == params.get("status_error", "no") local_image_list = [] remote_image_list = [] tls_obj = None func_name = None daemon_conf = None mig_result = None remote_session = None vm_session = None # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) dest_uri = params.get("virsh_migrate_desturi") vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() extra = "{} {}".format(extra, copy_storage_option) extra_args = {} if func_params_exists: extra_args.update({'func_params': params}) if cancel_migration: func_name = migration_test.do_cancel # For safety reasons, we'd better back up xmlfile. vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = vmxml.copy() try: if backingfile_type: if backingfile_type == "nfs": prepare_nfs_backingfile(vm, params) if extra.count("copy-storage-all") and precreation: blk_source = vm.get_first_disk_devices()['source'] vsize = utils_misc.get_image_info(blk_source).get("vsize") remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, r'[$#%]') utils_misc.make_dirs(os.path.dirname(blk_source), remote_session) disk_cmd = ("qemu-img create -f %s %s %s" % (disk_format, blk_source, vsize)) status, stdout = utils_misc.cmd_status_output( disk_cmd, session=remote_session) logging.debug("status: {}, stdout: {}".format(status, stdout)) remote_image_list.append(blk_source) remote_session.close() # Update libvirtd configuration if daemon_conf_dict: if os.path.exists(log_file): os.remove(log_file) daemon_conf = libvirt.customize_libvirt_config(daemon_conf_dict) if extra.count("--tls"): tls_obj = TLSConnection(params) if tls_recovery: tls_obj.auto_recover = True tls_obj.conn_setup() if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check local guest network connection before migration vm_session = vm.wait_for_login(restart_network=True) migration_test.ping_vm(vm, params) # Execute migration process vms = [vm] migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, extra_opts=extra, func=func_name, **extra_args) mig_result = migration_test.ret migration_test.check_result(mig_result, params) if migrate_again and status_error: logging.debug("Sleeping 10 seconds before rerun migration") time.sleep(10) if cancel_migration: func_name = None params["status_error"] = "no" migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, extra_opts=extra, func=func_name, **extra_args) mig_result = migration_test.ret migration_test.check_result(mig_result, params) if int(mig_result.exit_status) == 0: migration_test.ping_vm(vm, params, uri=dest_uri) if check_str_local_log: libvirt.check_logfile(check_str_local_log, log_file) finally: logging.debug("Recover test environment") # Clean VM on destination and source try: migration_test.cleanup_dest_vm(vm, vm.connect_uri, dest_uri) if vm.is_alive(): vm.destroy(gracefully=False) except Exception as err: logging.error(err) logging.info("Recovery VM XML configration") orig_config_xml.sync() if daemon_conf: logging.debug("Recover the configurations") libvirt.customize_libvirt_config(None, is_recover=True, config_object=daemon_conf) if tls_obj: logging.debug("Clean up local objs") del tls_obj for source_file in local_image_list: libvirt.delete_local_disk("file", path=source_file) for img in remote_image_list: remote.run_remote_cmd("rm -rf %s" % img, params) if remote_session: remote_session.close()
def run(test, params, env): """ Run the test :param test: test object :param params: Dictionary with the test parameters :param env: Dictionary with test environment. """ libvirt_version.is_libvirt_feature_supported(params) vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() bk_uri = vm.connect_uri migration_test = migration.MigrationTest() migration_test.check_parameters(params) extra_args = migration_test.update_virsh_migrate_extra_args(params) extra = params.get("virsh_migrate_extra") postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (extra, postcopy_options) params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) dest_uri = params.get("virsh_migrate_desturi") options = params.get("virsh_migrate_options", "--live --p2p --persistent --verbose") virsh_options = params.get("virsh_options", "") stress_package = params.get("stress_package") action_during_mig = params.get("action_during_mig") migrate_speed = params.get("migrate_speed") migrate_speed_again = params.get("migrate_speed_again") migrate_again = "yes" == params.get("migrate_again", "no") vm_state_after_abort = params.get("vm_state_after_abort") return_port = "yes" == params.get("return_port", "no") params['server_pwd'] = params.get("migrate_dest_pwd") params['server_ip'] = params.get("migrate_dest_host") params['server_user'] = params.get("remote_user", "root") is_storage_migration = True if extra.count('--copy-storage-all') else False setup_tls = "yes" == params.get("setup_tls", "no") qemu_conf_dest = params.get("qemu_conf_dest", "{}") migrate_tls_force_default = "yes" == params.get("migrate_tls_force_default", "no") poweroff_src_vm = "yes" == params.get("poweroff_src_vm", "no") check_port = "yes" == params.get("check_port", "no") server_ip = params.get("migrate_dest_host") server_user = params.get("remote_user", "root") server_pwd = params.get("migrate_dest_pwd") server_params = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} qemu_conf_list = eval(params.get("qemu_conf_list", "[]")) qemu_conf_path = params.get("qemu_conf_path") min_port = params.get("min_port") vm_session = None qemu_conf_remote = None (remove_key_local, remove_key_remote) = (None, None) # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() local_both_conf_obj = None remote_file_list = [] conn_obj_list = [] try: # Setup default value for migrate_tls_force if migrate_tls_force_default: value_list = ["migrate_tls_force"] # Setup migrate_tls_force default value on remote server_params['file_path'] = "/etc/libvirt/qemu.conf" remove_key_remote = libvirt_config.remove_key_in_conf(value_list, "qemu", remote_params=server_params) # Setup migrate_tls_force default value on local remove_key_local = libvirt_config.remove_key_in_conf(value_list, "qemu") if check_port: server_params['file_path'] = qemu_conf_path remove_key_remote = libvirt_config.remove_key_in_conf(qemu_conf_list, "qemu", remote_params=server_params) # Update only remote qemu conf if qemu_conf_dest: qemu_conf_remote = libvirt_remote.update_remote_file( server_params, qemu_conf_dest, "/etc/libvirt/qemu.conf") # Update local or both sides configuration files local_both_conf_obj = update_local_or_both_conf_file(params) # Setup TLS if setup_tls: conn_obj_list.append(migration_base.setup_conn_obj('tls', params, test)) # Update guest disk xml if not is_storage_migration: libvirt.set_vm_disk(vm, params) else: remote_file_list.append(libvirt_disk.create_remote_disk_by_same_metadata(vm, params)) if check_port: # Create a remote runner runner_on_target = remote_old.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) cmd = "nc -l -p %s &" % min_port remote_old.run_remote_cmd(cmd, params, runner_on_target, ignore_status=False) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) vm_session = vm.wait_for_login() if action_during_mig: if poweroff_src_vm: params.update({'vm_session': vm_session}) action_during_mig = migration_base.parse_funcs(action_during_mig, test, params) if stress_package: migration_test.run_stress_in_vm(vm, params) mode = 'both' if '--postcopy' in postcopy_options else 'precopy' if migrate_speed: migration_test.control_migrate_speed(vm_name, int(migrate_speed), mode) # Execute migration process migration_base.do_migration(vm, migration_test, None, dest_uri, options, virsh_options, extra, action_during_mig, extra_args) func_returns = dict(migration_test.func_ret) migration_test.func_ret.clear() logging.debug("Migration returns function results:%s", func_returns) if return_port: port_used = get_used_port(func_returns) if check_port: port_used = get_used_port(func_returns) if int(port_used) != int(min_port) + 1: test.fail("Wrong port for migration.") if vm_state_after_abort: check_vm_state_after_abort(vm_name, vm_state_after_abort, bk_uri, dest_uri, test) if migrate_again: if not vm.is_alive(): vm.start() vm_session = vm.wait_for_login() action_during_mig = migration_base.parse_funcs(params.get('action_during_mig_again'), test, params) extra_args['status_error'] = params.get("migrate_again_status_error", "no") if params.get("virsh_migrate_extra_mig_again"): extra = params.get("virsh_migrate_extra_mig_again") if params.get('scp_list_client_again'): params['scp_list_client'] = params.get('scp_list_client_again') # Recreate tlsconnection object using new parameter values conn_obj_list.append(migration_base.setup_conn_obj('tls', params, test)) if migrate_speed_again: migration_test.control_migrate_speed(vm_name, int(migrate_speed_again), mode) migration_base.do_migration(vm, migration_test, None, dest_uri, options, virsh_options, extra, action_during_mig, extra_args) if return_port: func_returns = dict(migration_test.func_ret) logging.debug("Migration returns function " "results:%s", func_returns) port_second = get_used_port(func_returns) if port_used != port_second: test.fail("Expect same port '{}' is used as previous one, " "but found new one '{}'".format(port_used, port_second)) else: logging.debug("Same port '%s' was used as " "expected", port_second) if int(migration_test.ret.exit_status) == 0: migration_test.post_migration_check([vm], params, uri=dest_uri) finally: logging.info("Recover test environment") vm.connect_uri = bk_uri if vm_session: vm_session.close() # Clean VM on destination and source migration_test.cleanup_vm(vm, dest_uri) # Restore remote qemu conf and restart libvirtd if qemu_conf_remote: logging.debug("Recover remote qemu configurations") del qemu_conf_remote # Restore local or both sides conf and restart libvirtd recover_config_file(local_both_conf_obj, params) if remove_key_remote: del remove_key_remote if remove_key_local: libvirt.customize_libvirt_config(None, config_object=remove_key_local, config_type='qemu') # Clean up connection object, like TLS migration_base.cleanup_conn_obj(conn_obj_list, test) for one_file in remote_file_list: if one_file: remote_old.run_remote_cmd("rm -rf %s" % one_file, params) orig_config_xml.sync()
def check_dest_stats(vm_name): """domstats in target""" cmd = "virsh domstats {}".format(vm_name) dest_stats = remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) logging.debug("domstats in destination: {}".format(dest_stats))
def run(test, params, env): """ Test migration over unix socket. 1) Migrate vm over unix socket 2) Migrate vm over unix socket - libvirt tunnelled(--tunnelled) 3) Migrate vm over unix socket - enable multifd(--parallel) 4) Migrate vm with copy storage over unix socket - one disk 5) Migrate vm with copy storage over unix socket - multiple disks 6) Abort migration over unix socket and migrate again 7) Abort migration with copy storage over unix socket, and migrate again 8) Migrate vm with copy storage over unix socket - multiple disks - enable multifd(--parallel) :param test: test object :param params: Dictionary with the test parameters :param env: Dictionary with test environment. """ def update_disk(vm, params): """ Update disk for testing. :param vm: vm object. :param params: the parameters used. :return: updated images. """ local_image_list = [] remote_image_list = [] vm_did_list = [] # Change the disk of the vm if storage_type == "nfs": libvirt.set_vm_disk(vm, params) else: disk_format = params.get("disk_format", "qcow2") disk_num = eval(params.get("disk_num", "1")) blk_source = vm.get_first_disk_devices()['source'] vsize = utils_misc.get_image_info(blk_source).get("vsize") remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, r'[$#%]') # Create disk on remote host utils_misc.make_dirs(os.path.dirname(blk_source), remote_session) libvirt_disk.create_disk("file", disk_format=disk_format, path=blk_source, size=vsize, session=remote_session) remote_image_list.append(blk_source) for idx in range(2, disk_num + 1): disk_path = os.path.join(os.path.dirname(blk_source), "test%s.img" % str(idx)) # Create disk on local libvirt_disk.create_disk("file", disk_format=disk_format, path=disk_path) local_image_list.append(disk_path) target_dev = 'vd' + chr(idx + ord('a') - 1) new_disk_dict = {"driver_type": disk_format} vm_was_running = vm.is_alive() libvirt_pcicontr.reset_pci_num(vm_name) if vm_was_running and not vm.is_alive(): vm.start() vm.wait_for_login().close() result = libvirt.attach_additional_device( vm_name, target_dev, disk_path, new_disk_dict, False) libvirt.check_exit_status(result) libvirt_disk.create_disk("file", disk_format=disk_format, path=disk_path, session=remote_session) remote_image_list.append(disk_path) vm_did_list.append(target_dev) remote_session.close() return local_image_list, remote_image_list, vm_did_list def check_socket(params): """ Check sockets' number :param params: the parameters used :raise: test.fail when command fails """ postcopy_options = params.get("postcopy_options") vm_name = params.get("migrate_main_vm") exp_num = params.get("expected_socket_num", "2") if postcopy_options: migration_test.set_migratepostcopy(vm_name) cmd = ("netstat -xanp|grep -E \"CONNECTED" ".*(desturi-socket|migrateuri-socket)\" | wc -l") res = process.run(cmd, shell=True).stdout_text.strip() if res != exp_num: test.fail("There should be {} connected unix sockets, " "but found {} sockets.".format(exp_num, res)) migration_test = migration.MigrationTest() migration_test.check_parameters(params) # Local variables virsh_args = {"debug": True} server_ip = params["server_ip"] = params.get("remote_ip") server_user = params["server_user"] = params.get("remote_user", "root") server_pwd = params["server_pwd"] = params.get("remote_pwd") client_ip = params["client_ip"] = params.get("local_ip") client_pwd = params["client_pwd"] = params.get("local_pwd") virsh_options = params.get("virsh_options", "") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options", "--live --p2p --verbose") storage_type = params.get("storage_type") disk_num = params.get("disk_num") desturi_port = params.get("desturi_port", "22222") migrateuri_port = params.get("migrateuri_port", "33333") disks_uri_port = params.get("disks_uri_port", "44444") migrate_again = "yes" == params.get("migrate_again", "no") action_during_mig = params.get("action_during_mig") if action_during_mig: action_during_mig = eval(action_during_mig) extra_args = migration_test.update_virsh_migrate_extra_args(params) mig_result = None local_image_list = [] remote_image_list = [] vm_did_list = [] if not libvirt_version.version_compare(6, 6, 0): test.cancel("This libvirt version doesn't support " "migration over unix.") if storage_type == "nfs": # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # params for migration connection params["virsh_migrate_connect_uri"] = libvirt_vm.complete_uri( params.get("migrate_source_host")) src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") dest_uri_ssh = libvirt_vm.complete_uri(params.get("migrate_dest_host")) unix_obj = None vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() bk_uri = vm.connect_uri postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (extra, postcopy_options) # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() try: unix_obj = utils_conn.UNIXSocketConnection(params) unix_obj.conn_setup() unix_obj.auto_recover = True local_image_list, remote_image_list, vm_did_list = update_disk( vm, params) if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) vm_session = vm.wait_for_login() for did in vm_did_list: utils_disk.linux_disk_check(vm_session, did) # Execute migration process vms = [vm] migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=600, ignore_status=True, virsh_opt=virsh_options, func=action_during_mig, extra_opts=extra, **extra_args) mig_result = migration_test.ret if migrate_again: logging.debug( "Sleeping 10 seconds before rerunning the migration.") time.sleep(10) if params.get("migrate_again_clear_func", "yes") == "yes": action_during_mig = None extra_args["status_error"] = params.get( "migrate_again_status_error", "no") migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, extra_opts=extra, func=action_during_mig, **extra_args) mig_result = migration_test.ret if int(mig_result.exit_status) == 0: vm.connect_uri = dest_uri_ssh if not utils_misc.wait_for( lambda: virsh.is_alive( vm_name, uri=dest_uri_ssh, debug=True), 60): test.fail("The migrated VM should be alive!") if vm_did_list: vm_session_after_mig = vm.wait_for_serial_login(timeout=240) for did in vm_did_list: vm_session_after_mig.cmd("echo mytest >> /mnt/%s1/mytest" % did) finally: logging.info("Recover test environment") vm.connect_uri = bk_uri # Clean VM on destination and source migration_test.cleanup_vm(vm, dest_uri) orig_config_xml.sync() # Remove image files for source_file in local_image_list: libvirt.delete_local_disk("file", path=source_file) for img in remote_image_list: remote.run_remote_cmd("rm -rf %s" % img, params)
def check_domjobinfo_output(option="", is_mig_compelete=False): """ Check all items in domjobinfo of the guest on both remote and local :param option: options for domjobinfo :param is_mig_compelete: False for domjobinfo checking during migration, True for domjobinfo checking after migration :raise: test.fail if the value of given item is unexpected """ expected_list_during_mig = ["Job type", "Operation", "Time elapsed", "Data processed", "Data remaining", "Data total", "Memory processed", "Memory remaining", "Memory total", "Memory bandwidth", "Dirty rate", "Page size", "Iteration", "Constant pages", "Normal pages", "Normal data", "Expected downtime", "Setup time"] if libvirt_version.version_compare(4, 10, 0): expected_list_during_mig.insert(13, "Postcopy requests") expected_list_after_mig_src = copy.deepcopy(expected_list_during_mig) expected_list_after_mig_src[-2] = 'Total downtime' expected_list_after_mig_dest = copy.deepcopy(expected_list_after_mig_src) # Check version in remote if not expected_list_after_mig_dest.count("Postcopy requests"): remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, "#") if libvirt_version.version_compare(4, 10, 0, session=remote_session): expected_list_after_mig_dest.insert(14, "Postcopy requests") remote_session.close() expect_dict = {"src_notdone": {"Job type": "Unbounded", "Operation": "Outgoing migration", "all_items": expected_list_during_mig}, "dest_notdone": {"error": "Operation not supported: mig" "ration statistics are availab" "le only on the source host"}, "src_done": {"Job type": "Completed", "Operation": "Outgoing migration", "all_items": expected_list_after_mig_src}, "dest_done": {"Job type": "Completed", "Operation": "Incoming migration", "all_items": expected_list_after_mig_dest}} pc_opt = False if postcopy_options: pc_opt = True if is_mig_compelete: expect_dict["dest_done"].clear() expect_dict["dest_done"]["Job type"] = "None" else: set_migratepostcopy() vm_ref = '{}{}'.format(vm_name, option) src_jobinfo = virsh.domjobinfo(vm_ref, **virsh_args) cmd = "virsh domjobinfo {} {}".format(vm_name, option) dest_jobinfo = remote.run_remote_cmd(cmd, cmd_parms, runner_on_target) if not is_mig_compelete: search_jobinfo_output(src_jobinfo.stdout, expect_dict["src_notdone"], postcopy_req=pc_opt) search_jobinfo_output(dest_jobinfo.stderr, expect_dict["dest_notdone"]) else: search_jobinfo_output(src_jobinfo.stdout, expect_dict["src_done"]) search_jobinfo_output(dest_jobinfo.stdout, expect_dict["dest_done"])
def run(test, params, env): """ Test migration with memory related configuration :param test: test object :param params: Dictionary with the test parameters :param env: Dictionary with test environment. """ migrate_vm_back = "yes" == params.get("migrate_vm_back", "no") remote_ip = params.get("remote_ip") remote_user = params.get("remote_user") remote_pwd = params.get("remote_pwd") local_ip = params.get("local_ip") local_pwd = params.get("local_pwd") ballooned_mem = params.get("ballooned_mem") check = params.get("check") remove_dict = {} src_libvirt_file = None remote_virsh_dargs = { 'remote_ip': remote_ip, 'remote_user': remote_user, 'remote_pwd': remote_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True } migration_test = migration.MigrationTest() migration_test.check_parameters(params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") virsh_args = {"debug": True} virsh_options = params.get("virsh_options", "") options = params.get("virsh_migrate_options", "--live --verbose") func_params_exists = "yes" == params.get("func_params_exists", "yes") log_file = params.get("log_outputs", "/var/log/libvirt/libvirtd_daemons.log") check_str_local_log = params.get("check_str_local_log", "") libvirtd_conf_dict = eval(params.get("libvirtd_conf_dict", '{}')) func_name = None libvirtd_conf = None mig_result = None # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) dest_uri = params.get("virsh_migrate_desturi") vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() extra_args = {} if func_params_exists: extra_args.update({'func_params': params}) # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() try: if check == "mem_balloon": # Update memory balloon device to correct model membal_dict = { 'membal_model': 'virtio', 'membal_stats_period': '10' } libvirt.update_memballoon_xml(new_xml, membal_dict) if check == "mem_device": libvirt_cpu.add_cpu_settings(new_xml, params) dimm_params = { k.replace('memdev_', ''): v for k, v in params.items() if k.startswith('memdev_') } dimm_xml = utils_hotplug.create_mem_xml(**dimm_params) libvirt.add_vm_device(new_xml, dimm_xml) logging.debug(virsh.dumpxml(vm_name)) # Change the disk of the vm libvirt.set_vm_disk(vm, params) remove_dict = {"do_search": '{"%s": "ssh:/"}' % dest_uri} src_libvirt_file = libvirt_config.remove_key_for_modular_daemon( remove_dict) # Update libvirtd configuration if libvirtd_conf_dict: if os.path.exists(log_file): logging.debug("Delete local libvirt log file '%s'", log_file) os.remove(log_file) logging.debug("Update libvirtd configuration file") conf_type = params.get("conf_type", "libvirtd") if conf_type == "libvirtd" and utils_split_daemons.is_modular_daemon( ): conf_type = "virtqemud" libvirtd_conf = libvirt.customize_libvirt_config( libvirtd_conf_dict, conf_type) try: if not vm.is_alive(): vm.start() except virt_vm.VMStartError as e: logging.info("Failed to start VM") test.fail("Failed to start VM: %s" % vm_name) logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check local guest network connection before migration vm.wait_for_login(restart_network=True).close() migration_test.ping_vm(vm, params) # Execute migration process vms = [vm] migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, func=func_name, **extra_args) mig_result = migration_test.ret migration_test.check_result(mig_result, params) if int(mig_result.exit_status) == 0: migration_test.ping_vm(vm, params, uri=dest_uri) if check_str_local_log: libvirt.check_logfile(check_str_local_log, log_file) if check == "mem_balloon": remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) remote_virsh_session.setmem(vm_name, ballooned_mem, None, None, False, "", **virsh_args) def check_mem_balloon(): """Check if memory balloon worked""" memstat_ouput = remote_virsh_session.dommemstat( vm_name, "", **virsh_args) memstat_after = memstat_ouput.stdout_text mem_after = memstat_after.splitlines()[0].split()[1] if mem_after != ballooned_mem: logging.debug("Current memory size is: %s" % mem_after) return False return True check_ret = utils_misc.wait_for(check_mem_balloon, timeout=20) if not check_ret: test.fail("Memory is not ballooned to the expected size: %s" % ballooned_mem) remote_virsh_session.close_session() # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=remote_ip, username=remote_user, password=remote_pwd) if check == "mem_device": qemu_checks = params.get('qemu_checks', '').split('`') logging.debug("qemu_checks:%s" % qemu_checks[0]) for qemu_check in qemu_checks: libvirt.check_qemu_cmd_line(qemu_check, False, params, runner_on_target) if migrate_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=remote_ip, server_pwd=remote_pwd, client_ip=local_ip, client_pwd=local_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine src_full_uri = libvirt_vm.complete_uri( params.get("migrate_source_host")) migration_test.migrate_pre_setup(src_full_uri, params) cmd = "virsh migrate %s %s %s" % (vm_name, options, src_full_uri) logging.debug("Start migration: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) logging.info(cmd_result) if cmd_result.exit_status: destroy_cmd = "virsh destroy %s" % vm_name remote.run_remote_cmd(destroy_cmd, params, runner_on_target, ignore_status=False) test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) finally: logging.debug("Recover test environment") # Clean VM on destination and source try: migration_test.cleanup_dest_vm(vm, vm.connect_uri, dest_uri) if vm.is_alive(): vm.destroy(gracefully=False) except Exception as err: logging.error(err) logging.info("Recovery VM XML configration") orig_config_xml.sync() if libvirtd_conf: logging.debug("Recover the configurations") libvirt.customize_libvirt_config(None, is_recover=True, config_object=libvirtd_conf) if src_libvirt_file: src_libvirt_file.restore()
def run(test, params, env): """ Test virsh migrate command. """ def check_vm_network_accessed(session=None, ping_dest="www.baidu.com"): """ The operations to the VM need to be done before or after migration happens :param session: The session object to the host :param ping_dest: The destination to be ping :raise: test.fail when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") status, output = utils_test.ping(ping_dest, count=10, timeout=20, output_func=logging.debug, session=session) if status != 0: test.fail("Ping failed, status: %s," " output: %s" % (status, output)) def get_vm_ifaces(session=None): """ Get interfaces of vm :param session: The session object to the host :return: interfaces """ p_iface, v_iface = utils_net.get_remote_host_net_ifs(session) return p_iface def check_vm_iface_num(iface_list, exp_num=3): """ Check he number of interfaces :param iface_list: The interface list :param exp_num: The expected number :raise: test.fail when interfaces' number is not equal to exp_num """ if len(iface_list) != exp_num: test.fail("%d interfaces should be found on the vm, " "but find %s." % (exp_num, iface_list)) def create_or_del_networks(pf_name, params, remote_virsh_session=None, is_del=False): """ Create or delete network on local or remote :param params: Dictionary with the test parameters :param pf_name: The name of PF :param remote_virsh_session: The virsh session object to the remote host :param is_del: Whether the networks should be deleted :raise: test.fail when fails to define/start network """ net_hostdev_name = params.get("net_hostdev_name", "hostdev-net") net_hostdev_fwd = params.get("net_hostdev_fwd", '{"mode": "hostdev", "managed": "yes"}') net_bridge_name = params.get("net_bridge_name", "host-bridge") net_bridge_fwd = params.get("net_bridge_fwd", '{"mode": "bridge"}') bridge_name = params.get("bridge_name", "br0") net_dict = {"net_name": net_hostdev_name, "net_forward": net_hostdev_fwd, "net_forward_pf": '{"dev": "%s"}' % pf_name} bridge_dict = {"net_name": net_bridge_name, "net_forward": net_bridge_fwd, "net_bridge": '{"name": "%s"}' % bridge_name} if not is_del: for net_params in (net_dict, bridge_dict): net_dev = libvirt.create_net_xml(net_params.get("net_name"), net_params) if not remote_virsh_session: if net_dev.get_active(): net_dev.undefine() net_dev.define() net_dev.start() else: remote.scp_to_remote(server_ip, '22', server_user, server_pwd, net_dev.xml, net_dev.xml, limit="", log_filename=None, timeout=600, interface=None) remote_virsh_session.net_define(net_dev.xml, **virsh_args) remote_virsh_session.net_start(net_params.get("net_name"), **virsh_args) else: virsh_session = virsh if remote_virsh_session: virsh_session = remote_virsh_session for nname in (net_hostdev_name, net_bridge_name): if nname not in virsh_session.net_state_dict(): continue virsh_session.net_destroy(nname, debug=True, ignore_status=True) virsh_session.net_undefine(nname, debug=True, ignore_status=True) def check_vm_network_connection(net_name, expected_conn=0): """ Check network connections in network xml :param net_name: The network to be checked :param expected_conn: The expected value :raise: test.fail when fails """ output = virsh.net_dumpxml(net_name, debug=True).stdout_text if expected_conn == 0: reg_pattern = r"<network>" else: reg_pattern = r"<network connections='(\d)'>" res = re.findall(reg_pattern, output, re.I) if not res: test.fail("Unable to find expected connection in %s." % net_name) if expected_conn != 0: if expected_conn != int(res[0]): test.fail("Unable to get expected connection number." "Expected: %s, Actual %s" % (expected_conn, int(res[0]))) def get_hostdev_addr_from_xml(): """ Get VM hostdev address :return: pci driver id """ address_dict = {} for ifac in vm_xml.VMXML.new_from_dumpxml(vm_name).devices.by_device_tag("interface"): if ifac.type_name == "hostdev": address_dict = ifac.hostdev_address.attrs return libvirt.pci_info_from_address(address_dict, 16, "id") def check_vfio_pci(pci_path, status_error=False): """ Check if vf driver is vfio-pci :param pci_path: The absolute path of pci device :param status_error: Whether the driver should be vfio-pci """ cmd = "readlink %s/driver | awk -F '/' '{print $NF}'" % pci_path output = process.run(cmd, shell=True, verbose=True).stdout_text.strip() if (output == "vfio-pci") == status_error: test.fail("Get incorrect dirver %s, it should%s be vfio-pci." % (output, ' not' if status_error else '')) def update_iface_xml(vmxml): """ Update interfaces for guest :param vmxml: vm_xml.VMXML object """ vmxml.remove_all_device_by_type('interface') vmxml.sync() iface_dict = {"type": "network", "source": "{'network': 'host-bridge'}", "mac": mac_addr, "model": "virtio", "teaming": '{"type":"persistent"}', "alias": '{"name": "ua-backup0"}', "inbound": '{"average":"5"}', "outbound": '{"average":"5"}'} iface_dict2 = {"type": "network", "source": "{'network': 'hostdev-net'}", "mac": mac_addr, "model": "virtio", "teaming": '{"type":"transient", "persistent": "ua-backup0"}'} iface = interface.Interface('network') for ifc in (iface_dict, iface_dict2): iface.xml = libvirt.modify_vm_iface(vm.name, "get_xml", ifc) vmxml.add_device(iface) vmxml.sync() migration_test = migration.MigrationTest() migration_test.check_parameters(params) # Params for NFS shared storage shared_storage = params.get("migrate_shared_storage", "") if shared_storage == "": default_guest_asset = defaults.get_default_guest_os_info()['asset'] default_guest_asset = "%s.qcow2" % default_guest_asset shared_storage = os.path.join(params.get("nfs_mount_dir"), default_guest_asset) logging.debug("shared_storage:%s", shared_storage) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} virsh_options = params.get("virsh_options", "") server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") client_ip = params.get("client_ip") client_pwd = params.get("client_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options") bridge_name = params.get("bridge_name", "br0") net_hostdev_name = params.get("net_hostdev_name", "hostdev-net") net_bridge_name = params.get("net_bridge_name", "host-bridge") driver = params.get("driver", "ixgbe") vm_tmp_file = params.get("vm_tmp_file", "/tmp/test.txt") cmd_during_mig = params.get("cmd_during_mig") net_failover_test = "yes" == params.get("net_failover_test", "no") cancel_migration = "yes" == params.get("cancel_migration", "no") try: vf_no = int(params.get("vf_no", "4")) except ValueError as e: test.error(e) migr_vm_back = "yes" == params.get("migrate_vm_back", "no") err_msg = params.get("err_msg") status_error = "yes" == params.get("status_error", "no") cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} remote_virsh_dargs = {'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True} destparams_dict = copy.deepcopy(params) remote_virsh_session = None vm_session = None vm = None mig_result = None func_name = None extra_args = {} default_src_vf = 0 default_dest_vf = 0 default_src_rp_filter = 1 default_dest_rp_filer = 1 if not libvirt_version.version_compare(6, 0, 0): test.cancel("This libvirt version doesn't support migration with " "net failover devices.") # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) params["virsh_migrate_connect_uri"] = libvirt_vm.complete_uri( params.get("migrate_source_host")) src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") if net_failover_test: src_pf, src_pf_pci = utils_sriov.find_pf(driver) logging.debug("src_pf is %s. src_pf_pci: %s", src_pf, src_pf_pci) params['pf_name'] = src_pf dest_pf, dest_pf_pci = utils_sriov.find_pf(driver, server_session) logging.debug("dest_pf is %s. dest_pf_pci: %s", dest_pf, dest_pf_pci) destparams_dict['pf_name'] = dest_pf src_pf_pci_path = utils_misc.get_pci_path(src_pf_pci) dest_pf_pci_path = utils_misc.get_pci_path(dest_pf_pci, server_session) cmd = "cat %s/sriov_numvfs" % (src_pf_pci_path) default_src_vf = process.run(cmd, shell=True, verbose=True).stdout_text cmd = "cat %s/sriov_numvfs" % (dest_pf_pci_path) status, default_dest_vf = utils_misc.cmd_status_output(cmd, shell=True, session=server_session) if status: test.error("Unable to get default sriov_numvfs on target!" "status: %s, output: %s" % (status, default_dest_vf)) if not utils_sriov.set_vf(src_pf_pci_path, vf_no): test.error("Failed to set vf on source.") if not utils_sriov.set_vf(dest_pf_pci_path, vf_no, session=server_session): test.error("Failed to set vf on target.") # Create PF and bridge connection on source and target host cmd = 'cat /proc/sys/net/ipv4/conf/all/rp_filter' default_src_rp_filter = process.run(cmd, shell=True, verbose=True).stdout_text status, default_dest_rp_filter = utils_misc.cmd_status_output(cmd, shell=True, session=server_session) if status: test.error("Unable to get default rp_filter on target!" "status: %s, output: %s" % (status, default_dest_rp_filter)) cmd = 'echo 0 >/proc/sys/net/ipv4/conf/all/rp_filter' process.run(cmd, shell=True, verbose=True) utils_misc.cmd_status_output(cmd, shell=True, session=server_session) utils_sriov.add_or_del_connection(params, is_del=False) utils_sriov.add_or_del_connection(destparams_dict, is_del=False, session=server_session) if not remote_virsh_session: remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) create_or_del_networks(dest_pf, params, remote_virsh_session=remote_virsh_session) remote_virsh_session.close_session() create_or_del_networks(src_pf, params) # Change network interface xml mac_addr = utils_net.generate_mac_address_simple() update_iface_xml(new_xml) # Change the disk of the vm libvirt.set_vm_disk(vm, params) if not vm.is_alive(): vm.start() # Check local guest network connection before migration if vm.serial_console is not None: vm.cleanup_serial_console() vm.create_serial_console() vm_session = vm.wait_for_serial_login(timeout=240) if net_failover_test: utils_net.restart_guest_network(vm_session) iface_list = get_vm_ifaces(vm_session) vm_ipv4, vm_ipv6 = utils_net.get_linux_ipaddr(vm_session, iface_list[0]) check_vm_network_accessed(ping_dest=vm_ipv4) if net_failover_test: check_vm_iface_num(iface_list) check_vm_network_connection(net_hostdev_name, 1) check_vm_network_connection(net_bridge_name, 1) hostdev_pci_id = get_hostdev_addr_from_xml() vf_path = utils_misc.get_pci_path(hostdev_pci_id) check_vfio_pci(vf_path) if cmd_during_mig: s, o = utils_misc.cmd_status_output(cmd_during_mig, shell=True, session=vm_session) if s: test.fail("Failed to run %s in vm." % cmd_during_mig) if extra.count("--postcopy"): func_name = virsh.migrate_postcopy extra_args.update({'func_params': params}) if cancel_migration: func_name = migration_test.do_cancel # Execute migration process vms = [vm] migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, func=func_name, extra_opts=extra, **extra_args) mig_result = migration_test.ret migration_test.check_result(mig_result, params) if int(mig_result.exit_status) == 0: server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") check_vm_network_accessed(server_session, vm_ipv4) server_session.close() if net_failover_test: # Check network connection check_vm_network_connection(net_hostdev_name) check_vm_network_connection(net_bridge_name) # VF driver should not be vfio-pci check_vfio_pci(vf_path, True) cmd_parms.update({'vm_ip': vm_ipv4, 'vm_pwd': params.get("password")}) vm_after_mig = remote.VMManager(cmd_parms) vm_after_mig.setup_ssh_auth() cmd = "ip link" cmd_result = vm_after_mig.run_command(cmd) libvirt.check_result(cmd_result) p_iface = re.findall(r"\d+:\s+(\w+):\s+.*", cmd_result.stdout_text) p_iface = [x for x in p_iface if x != 'lo'] check_vm_iface_num(p_iface) # Check the output of ping command cmd = 'cat %s' % vm_tmp_file cmd_result = vm_after_mig.run_command(cmd) libvirt.check_result(cmd_result) if re.findall('Destination Host Unreachable', cmd_result.stdout_text, re.M): test.fail("The network does not work well during " "the migration peirod. ping output: %s" % cmd_result.stdout_text) # Execute migration from remote if migr_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=client_ip, server_pwd=client_pwd, client_ip=server_ip, client_pwd=server_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine migration_test.migrate_pre_setup(src_uri, params) cmd = "virsh migrate %s %s %s" % (vm_name, virsh_options, src_uri) logging.debug("Start migration: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) logging.info(cmd_result) if cmd_result.exit_status: test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) logging.debug("migration back done") check_vm_network_accessed(ping_dest=vm_ipv4) if net_failover_test: if vm_session: vm_session.close() vm_session = vm.wait_for_login() iface_list = get_vm_ifaces(vm_session) check_vm_iface_num(iface_list) else: check_vm_network_accessed(ping_dest=vm_ipv4) if net_failover_test: iface_list = get_vm_ifaces(vm_session) check_vm_iface_num(iface_list) finally: logging.debug("Recover test environment") # Clean VM on destination migration_test.cleanup_dest_vm(vm, vm.connect_uri, dest_uri) if vm.is_alive(): vm.destroy(gracefully=False) logging.info("Recovery VM XML configration") orig_config_xml.sync() logging.debug("The current VM XML:\n%s", orig_config_xml.xmltreefile) server_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") if 'src_pf' in locals(): cmd = 'echo %s >/proc/sys/net/ipv4/conf/all/rp_filter' % default_src_rp_filter process.run(cmd, shell=True, verbose=True) utils_sriov.add_or_del_connection(params, is_del=True) create_or_del_networks(src_pf, params, is_del=True) if 'dest_pf' in locals(): cmd = 'echo %s >/proc/sys/net/ipv4/conf/all/rp_filter' % default_dest_rp_filter utils_misc.cmd_status_output(cmd, shell=True, session=server_session) utils_sriov.add_or_del_connection(destparams_dict, session=server_session, is_del=True) remote_virsh_session = virsh.VirshPersistent(**remote_virsh_dargs) create_or_del_networks(dest_pf, params, remote_virsh_session, is_del=True) remote_virsh_session.close_session() if 'dest_pf_pci_path' in locals() and default_dest_vf != vf_no: utils_sriov.set_vf(dest_pf_pci_path, default_dest_vf, server_session) if 'src_pf_pci_path' in locals() and default_src_vf != vf_no: utils_sriov.set_vf(src_pf_pci_path, default_src_vf) # Clean up of pre migration setup for local machine if migr_vm_back: if 'ssh_connection' in locals(): ssh_connection.auto_recover = True migration_test.migrate_pre_setup(src_uri, params, cleanup=True) server_session.close() if remote_virsh_session: remote_virsh_session.close_session() logging.info("Remove local NFS image") source_file = params.get("source_file") if source_file: libvirt.delete_local_disk("file", path=source_file)
def run(test, params, env): """ Run the test :param test: test object :param params: Dictionary with the test parameters :param env: Dictionary with test environment. """ libvirt_version.is_libvirt_feature_supported(params) vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() bk_uri = vm.connect_uri migration_test = migration.MigrationTest() migration_test.check_parameters(params) extra_args = migration_test.update_virsh_migrate_extra_args(params) extra = params.get("virsh_migrate_extra") postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (extra, postcopy_options) params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) dest_uri = params.get("virsh_migrate_desturi") options = params.get("virsh_migrate_options", "--live --p2p --persistent --verbose") virsh_options = params.get("virsh_options", "") migrate_again = "yes" == params.get("migrate_again", "no") src_state = params.get("virsh_migrate_src_state", "shut off") set_src_and_dst_hostname = "yes" == params.get("set_src_and_dst_hostname", "no") src_hostname = params.get("src_hostname") dst_hostname = params.get("dst_hostname") server_ip = params.get("remote_ip") server_user = params.get("remote_user", "root") server_pwd = params.get("remote_pwd") server_params = { 'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd } dst_session = None dst_libvirtd = None src_libvirtd = None src_NM_service = None dest_NM_service = None old_dst_hostname = None old_source_hostname = None # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() try: # Update guest disk xml libvirt.set_vm_disk(vm, params) if set_src_and_dst_hostname: old_dst_hostname = get_hostname(test, remote_params=server_params) old_source_hostname = get_hostname(test) dst_session = remote_old.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") dst_libvirtd = utils_libvirtd.Libvirtd(session=dst_session) src_libvirtd = utils_libvirtd.Libvirtd() try: # Check local NetworkManager service NM = utils_path.find_command("NetworkManager") except utils_path.CmdNotFoundError: logging.debug("No NetworkManager command found on source.") NM = None if NM is not None: # Stop local NetworkManager service src_NM_service = service.Factory.create_service( "NetworkManager") if src_NM_service is not None: stop_NM(src_NM_service) set_hostname(src_hostname, test) src_libvirtd.restart() # Check remote NetworkManager service check_cmd = "rpm -q NetworkManager" check_result = remote_old.run_remote_cmd(check_cmd, server_params, ignore_status=False) if check_result.exit_status: logging.debug("No NetworkManager command found on target.") else: # Stop remote NetworkManager service remote_runner = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) runner = remote_runner.run dest_NM_service = service.Factory.create_service( "NetworkManager", run=runner) if dest_NM_service is not None: stop_NM(dest_NM_service) set_hostname(dst_hostname, test, remote_params=server_params) dst_libvirtd.restart() if not vm.is_alive(): vm.start() logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) vm.wait_for_login() # Execute migration process migration_base.do_migration(vm, migration_test, None, dest_uri, options, virsh_options, extra, None, extra_args) func_returns = dict(migration_test.func_ret) migration_test.func_ret.clear() logging.debug("Migration returns function results:%s", func_returns) if migrate_again: if not vm.is_alive(): vm.start() vm.wait_for_login() extra_args['status_error'] = params.get( "migrate_again_status_error", "no") if params.get("virsh_migrate_extra_mig_again"): extra = params.get("virsh_migrate_extra_mig_again") migration_base.do_migration(vm, migration_test, None, dest_uri, options, virsh_options, extra, None, extra_args) if int(migration_test.ret.exit_status) == 0: migration_test.post_migration_check([vm], params, uri=dest_uri) if not libvirt.check_vm_state(vm_name, state=src_state, uri=bk_uri): test.fail("Can't get the expected vm state '%s'" % src_state) finally: logging.info("Recover test environment") vm.connect_uri = bk_uri # Clean VM on destination and source migration_test.cleanup_vm(vm, dest_uri) if set_src_and_dst_hostname: if src_NM_service is not None: start_NM(src_NM_service) if dest_NM_service is not None: start_NM(dest_NM_service) set_hostname(old_dst_hostname, test, remote_params=server_params) if dst_libvirtd: dst_libvirtd.restart() if dst_session: dst_session.close() set_hostname(old_source_hostname, test) if src_libvirtd: src_libvirtd.restart() orig_config_xml.sync()
def run(test, params, env): """ Test migration with glusterfs. """ def create_or_clean_backend_dir(g_uri, params, session=None, is_clean=False): """ Create/cleanup backend directory :params g_uri: glusterfs uri :params params: the parameters to be checked :params session: VM/remote session object :params is_cleanup: True for cleanup backend directory; False for create one. :return: gluster_img if is_clean is equal to True """ mount_point = params.get("gluster_mount_dir") is_symlink = params.get("gluster_create_symlink") == "yes" symlink_name = params.get("gluster_symlink") gluster_img = None if not is_clean: if not utils_misc.check_exists(mount_point, session): utils_misc.make_dirs(mount_point, session) if gluster.glusterfs_is_mounted(mount_point, session): gluster.glusterfs_umount(g_uri, mount_point, session) gluster.glusterfs_mount(g_uri, mount_point, session) gluster_img = os.path.join(mount_point, disk_img) if is_symlink: utils_misc.make_symlink(mount_point, symlink_name) utils_misc.make_symlink(mount_point, symlink_name, remote_session) gluster_img = os.path.join(symlink_name, disk_img) return gluster_img else: if is_symlink: utils_misc.rm_link(symlink_name, session) gluster.glusterfs_umount(g_uri, mount_point, session) if utils_misc.check_exists(mount_point, session): utils_misc.safe_rmdir(gluster_mount_dir, session=session) # Local variables virsh_args = {"debug": True} server_ip = params["server_ip"] = params.get("remote_ip") server_user = params["server_user"] = params.get("remote_user", "root") server_pwd = params["server_pwd"] = params.get("remote_pwd") client_ip = params["client_ip"] = params.get("local_ip") client_pwd = params["client_pwd"] = params.get("local_pwd") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options", "--live --p2p --verbose") virsh_options = params.get("virsh_options", "") vol_name = params.get("vol_name") disk_format = params.get("disk_format", "qcow2") gluster_mount_dir = params.get("gluster_mount_dir") status_error = "yes" == params.get("status_error", "no") err_msg = params.get("err_msg") host_ip = params.get("gluster_server_ip", "") migrate_vm_back = params.get("migrate_vm_back", "no") == "yes" selinux_local = params.get('set_sebool_local', 'yes') == "yes" selinux_remote = params.get('set_sebool_remote', 'no') == "yes" sebool_fusefs_local = params.get('set_sebool_fusefs_local', 'yes') sebool_fusefs_remote = params.get('set_sebool_fusefs_remote', 'yes') test_dict = dict(params) test_dict["local_boolean_varible"] = "virt_use_fusefs" test_dict["remote_boolean_varible"] = "virt_use_fusefs" remote_dargs = { 'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd, 'file_path': "/etc/libvirt/libvirt.conf" } remove_pkg = False seLinuxBool = None seLinuxfusefs = None gluster_uri = None mig_result = None remove_dict = {} remote_libvirt_file = None src_libvirt_file = None # Make sure all of parameters are assigned a valid value migrate_test = migration.MigrationTest() migrate_test.check_parameters(params) extra_args = migrate_test.update_virsh_migrate_extra_args(params) # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) params["virsh_migrate_connect_uri"] = libvirt_vm.complete_uri( params.get("migrate_source_host")) src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") # For --postcopy enable postcopy_options = params.get("postcopy_options") if postcopy_options: extra = "%s %s" % (virsh_options, postcopy_options) func_name = virsh.migrate_postcopy vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() # Back up xml file. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) # Configure selinux if selinux_local or selinux_remote: seLinuxBool = utils_misc.SELinuxBoolean(params) seLinuxBool.setup() if sebool_fusefs_local or sebool_fusefs_remote: seLinuxfusefs = utils_misc.SELinuxBoolean(test_dict) seLinuxfusefs.setup() # Setup glusterfs disk_img = "gluster.%s" % disk_format params['disk_img'] = disk_img host_ip = gluster.setup_or_cleanup_gluster(is_setup=True, **params) logging.debug("host ip: %s ", host_ip) # Check if gluster server is deployed locally if not host_ip: logging.debug("Enable port 24007 and 49152:49216") migrate_test.migrate_pre_setup(src_uri, params, ports="24007") migrate_test.migrate_pre_setup(src_uri, params) gluster_uri = "{}:{}".format(client_ip, vol_name) else: gluster_uri = "{}:{}".format(host_ip, vol_name) remote_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") if gluster_mount_dir: # The package 'glusterfs-fuse' is not installed on target # which makes issue when trying to 'mount -t glusterfs' pkg_name = 'glusterfs-fuse' logging.debug("Check if glusterfs-fuse is installed") pkg_mgr = utils_package.package_manager(remote_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("glusterfs-fuse will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) else: remove_pkg = True gluster_img = create_or_clean_backend_dir(gluster_uri, params) create_or_clean_backend_dir(gluster_uri, params, remote_session) # Get the image path image_source = vm.get_first_disk_devices()['source'] image_info = utils_misc.get_image_info(image_source) if image_info["format"] == disk_format: disk_cmd = "cp -f %s %s" % (image_source, gluster_img) else: # Convert the disk format disk_cmd = ("qemu-img convert -f %s -O %s %s %s" % (image_info["format"], disk_format, image_source, gluster_img)) process.run("%s; chmod a+rw %s" % (disk_cmd, gluster_mount_dir), shell=True) logging.debug("Gluster Image is %s", gluster_img) gluster_backend_disk = {'disk_source_name': gluster_img} # Update disk xml with gluster image in backend dir libvirt.set_vm_disk(vm, gluster_backend_disk) remote_session.close() vm_xml_cxt = virsh.dumpxml(vm_name).stdout_text.strip() logging.debug("The VM XML with gluster disk source: \n%s", vm_xml_cxt) vm.wait_for_login().close() migrate_test.ping_vm(vm, params) remove_dict = {"do_search": '{"%s": "ssh:/"}' % dest_uri} src_libvirt_file = libvirt_config.remove_key_for_modular_daemon( remove_dict) vms = [vm] migrate_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, extra_opts=extra, **extra_args) migrate_test.ping_vm(vm, params, dest_uri) if migrate_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=client_ip, server_pwd=client_pwd, client_ip=server_ip, client_pwd=server_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine migrate_test.migrate_pre_setup(src_uri, params) remove_dict = {"do_search": ('{"%s": "ssh:/"}' % src_uri)} remote_libvirt_file = libvirt_config\ .remove_key_for_modular_daemon(remove_dict, remote_dargs) cmd = "virsh migrate %s %s %s %s" % (vm_name, options, virsh_options, src_uri) logging.debug("Start migrating: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) logging.info(cmd_result) if cmd_result.exit_status: destroy_cmd = "virsh destroy %s" % vm_name remote.run_remote_cmd(destroy_cmd, params, runner_on_target, ignore_status=False) test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) finally: logging.info("Recover test environment") migrate_test.cleanup_vm(vm, dest_uri) orig_config_xml.sync() if src_libvirt_file: src_libvirt_file.restore() if remote_libvirt_file: del remote_libvirt_file # Clean up of pre migration setup for local machine if migrate_vm_back: if 'ssh_connection' in locals(): ssh_connection.auto_recover = True migrate_test.migrate_pre_setup(src_uri, params, cleanup=True) # Cleanup selinu configuration if seLinuxBool: seLinuxBool.cleanup() if seLinuxfusefs: seLinuxfusefs.cleanup() # Disable ports 24007 and 49152:49216 if not host_ip: logging.debug("Disable 24007 and 49152:49216 in Firewall") migrate_test.migrate_pre_setup(src_uri, params, cleanup=True, ports="24007") migrate_test.migrate_pre_setup(src_uri, params, cleanup=True) gluster.setup_or_cleanup_gluster(False, **params) # Cleanup backend directory/symlink if gluster_mount_dir and gluster_uri: remote_session = remote.wait_for_login('ssh', server_ip, '22', server_user, server_pwd, r"[\#\$]\s*$") create_or_clean_backend_dir(gluster_uri, params, is_clean=True) create_or_clean_backend_dir(gluster_uri, params, remote_session, True) if remove_pkg: pkg_mgr = utils_package.package_manager( remote_session, pkg_name) if pkg_mgr.is_installed(pkg_name): logging.debug("glusterfs-fuse will be uninstalled") if not pkg_mgr.remove(): logging.error("Package '%s' un-installation fails", pkg_name) remote_session.close()
def run(test, params, env): """ Test migration with special network settings 1) migrate guest with bridge type interface connected to ovs bridge 2) migrate guest with direct type interface when a macvtap device name exists on dest host :param test: test object :param params: Dictionary with the test parameters :param env: Dictionary with test environment. """ def check_vm_network_accessed(ping_dest, session=None): """ The operations to the VM need to be done before or after migration happens :param ping_dest: The destination to be ping :param session: The session object to the host :raise: test.fail when ping fails """ # Confirm local/remote VM can be accessed through network. logging.info("Check VM network connectivity") status, output = utils_net.ping(ping_dest, count=10, timeout=20, output_func=logging.debug, session=session) if status != 0: test.fail("Ping failed, status: %s, output: %s" % (status, output)) def vm_sync(vmxml, vm_name=None, virsh_instance=virsh): """ A wrapper to sync vm xml on localhost and remote host :param vmxml: domain VMXML instance :param vm_name: The name of VM :param virsh_instance: virsh instance object """ if vm_name and virsh_instance != virsh: remote.scp_to_remote(server_ip, '22', server_user, server_pwd, vmxml.xml, vmxml.xml) if virsh_instance.domain_exists(vm_name): if virsh_instance.is_alive(vm_name): virsh_instance.destroy(vm_name, ignore_status=True) virsh_instance.undefine(vmxml.xml, ignore_status=True) virsh_instance.define(vmxml.xml, debug=True) else: vmxml.sync() def update_iface_xml(vm_name, iface_dict, virsh_instance=virsh): """ Update interfaces for guest :param vm_name: The name of VM :param iface_dict: The interface configurations params :param virsh_instance: virsh instance object """ logging.debug("update iface xml") vmxml = vm_xml.VMXML.new_from_inactive_dumpxml( vm_name, virsh_instance=virsh_instance) vmxml.remove_all_device_by_type('interface') vm_sync(vmxml, vm_name, virsh_instance=virsh_instance) iface = interface.Interface('network') iface.xml = libvirt.modify_vm_iface(vm_name, "get_xml", iface_dict, virsh_instance=virsh_instance) vmxml.add_device(iface) vmxml.xmltreefile.write() vm_sync(vmxml, vm_name, virsh_instance=virsh_instance) logging.debug("VM XML after updating interface: %s" % vmxml) def update_net_dict(net_dict, runner=utils_net.local_runner): """ Update network dict :param net_dict: The network dict to be updated :param runner: Command runner :return: Updated network dict """ if net_dict.get("net_name", "") == "direct-macvtap": logging.info("Updating network iface name") iface_name = utils_net.get_net_if(runner=runner, state="UP")[0] net_dict.update({"forward_iface": iface_name}) else: # TODO: support other types logging.info("No need to update net_dict. We only support to " "update direct-macvtap type for now.") logging.debug("net_dict is %s" % net_dict) return net_dict def get_remote_direct_mode_vm_mac(vm_name, uri): """ Get mac of remote direct mode VM :param vm_name: The name of VM :param uri: The uri on destination :return: mac :raise: test.fail when the result of virsh domiflist is incorrect """ vm_mac = None res = virsh.domiflist( vm_name, uri=uri, ignore_status=False).stdout_text.strip().split("\n") if len(res) < 2: test.fail("Unable to get remote VM's mac: %s" % res) else: vm_mac = res[-1].split()[-1] return vm_mac def create_fake_tap(remote_session): """ Create a fake macvtap on destination host. :param remote_session: The session to the destination host. :return: The new tap device """ tap_cmd = "ls /dev/tap* |awk -F 'tap' '{print $NF}'" tap_idx = remote_session.cmd_output(tap_cmd).strip() if not tap_idx: test.fail("Unable to get tap index using %s." % tap_cmd) fake_tap_dest = 'tap'+str(int(tap_idx)+1) logging.debug("creating a fake tap %s...", fake_tap_dest) cmd = "touch /dev/%s" % fake_tap_dest remote_session.cmd(cmd) return fake_tap_dest migration_test = migration.MigrationTest() migration_test.check_parameters(params) libvirt_version.is_libvirt_feature_supported(params) # Params to update disk using shared storage params["disk_type"] = "file" params["disk_source_protocol"] = "netfs" params["mnt_path_name"] = params.get("nfs_mount_dir") # Local variables virsh_args = {"debug": True} server_ip = params.get("server_ip") server_user = params.get("server_user", "root") server_pwd = params.get("server_pwd") client_ip = params.get("client_ip") client_pwd = params.get("client_pwd") virsh_options = params.get("virsh_options", "") extra = params.get("virsh_migrate_extra") options = params.get("virsh_migrate_options", "--live --p2p --verbose") restart_dhclient = params.get("restart_dhclient", "dhclient -r; dhclient") ping_dest = params.get("ping_dest", "www.baidu.com") extra_args = migration_test.update_virsh_migrate_extra_args(params) migrate_vm_back = "yes" == params.get("migrate_vm_back", "no") target_vm_name = params.get("target_vm_name") direct_mode = "yes" == params.get("direct_mode", "no") check_macvtap_exists = "yes" == params.get("check_macvtap_exists", "no") create_fake_tap_dest = "yes" == params.get("create_fake_tap_dest", "no") macvtap_cmd = params.get("macvtap_cmd") modify_target_vm = "yes" == params.get("modify_target_vm", "no") ovs_bridge_name = params.get("ovs_bridge_name") network_dict = eval(params.get("network_dict", '{}')) iface_dict = eval(params.get("iface_dict", '{}')) remote_virsh_dargs = {'remote_ip': server_ip, 'remote_user': server_user, 'remote_pwd': server_pwd, 'unprivileged_user': None, 'ssh_remote_auth': True} cmd_parms = {'server_ip': server_ip, 'server_user': server_user, 'server_pwd': server_pwd} virsh_session_remote = None libvirtd_conf = None mig_result = None target_org_xml = None target_vm_session = None target_vm = None exp_macvtap = [] fake_tap_dest = None # params for migration connection params["virsh_migrate_desturi"] = libvirt_vm.complete_uri( params.get("migrate_dest_host")) params["virsh_migrate_connect_uri"] = libvirt_vm.complete_uri( params.get("migrate_source_host")) src_uri = params.get("virsh_migrate_connect_uri") dest_uri = params.get("virsh_migrate_desturi") vm_name = params.get("migrate_main_vm") vm = env.get_vm(vm_name) vm.verify_alive() bk_uri = vm.connect_uri postcopy_options = params.get("postcopy_options") action_during_mig = None if postcopy_options: extra = "%s %s" % (extra, postcopy_options) action_during_mig = virsh.migrate_postcopy # For safety reasons, we'd better back up xmlfile. new_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) orig_config_xml = new_xml.copy() try: # Create a remote runner for later use runner_on_target = remote.RemoteRunner(host=server_ip, username=server_user, password=server_pwd) remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, r'[$#%]') virsh_session_remote = virsh.VirshPersistent(**remote_virsh_dargs) if target_vm_name: target_vm = libvirt_vm.VM(target_vm_name, params, vm.root_dir, vm.address_cache) target_vm.connect_uri = dest_uri if not virsh_session_remote.domain_exists(target_vm_name): test.error("VM %s should be installed on %s." % (target_vm_name, server_ip)) # Backup guest's xml on remote target_org_xml = vm_xml.VMXML.new_from_inactive_dumpxml( target_vm_name, virsh_instance=virsh_session_remote) # Scp original xml to remote for restoration remote.scp_to_remote(server_ip, '22', server_user, server_pwd, target_org_xml.xml, target_org_xml.xml) logging.debug("target xml is %s" % target_org_xml) if ovs_bridge_name: status, stdout = utils_net.create_ovs_bridge(ovs_bridge_name) if status: test.fail("Failed to create ovs bridge on local. Status: %s" "Stdout: %s" % (status, stdout)) status, stdout = utils_net.create_ovs_bridge( ovs_bridge_name, session=remote_session) if status: test.fail("Failed to create ovs bridge on remote. Status: %s" "Stdout: %s" % (status, stdout)) if network_dict: update_net_dict(network_dict, runner=remote_session.cmd) libvirt_network.create_or_del_network( network_dict, remote_args=remote_virsh_dargs) logging.info("dest: network created") update_net_dict(network_dict) libvirt_network.create_or_del_network(network_dict) logging.info("localhost: network created") if target_vm_name: if modify_target_vm and iface_dict: logging.info("Updating remote VM's interface") update_iface_xml(target_vm_name, iface_dict, virsh_instance=virsh_session_remote) target_vm.start() target_vm_session = target_vm.wait_for_serial_login(timeout=240) check_vm_network_accessed(ping_dest, session=target_vm_session) if check_macvtap_exists and macvtap_cmd: # Get macvtap device's index on remote after target_vm started idx = remote_session.cmd_output(macvtap_cmd).strip() if not idx: test.fail("Unable to get macvtap index using %s." % macvtap_cmd) # Generate the expected macvtap devices' index list exp_macvtap = ['macvtap'+idx, 'macvtap'+str(int(idx)+1)] if create_fake_tap_dest: fake_tap_dest = create_fake_tap(remote_session) remote_session.close() # Change domain network xml if iface_dict: if "mac" not in iface_dict: mac = utils_net.generate_mac_address_simple() iface_dict.update({'mac': mac}) else: mac = iface_dict["mac"] update_iface_xml(vm_name, iface_dict) # Change the disk of the vm libvirt.set_vm_disk(vm, params) if not vm.is_alive(): try: vm.start() except virt_vm.VMStartError as err: test.fail("Failed to start VM: %s" % err) logging.debug("Guest xml after starting:\n%s", vm_xml.VMXML.new_from_dumpxml(vm_name)) # Check local guest network connection before migration if vm.serial_console is not None: vm.cleanup_serial_console() vm.create_serial_console() vm_session = vm.wait_for_serial_login(timeout=240) if not utils_package.package_install('dhcp-client', session=vm_session): test.error("Failed to install dhcp-client on guest.") utils_net.restart_guest_network(vm_session) vm_ip = utils_net.get_guest_ip_addr(vm_session, mac) logging.debug("VM IP Addr: %s", vm_ip) if direct_mode: check_vm_network_accessed(ping_dest, session=vm_session) else: check_vm_network_accessed(vm_ip) # Execute migration process vms = [vm] migration_test.do_migration(vms, None, dest_uri, 'orderly', options, thread_timeout=900, ignore_status=True, virsh_opt=virsh_options, func=action_during_mig, extra_opts=extra, **extra_args) mig_result = migration_test.ret # Check network accessibility after migration if int(mig_result.exit_status) == 0: vm.connect_uri = dest_uri if vm.serial_console is not None: vm.cleanup_serial_console() vm.create_serial_console() vm_session_after_mig = vm.wait_for_serial_login(timeout=240) vm_session_after_mig.cmd(restart_dhclient) check_vm_network_accessed(ping_dest, session=vm_session_after_mig) if check_macvtap_exists and macvtap_cmd: remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, r'[$#%]') # Check macvtap devices' index after migration idx = remote_session.cmd_output(macvtap_cmd) act_macvtap = ['macvtap'+i for i in idx.strip().split("\n")] if act_macvtap != exp_macvtap: test.fail("macvtap devices after migration are incorrect!" " Actual: %s, Expected: %s. " % (act_macvtap, exp_macvtap)) else: if fake_tap_dest: res = remote.run_remote_cmd("ls /dev/%s" % fake_tap_dest, params, runner_on_target) libvirt.check_exit_status(res) if target_vm_session: check_vm_network_accessed(ping_dest, session=target_vm_session) # Execute migration from remote if migrate_vm_back: ssh_connection = utils_conn.SSHConnection(server_ip=client_ip, server_pwd=client_pwd, client_ip=server_ip, client_pwd=server_pwd) try: ssh_connection.conn_check() except utils_conn.ConnectionError: ssh_connection.conn_setup() ssh_connection.conn_check() # Pre migration setup for local machine migration_test.migrate_pre_setup(src_uri, params) cmd = "virsh migrate %s %s %s" % (vm_name, options, src_uri) logging.debug("Start migration: %s", cmd) cmd_result = remote.run_remote_cmd(cmd, params, runner_on_target) logging.info(cmd_result) if cmd_result.exit_status: test.fail("Failed to run '%s' on remote: %s" % (cmd, cmd_result)) logging.debug("VM is migrated back.") vm.connect_uri = bk_uri if vm.serial_console is not None: vm.cleanup_serial_console() vm.create_serial_console() vm_session_after_mig_bak = vm.wait_for_serial_login(timeout=240) vm_session_after_mig_bak.cmd(restart_dhclient) check_vm_network_accessed(ping_dest, vm_session_after_mig_bak) finally: logging.debug("Recover test environment") vm.connect_uri = bk_uri migration_test.cleanup_vm(vm, dest_uri) logging.info("Recovery VM XML configration") orig_config_xml.sync() remote_session = remote.remote_login("ssh", server_ip, "22", server_user, server_pwd, r'[$#%]') if target_vm and target_vm.is_alive(): target_vm.destroy(gracefully=False) if target_org_xml and target_vm_name: logging.info("Recovery XML configration for %s.", target_vm_name) virsh_session_remote = virsh.VirshPersistent(**remote_virsh_dargs) vm_sync(target_org_xml, vm_name=target_vm_name, virsh_instance=virsh_session_remote) virsh_session_remote.close_session() if fake_tap_dest: remote_session.cmd_output_safe("rm -rf /dev/%s" % fake_tap_dest) if network_dict: libvirt_network.create_or_del_network( network_dict, is_del=True, remote_args=remote_virsh_dargs) libvirt_network.create_or_del_network(network_dict, is_del=True) if ovs_bridge_name: utils_net.delete_ovs_bridge(ovs_bridge_name) utils_net.delete_ovs_bridge(ovs_bridge_name, session=remote_session) remote_session.close() if target_vm_session: target_vm_session.close() if virsh_session_remote: virsh_session_remote.close_session() if migrate_vm_back: if 'ssh_connection' in locals(): ssh_connection.auto_recover = True migration_test.migrate_pre_setup(src_uri, params, cleanup=True) logging.info("Remove local NFS image") source_file = params.get("source_file") if source_file: libvirt.delete_local_disk("file", path=source_file)