def monitor_event(params): """ Monitor event on source/target host :param params: dict, used to setup the connection :return: virsh session and remote virsh session to catch events """ expected_event_src = params.get("expected_event_src") expected_event_target = params.get("expected_event_target") remote_pwd = params.get("migrate_dest_pwd") remote_ip = params.get("migrate_dest_host") remote_user = params.get("remote_user", "root") cmd = "event --loop --all" if expected_event_src: logging.debug("Running virsh command on source: %s", cmd) virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True) virsh_session.sendline(cmd) if expected_event_target: logging.debug("Running virsh command on target: %s", cmd) virsh_dargs = { 'remote_ip': remote_ip, 'remote_user': remote_user, 'remote_pwd': remote_pwd, 'unprivileged_user': None, 'virsh_exec': virsh.VIRSH_EXEC, 'auto_close': True, 'uri': 'qemu+ssh://%s/system' % remote_ip } remote_virsh_session = virsh.VirshSession(**virsh_dargs) remote_virsh_session.sendline(cmd) return virsh_session, remote_virsh_session
def set_migratepostcopy(self, vm_name, uri=None): """ Switch to postcopy during migration. :param vm_name: vm's name :param uri: target virsh uri :raise: test.error when command fails """ cmd = "event --loop --all" virsh_event_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True, uri=uri) virsh_event_session.sendline(cmd) if not utils_misc.wait_for( lambda: not virsh.migrate_postcopy( vm_name, uri=uri, debug=True).exit_status, 10): raise exceptions.TestError("Failed to set migration postcopy.") exp_str = "Suspended Post-copy" if not utils_misc.wait_for( lambda: re.findall( exp_str, virsh_event_session.get_stripped_output()), 30): raise exceptions.TestError( "Unalbe to find event {}".format(exp_str))
def _run_collect_event_cmd(): """ To execute virsh event command to collect the domain events :return: VirshSession to retrieve the events """ cmd = "event --loop --all" virsh_event_session = virsh.VirshSession( virsh_exec=virsh.VIRSH_EXEC, auto_close=True, uri=srcuri) virsh_event_session.sendline(cmd) logging.debug("Begin to collect domain events...") return virsh_event_session
def set_secret_value(test, params): """ Set the secet value :params: the parameter dictionary """ uuid = params.get("secret_ref") options = params.get("set_secret_options") status_error = params.get("status_error", "no") secret_string = params.get("secret_base64_no_encoded") secret_encode = "yes" == params.get("secret_string_base64_encode", "yes") secret_file = "yes" == params.get("secret_file", "no") if options and "interactive" in options: cmd = "secret-set-value %s --%s" % (uuid, options) virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True) virsh_session.sendline(cmd) # Wait for 5s time.sleep(5) virsh_session.sendline(secret_string) # Wait for 5s to get stripped output time.sleep(5) output = virsh_session.get_stripped_output() exit_status = 0 if "Secret value set" in output else 1 result = process.CmdResult(cmd, output, output, exit_status) else: result = virsh.secret_set_value(uuid, secret_string, options=options, encode=secret_encode, use_file=secret_file, debug=True) status = result.exit_status # Don't check result if we don't need to. if params.get("check_set_status", "yes") == "no": return # Check status_error if status_error == "yes": if status: logging.info("It's an expected %s", result.stderr) else: test.fail("%d not a expected command " "return value", status) elif status_error == "no": if status: test.fail(result.stderr) else: # Check secret value if check_secret(params): logging.info(result.stdout.strip()) else: test.fail("The secret value " "mismatch with result")
def setup_blockjob_raw(): """ Prepare running domain and do blockcopy """ if not vm.is_alive(): vm.start() if os.path.exists(tmp_copy_path): process.run('rm -rf %s' % tmp_copy_path) cmd = "blockcopy %s %s %s --wait --verbose --transient-job " \ "--bandwidth 1000 " % (vm_name, dev, tmp_copy_path) virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True) virsh_session.sendline(cmd)
def run(test, params, env): """ Test watchdog device: 1.Add watchdog device to the guest xml. 2.Start the guest. 3.Trigger the watchdog in the guest. 4.Confirm the guest status. """ def trigger_watchdog(model): """ Trigger watchdog :param model: action when watchdog triggered """ watchdog_device = "device %s" % model if action == "dump": watchdog_action = "watchdog-action pause" else: watchdog_action = "watchdog-action %s" % action if not hotplug_test: vm_pid = vm.get_pid() with open("/proc/%s/cmdline" % vm_pid) as vm_cmdline_file: vm_cmdline = vm_cmdline_file.read() vm_cmdline = vm_cmdline.replace('\x00', ' ') if not all(option in vm_cmdline for option in (watchdog_device, watchdog_action)): test.fail("Can not find %s or %s in qemu cmd line" % (watchdog_device, watchdog_action)) cmd = "gsettings set org.gnome.settings-daemon.plugins.power button-power shutdown" session.cmd(cmd, ignore_all_errors=True) try: if model == "ib700": try: session.cmd("modprobe ib700wdt") except aexpect.ShellCmdError: session.close() test.fail("Failed to load module ib700wdt") session.cmd("dmesg | grep -i %s && lsmod | grep %s" % (model, model)) session.cmd("echo 1 > /dev/watchdog") except aexpect.ShellCmdError as e: session.close() test.fail("Failed to trigger watchdog: %s" % e) def watchdog_attached(vm_name): """ Confirm whether watchdog device is attached to vm by checking domain dumpxml :param vm_name: vm name """ vmxml = vm_xml.VMXML.new_from_dumpxml(vm_name) if vmxml.xmltreefile.find('devices/watchdog'): return True else: return False def confirm_guest_status(): """ Confirm the guest status after watchdog triggered """ def _booting_completed(): session = vm.wait_for_login() status = None second_boot_time = None try: status, second_boot_time = session.cmd_status_output("uptime --since") logging.debug("The second boot time is %s", second_boot_time) except (aexpect.ShellStatusError, aexpect.ShellProcessTerminatedError) as e: logging.error("Exception caught:%s", e) session.close() return second_boot_time > first_boot_time def _inject_nmi(): session = vm.wait_for_login() status, output = session.cmd_status_output("dmesg | grep -i nmi") session.close() if status == 0: logging.debug(output) return True return False def _inject_nmi_event(): virsh_session.send_ctrl("^C") output = virsh_session.get_stripped_output() if "inject-nmi" not in output: return False return True def _check_dump_file(dump_path, domain_id): dump_file = glob.glob('%s%s-*' % (dump_path, domain_id)) if len(dump_file): logging.debug("Find the auto core dump file:\n%s", dump_file[0]) os.remove(dump_file[0]) return True return False if action in ["poweroff", "shutdown"]: if not utils_misc.wait_for(lambda: vm.state() == "shut off", 180, 10): test.fail("Guest not shutdown after watchdog triggered") else: logging.debug("Guest is in shutdown state after watchdog triggered") elif action == "reset": if not utils_misc.wait_for(_booting_completed, 600, 10): test.fail("Guest not reboot after watchdog triggered") else: logging.debug("Guest is rebooted after watchdog triggered") elif action == "pause": if utils_misc.wait_for(lambda: vm.state() == "paused", 180, 10): logging.debug("Guest is in paused status after watchdog triggered.") cmd_output = virsh.domstate(vm_name, '--reason').stdout.strip() logging.debug("Check guest status: %s\n", cmd_output) if cmd_output != "paused (watchdog)": test.fail("The domstate is not correct after dump by watchdog") else: test.fail("Guest not pause after watchdog triggered") elif action == "none": if utils_misc.wait_for(lambda: vm.state() == "shut off", 180, 10): test.fail("Guest shutdown unexpectedly") else: logging.debug("Guest is not in shutoff state since watchdog action is none.") elif action == "inject-nmi": if not utils_misc.wait_for(_inject_nmi, 180, 10): test.fail("Guest not receive inject-nmi after watchdog triggered\n") elif not utils_misc.wait_for(_inject_nmi_event, 180, 10): test.fail("No inject-nmi watchdog event caught") else: logging.debug("Guest received inject-nmi and inject-nmi watchdog event " " has been caught.") virsh_session.close() elif action == "dump": domain_id = vm.get_id() dump_path = "/var/lib/libvirt/qemu/dump/" if not utils_misc.wait_for(lambda: _check_dump_file(dump_path, domain_id), 180, 10): test.fail("No auto core dump file found after watchdog triggered") else: logging.debug("VM core has been dumped after watchdog triggered.") name_length = params.get("name_length", "default") vm_name = params.get("main_vm", "avocado-vt-vm1") vm = env.get_vm(params["main_vm"]) model = params.get("model") action = params.get("action") model_test = params.get("model_test") == "yes" hotplug_test = params.get("hotplug_test") == "yes" hotunplug_test = params.get("hotunplug_test") == "yes" machine_type = params.get("machine_type") if machine_type == "q35" and model == "ib700": test.cancel("ib700wdt watchdog device is not supported " "on guest with q35 machine type") # Backup xml file vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) backup_xml = vmxml.copy() # Rename the guest name to the length defined in the config file if name_length != "default": origin_name = vm_name name_length = int(params.get("name_length", "1")) vm_name = ''.join([random.choice(string.ascii_letters+string.digits) for _ in range(name_length)]) vm_xml.VMXML.vm_rename(vm, vm_name) # Generate the renamed xml file vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) # Coldplug pcie-to-pci-bridge to vm xml for q35 guest as i6300esb watchdog # device can only be plugged to conventional PCI slot if (machine_type == 'q35' and not vmxml.get_controllers(controller_type='pci', model='pcie-to-pci-bridge')): logging.debug("Add pcie-root-port and pcie-to-pci-bridge controller to vm") pcie_root_port = Controller("pci") pcie_pci_bridge = Controller("pci") pcie_root_port.model = "pcie-root-port" pcie_pci_bridge.model = "pcie-to-pci-bridge" pcie_root_port.model_name = {'name': 'pcie-root-port'} pcie_pci_bridge.model_name = {'name': 'pcie-pci-bridge'} vmxml.add_device(pcie_root_port) vmxml.add_device(pcie_pci_bridge) vmxml.sync() if hotplug_test: vm.start() session = vm.wait_for_login() # Add watchdog device to domain vmxml.remove_all_device_by_type('watchdog') watchdog_dev = Watchdog() watchdog_dev.model_type = model watchdog_dev.action = action chars = string.ascii_letters + string.digits + '-_' alias_name = 'ua-' + ''.join(random.choice(chars) for _ in list(range(64))) watchdog_dev.alias = {'name': alias_name} try: if model_test or hotunplug_test: vmxml.add_device(watchdog_dev) vmxml.sync() try: vm.start() except Exception: test.fail("VM startup after adding watchdog device failed!") elif hotplug_test: watchdog_xml = watchdog_dev.xml attach_result = virsh.attach_device(vm_name, watchdog_xml, ignore_status=False, debug=True) if not utils_misc.wait_for(lambda: watchdog_attached(vm.name), 60): test.fail("Failed to hotplug watchdog device.") session = vm.wait_for_login() # No need to trigger watchdog after hotunplug if hotunplug_test: cur_xml = vm_xml.VMXML.new_from_dumpxml(vm_name) cur_watchdog = cur_xml.xmltreefile.find('devices/watchdog') cur_watchdog_xml = Watchdog.new_from_element(cur_watchdog).xml detach_result = virsh.detach_device(vm_name, cur_watchdog_xml, ignore_status=True, debug=True) if detach_result.exit_status: test.fail("i6300esb watchdog device can NOT be detached successfully, " "result:\n%s" % detach_result) elif not utils_misc.wait_for(lambda: not watchdog_attached(vm.name), 60): test.fail("Failed to hotunplug watchdog device.") return if action == "reset": status, first_boot_time = session.cmd_status_output("uptime --since") logging.info("The first boot time is %s\n", first_boot_time) if action == "inject-nmi": virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True) event_cmd = "event --event watchdog --all --loop" virsh_session.sendline(event_cmd) trigger_watchdog(model) confirm_guest_status() finally: if vm.is_alive(): vm.destroy(gracefully=False) if name_length != "default": vm_xml.VMXML.vm_rename(vm, origin_name) backup_xml.sync()
def _migrate(self): """ 1.Set selinux state 2.Record vm uptime 3.For postcopy migration: 1) Set migration speed to low value 2) Monitor postcopy event 4.Do live migration 5.Check migration result: succeed or fail with expected error 6.For postcopy migration: check postcopy event 7.Do post migration check: check vm state, uptime, network """ # Set selinux state before migration # NOTE: if selinux state is set too early, it may be changed # in other methods unexpectedly, so set it just before migration logging.debug("Set selinux to enforcing before migration") utils_selinux.set_status(self.selinux_state) # TODO: Set selinux on migrate_dest_host # Check vm uptime before migration logging.debug("Check vm uptime before migration") self.uptime = {} for vm in self.vms: self.uptime[vm.name] = vm.uptime(connect_uri=vm.connect_uri) # Do postcopy/precopy related operations/setting if self.migrate_flags & VIR_MIGRATE_POSTCOPY: # Set migration speed to low value in case it finished too early # before postcopy mode starts for vm in self.vms: virsh.migrate_setspeed(vm.name, 1, uri=vm.connect_uri) # Monitor event "Suspended Post-copy" for postcopy migration logging.debug("Monitor the event for postcopy migration") virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True) self.objs_list.append(virsh_session) cmd = "event %s --loop --all --timestamp" % self.main_vm.name virsh_session.sendline(cmd) # Set func to be executed during postcopy migration func = virsh.migrate_postcopy else: # Set func to be executed during precopy migration func = None # Start to do migration logging.debug("Start to do migration") thread_timeout = self.migrate_thread_timeout self.obj_migration.do_migration(self.vms, self.src_uri, self.dest_uri, "orderly", options=self.virsh_migrate_options, thread_timeout=thread_timeout, ignore_status=True, virsh_uri=self.src_uri, func=func, shell=True) logging.info("Check migration result: succeed or" " fail with expected error") self.obj_migration.check_result(self.obj_migration.ret, self.params) # Check "suspended post-copy" event after postcopy migration if self.migrate_flags & VIR_MIGRATE_POSTCOPY: logging.debug("Check event after postcopy migration") virsh_session.send_ctrl("^c") events_output = virsh_session.get_stripped_output() logging.debug("Events_output are %s", events_output) pattern = "Suspended Post-copy" if pattern not in events_output: self.test.error("Migration didn't switch to postcopy mode") logging.debug("Do post migration check after migrate to dest") self.params["migrate_options"] = self.virsh_migrate_options self.obj_migration.post_migration_check(self.vms, self.params, self.uptime, uri=self.dest_uri)
def run(test, params, env): """ Test command: virsh start. 1) Get the params from params. 2) Prepare libvirtd's status. 3) Do the start operation. 4) Result check. 5) clean up. """ # get the params from params vm_name = params.get("main_vm", "avocado-vt-vm1") vm_ref = params.get("vm_ref", "vm1") opt = params.get("vs_opt", "") # Backup for recovery. vmxml_backup = libvirt_xml.vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) backup_name = vm_ref vm = None if vm_ref is not "": vm = env.get_vm(vm_ref) vmxml = libvirt_xml.VMXML() libvirtd_state = params.get("libvirtd", "on") pre_operation = params.get("vs_pre_operation", "") status_error = params.get("status_error", "no") try: # prepare before start vm if libvirtd_state == "on": utils_libvirtd.libvirtd_start() elif libvirtd_state == "off": utils_libvirtd.libvirtd_stop() if pre_operation == "rename": new_vm_name = params.get("vs_new_vm_name", "virsh_start_vm1") vm = libvirt_xml.VMXML.vm_rename(vm, new_vm_name) vm_ref = new_vm_name elif pre_operation == "undefine": vmxml = vmxml.new_from_dumpxml(vm_ref) vmxml.undefine() # do the start operation try: if pre_operation == "remote": # get the params for remote test remote_ip = params.get("remote_ip", "ENTER.YOUR.REMOTE.IP") remote_user = params.get("remote_user", "root") remote_pwd = params.get("remote_pwd", "ENTER.YOUR.REMOTE.PASSWORD") if pre_operation == "remote" and remote_ip.count( "ENTER.YOUR."): test.cancel("Remote test parameters not configured") ssh_key.setup_ssh_key(remote_ip, remote_user, remote_pwd) remote_uri = "qemu+ssh://%s/system" % remote_ip cmd_result = virsh.start(vm_ref, ignore_status=True, debug=True, uri=remote_uri) if cmd_result.exit_status: test.fail("Start vm failed.\n Detail: %s" % cmd_result) elif opt.count("console"): # With --console, start command will print the # dmesg of guest in starting and turn into the # login prompt. In this case, we start it with # --console and login vm in console by # remote.handle_prompts(). cmd = "start %s --console" % vm_ref virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True) virsh_session.sendline(cmd) remote.handle_prompts(virsh_session, params.get("username", ""), params.get("password", ""), r"[\#\$]\s*$", timeout=60, debug=True) elif opt.count("autodestroy"): # With --autodestroy, vm will be destroyed when # virsh session closed. Then we execute start # command in a virsh session and start vm with # --autodestroy. Then we closed the virsh session, # and check the vm is destroyed or not. virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True) cmd = "start %s --autodestroy" % vm_ref status = virsh_session.cmd_status(cmd) if status: test.fail("Failed to start vm with --autodestroy.") # Close the session, then the vm shoud be destroyed. virsh_session.close() elif opt.count("force-boot"): # With --force-boot, VM will be stared from boot # even we have saved it with virsh managedsave. # In this case, we start vm and execute sleep 1000&, # then save it with virsh managedsave. At last, we # start vm with --force-boot. To verify the result, # we check the sleep process. If the process exists, # force-boot failed, else case pass. vm.start() session = vm.wait_for_login() status = session.cmd_status("sleep 1000&") if status: test.error("Can not execute command in guest.") sleep_pid = session.cmd_output("echo $!").strip() virsh.managedsave(vm_ref) virsh.start(vm_ref, options=opt) else: cmd_result = virsh.start(vm_ref, options=opt) if cmd_result.exit_status: if status_error == "no": test.fail("Start vm failed.\n Detail: %s" % cmd_result) else: # start vm successfully if status_error == "yes": test.fail("Run successfully with wrong " "command!\n Detail:%s" % cmd_result) if opt.count("paused"): if not (vm.state() == "paused"): test.fail("VM is not paused when started with " "--paused.") elif opt.count("autodestroy"): if vm.is_alive(): test.fail("VM was started with --autodestroy," "but not destroyed when virsh session " "closed.") elif opt.count("force-boot"): session = vm.wait_for_login() status = session.cmd_status("ps %s |grep '[s]leep 1000'" % sleep_pid) if not status: test.fail("VM was started with --force-boot," "but it is restored from a" " managedsave.") else: if status_error == "no" and not vm.is_alive( ) and pre_operation != "remote": test.fail("VM was started but it is not alive.") except remote.LoginError as detail: test.fail("Failed to login guest.") finally: # clean up if libvirtd_state == "off": utils_libvirtd.libvirtd_start() elif pre_operation == "rename": libvirt_xml.VMXML.vm_rename(vm, backup_name) elif pre_operation == "remote": virsh.destroy(vm_ref, ignore_status=False, debug=True, uri=remote_uri) if vm and vm.is_paused(): vm.resume() # Restore VM vmxml_backup.sync()
def run(test, params, env): """ Test virsh migrate command. """ def cleanup_vm(vm, vm_name='', uri=''): """ Clean up vm in the src or destination host environment when doing the uni-direction migration. """ # Backup vm name and uri uri_bak = vm.connect_uri vm_name_bak = vm.name # Destroy and undefine vm vm.connect_uri = uri if uri else uri_bak vm.name = vm_name if vm_name else vm_name_bak logging.info("Cleaning up VM %s on %s", vm.name, vm.connect_uri) if vm.is_alive(): vm.destroy() if vm.is_persistent(): vm.undefine() # Restore vm connect_uri vm.connect_uri = uri_bak vm.name = vm_name_bak # Check whether there are unset parameters for v in list(itervalues(params)): if isinstance(v, string_types) and v.count("EXAMPLE"): test.cancel("Please set real value for %s" % v) # Params for virsh migrate options: live_migration = params.get("live_migration") == "yes" offline_migration = params.get("offline_migration") == "yes" persistent = params.get("persistent") == "yes" undefinesource = params.get("undefinesource") == "yes" p2p = params.get("p2p") == "yes" tunnelled = params.get("tunnelled") == "yes" postcopy = params.get("postcopy") == "yes" dname = params.get("dname") xml_option = params.get("xml_option") == "yes" persistent_xml_option = params.get("persistent_xml_option") == "yes" extra_options = params.get("virsh_migrate_extra", "") if live_migration and not extra_options.count("--live"): extra_options = "%s --live" % extra_options if offline_migration and not extra_options.count("--offline"): extra_options = "%s --offline" % extra_options if persistent and not extra_options.count("--persistent"): extra_options = "%s --persistent" % extra_options if undefinesource and not extra_options.count("--undefinesource"): extra_options = "%s --undefinesource" % extra_options if p2p and not extra_options.count("--p2p"): extra_options = "%s --p2p" % extra_options if tunnelled and not extra_options.count("--tunnelled"): extra_options = "%s --tunnelled" % extra_options if tunnelled and not extra_options.count("--p2p"): extra_options = "%s --p2p" % extra_options if postcopy and not extra_options.count("--postcopy"): extra_options = "%s --postcopy" % extra_options if dname and not extra_options.count("--dname"): extra_options = "%s --dname %s" % (extra_options, dname) if xml_option: pass if persistent_xml_option and not extra_options.count("--persistent"): extra_options = "%s --persistent" % extra_options if persistent_xml_option: pass # Set param migrate_options in case it is used somewhere: params.setdefault("migrate_options", extra_options) # Params for postcopy migration postcopy_timeout = int(params.get("postcopy_migration_timeout", "180")) # Params for migrate hosts: server_cn = params.get("server_cn") client_cn = params.get("client_cn") migrate_source_host = client_cn if client_cn else params.get( "migrate_source_host") migrate_dest_host = server_cn if server_cn else params.get( "migrate_dest_host") # Params for migrate uri transport = params.get("transport", "tls") transport_port = params.get("transport_port") uri_port = ":%s" % transport_port if transport_port else '' hypervisor_driver = params.get("hypervisor_driver", "qemu") hypervisor_mode = params.get("hypervisor_mode", 'system') if "virsh_migrate_desturi" not in list(params.keys()): params["virsh_migrate_desturi"] = "%s+%s://%s%s/%s" % ( hypervisor_driver, transport, migrate_dest_host, uri_port, hypervisor_mode) if "virsh_migrate_srcuri" not in list(params.keys()): params["virsh_migrate_srcuri"] = "%s:///%s" % (hypervisor_driver, hypervisor_mode) dest_uri = params.get("virsh_migrate_desturi") src_uri = params.get("virsh_migrate_srcuri") # Params for src vm cfg: src_vm_cfg = params.get("src_vm_cfg") src_vm_status = params.get("src_vm_status") with_graphic_passwd = params.get("with_graphic_passwd") graphic_passwd = params.get("graphic_passwd") # For test result check cancel_exception = False fail_exception = False exception = False result_check_pass = True # Objects(SSH, TLS and TCP, etc) to be cleaned up in finally objs_list = [] # VM objects for migration test vms = [] try: # Get a MigrationTest() Object logging.debug("Get a MigrationTest() object") obj_migration = migration.MigrationTest() # Setup libvirtd remote connection TLS connection env if transport == "tls": tls_obj = TLSConnection(params) # Setup CA, server(on dest host) and client(on src host) tls_obj.conn_setup() # Add tls_obj to objs_list objs_list.append(tls_obj) # Enable libvirtd remote connection transport port if transport == 'tls': transport_port = '16514' elif transport == 'tcp': transport_port = '16509' obj_migration.migrate_pre_setup(dest_uri, params, ports=transport_port) # Back up vm name for recovery in finally vm_name_backup = params.get("migrate_main_vm") # Get a vm object for migration logging.debug("Get a vm object for migration") vm = env.get_vm(vm_name_backup) # Back up vm xml for recovery in finally logging.debug("Backup vm xml before migration") vm_xml_backup = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) if not vm_xml_backup: test.error("Backing up xmlfile failed.") # Prepare shared disk in vm xml for live migration: # Change the source of the first disk of vm to shared disk if live_migration: logging.debug("Prepare shared disk in vm xml for live migration") storage_type = params.get("storage_type") if storage_type == 'nfs': logging.debug("Prepare nfs shared disk in vm xml") nfs_mount_dir = params.get("nfs_mount_dir") libvirt.update_vm_disk_source(vm.name, nfs_mount_dir) libvirt.update_vm_disk_driver_cache(vm.name, driver_cache="none") else: # TODO:Other storage types test.cancel("Other storage type is not supported for now") pass # Prepare graphic password in vm xml if with_graphic_passwd in ["yes", "no"]: logging.debug("Set VM graphic passwd in vm xml") # Get graphics list in vm xml vmxml_tmp = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name) graphics_list = vmxml_tmp.get_graphics_devices if not graphics_list: # Add spice graphic with passwd to vm xml logging.debug("Add spice graphic to vm xml") graphics.Graphics.add_graphic(vm.name, graphic_passwd, "spice") elif graphic_passwd: # Graphics already exist in vm xml and passwd is required # Add passwd to the first graphic device in vm xml logging.debug("Add graphic passwd to vm xml") vm_xml.VMXML.add_security_info(vmxml_tmp, graphic_passwd) vmxml_tmp.sync() else: # Graphics already exist in vm xml and non-passwd is required # Do nothing here as passwd has been removed by new_from_inactive_dumpxml() pass # Prepare for required src vm status. logging.debug("Turning %s into certain state.", vm.name) if src_vm_status == "running" and not vm.is_alive(): vm.start() elif src_vm_status == "shut off" and not vm.is_dead(): vm.destroy() # Prepare for required src vm persistency. logging.debug("Prepare for required src vm persistency") if src_vm_cfg == "persistent" and not vm.is_persistent(): logging.debug("Make src vm persistent") vm_xml_backup.define() elif src_vm_cfg == "transient" and vm.is_persistent(): logging.debug("Make src vm transient") vm.undefine() # Prepare for postcopy migration: install and run stress in VM if postcopy and src_vm_status == "running": logging.debug( "Install and run stress in vm for postcopy migration") pkg_name = 'stress' # Get a vm session logging.debug("Get a vm session") vm_session = vm.wait_for_login() if not vm_session: test.error("Can't get a vm session successfully") # Install package stress if it is not installed in vm logging.debug( "Check if stress tool is installed for postcopy migration") pkg_mgr = utils_package.package_manager(vm_session, pkg_name) if not pkg_mgr.is_installed(pkg_name): logging.debug("Stress tool will be installed") if not pkg_mgr.install(): test.error("Package '%s' installation fails" % pkg_name) # Run stress in vm logging.debug("Run stress in vm") stress_args = params.get("stress_args") vm_session.cmd('stress %s' % stress_args) # Prepare for --xml <updated_xml_file>. if xml_option: logging.debug("Preparing new xml file for --xml option.") # Get the vm xml vmxml_tmp = vm_xml.VMXML.new_from_dumpxml( vm.name, "--security-info --migratable") # Update something in the xml file: e.g. title # Note: VM ABI shall not be broken when migrating with updated_xml updated_title = "VM Title in updated xml" vmxml_tmp.title = updated_title # Add --xml to migrate extra_options extra_options = ("%s --xml=%s" % (extra_options, vmxml_tmp.xml)) # Prepare for --persistent-xml <updated_xml_file>. if persistent_xml_option: logging.debug( "Preparing new xml file for --persistent-xml option.") # Get the vm xml vmxml_persist_tmp = vm_xml.VMXML.new_from_inactive_dumpxml( vm.name, "--security-info") # Update something in the xml file: e.g. title # Note: VM ABI shall not be broken when migrating with updated_xml updated_persist_title = "VM Title in updated persist xml" vmxml_persist_tmp.title = updated_persist_title # Add --persistent-xml to migrate extra_options extra_options = ("%s --persistent-xml=%s" % (extra_options, vmxml_persist_tmp.xml)) # Prepare host env: clean up vm on dest host logging.debug("Clean up vm on dest host before migration") if dname: cleanup_vm(vm, dname, dest_uri) else: cleanup_vm(vm, vm.name, dest_uri) # Prepare host env: set selinux state before migration logging.debug("Set selinux to enforcing before migration") utils_selinux.set_status(params.get("selinux_state", "enforcing")) # Check vm network connectivity by ping before migration logging.debug("Check vm network before migration") if src_vm_status == "running": obj_migration.ping_vm(vm, params) # Get VM uptime before migration if src_vm_status == "running": vm_uptime = vm.uptime() logging.info("Check VM uptime before migration: %s", vm_uptime) # Print vm active xml before migration process.system_output("virsh dumpxml %s --security-info" % vm.name, shell=True) # Print vm inactive xml before migration process.system_output("virsh dumpxml %s --security-info --inactive" % vm.name, shell=True) # Do uni-direction migration. # NOTE: vm.connect_uri will be set to dest_uri once migration is complete successfully logging.debug("Start to do migration test.") vms.append(vm) if postcopy: # Monitor the qemu monitor event of "postcopy-active" for postcopy migration logging.debug( "Monitor the qemu monitor event for postcopy migration") virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True) cmd = "qemu-monitor-event --loop --domain %s --event MIGRATION" % vm.name virsh_session.sendline(cmd) # Do live migration and switch to postcopy by "virsh migrate-postcopy" logging.debug("Start to do postcopy migration") obj_migration.do_migration(vms, src_uri, dest_uri, "orderly", options="", thread_timeout=postcopy_timeout, ignore_status=True, func=virsh.migrate_postcopy, extra_opts=extra_options, shell=True) # Check migration result obj_migration.check_result(obj_migration.ret, params) # Check "postcopy-active" event after postcopy migration logging.debug( "Check postcopy-active event after postcopy migration") virsh_session.send_ctrl("^C") events_output = virsh_session.get_stripped_output() logging.debug("events_output are %s", events_output) pattern = "postcopy-active" if not re.search(pattern, events_output): test.fail("Migration didn't switch to postcopy mode") virsh_session.close() virsh_session.close() else: logging.debug("Start to do precopy migration") obj_migration.do_migration(vms, src_uri, dest_uri, "orderly", options="", ignore_status=True, extra_opts=extra_options) # Check migration result obj_migration.check_result(obj_migration.ret, params) """ # Check src vm after migration # First, update vm name and connect_uri to src vm's """ vm.name = vm_name_backup vm.connect_uri = src_uri logging.debug("Start to check %s state on src %s after migration.", vm.name, src_uri) # Check src vm status after migration: existence, running, shutoff, etc logging.debug("Check vm status on source after migration") if offline_migration: if src_vm_status == "shut off" and undefinesource: if vm.exists(): result_check_pass = False logging.error( "Src vm should not exist after offline migration" " with --undefinesource") logging.debug("Src vm state is %s" % vm.state()) elif not libvirt.check_vm_state( vm.name, src_vm_status, uri=vm.connect_uri): result_check_pass = False logging.error("Src vm should be %s after offline migration" % src_vm_status) logging.debug("Src vm state is %s" % vm.state()) if live_migration: if not undefinesource and src_vm_cfg == "persistent": if not libvirt.check_vm_state( vm.name, "shut off", uri=vm.connect_uri): result_check_pass = False logging.error( "Src vm should be shutoff after live migration") logging.debug("Src vm state is %s" % vm.state()) elif vm.exists(): result_check_pass = False logging.error("Src vm should not exist after live migration") logging.debug("Src vm state is %s" % vm.state()) # Check src vm status after migration: persistency logging.debug("Check vm persistency on source after migration") if src_vm_cfg == "persistent" and not undefinesource: if not vm.is_persistent(): # Src vm should be persistent after migration without --undefinesource result_check_pass = False logging.error("Src vm should be persistent after migration") elif vm.is_persistent(): result_check_pass = False logging.error("Src vm should be not be persistent after migration") """ # Check dst vm after migration # First, update vm name and connect_uri to dst vm's """ vm.name = dname if dname else vm.name vm.connect_uri = dest_uri logging.debug("Start to check %s state on target %s after migration.", vm.name, vm.connect_uri) # Check dst vm status after migration: running, shutoff, etc logging.debug("Check vm status on target after migration") if live_migration: if not libvirt.check_vm_state( vm.name, src_vm_status, uri=vm.connect_uri): result_check_pass = False logging.error("Dst vm should be %s after live migration", src_vm_status) elif vm.is_alive(): result_check_pass = False logging.error("Dst vm should not be alive after offline migration") # Print vm active xml after migration process.system_output("virsh -c %s dumpxml %s --security-info" % (vm.connect_uri, vm.name), shell=True) # Print vm inactive xml after migration process.system_output( "virsh -c %s dumpxml %s --security-info --inactive" % (vm.connect_uri, vm.name), shell=True) # Check dst vm xml after migration logging.debug("Check vm xml on target after migration") remote_virsh = virsh.Virsh(uri=vm.connect_uri) vmxml_active_tmp = vm_xml.VMXML.new_from_dumpxml( vm.name, "--security-info", remote_virsh) vmxml_inactive_tmp = vm_xml.VMXML.new_from_inactive_dumpxml( vm.name, "--security-info", remote_virsh) # Check dst vm xml after migration: --xml <updated_xml_file> if xml_option and not offline_migration: logging.debug("Check vm active xml for --xml") if not vmxml_active_tmp.title == updated_title: print("vmxml active tmp title is %s" % vmxml_active_tmp.title) result_check_pass = False logging.error("--xml doesn't take effect in migration") if xml_option and offline_migration: logging.debug("Check vm inactive xml for --xml") if not vmxml_active_tmp.title == updated_title: result_check_pass = False logging.error("--xml doesn't take effect in migration") # Check dst vm xml after migration: --persistent-xml <updated_xml_file> if persistent_xml_option: logging.debug("Check vm inactive xml for --persistent-xml") if not offline_migration and not vmxml_inactive_tmp.title == updated_persist_title: print("vmxml inactive tmp title is %s" % vmxml_inactive_tmp.title) result_check_pass = False logging.error( "--persistent-xml doesn't take effect in live migration") elif offline_migration and vmxml_inactive_tmp.title == updated_persist_title: result_check_pass = False logging.error( "--persistent-xml should not take effect in offline " "migration") # Check dst vm xml after migration: graphic passwd if with_graphic_passwd == "yes": logging.debug("Check graphic passwd in vm xml after migration") graphic_active = vmxml_active_tmp.devices.by_device_tag( 'graphics')[0] graphic_inactive = vmxml_inactive_tmp.devices.by_device_tag( 'graphics')[0] try: logging.debug("Check graphic passwd in active vm xml") if graphic_active.passwd != graphic_passwd: result_check_pass = False logging.error( "Graphic passwd in active xml of dst vm should be %s", graphic_passwd) logging.debug("Check graphic passwd in inactive vm xml") if graphic_inactive.passwd != graphic_passwd: result_check_pass = False logging.error( "Graphic passwd in inactive xml of dst vm should be %s", graphic_passwd) except LibvirtXMLNotFoundError: result_check_pass = False logging.error("Graphic passwd lost in dst vm xml") # Check dst vm uptime, network, etc after live migration if live_migration: # Check dst VM uptime after migration # Note: migrated_vm_uptime should be greater than the vm_uptime got # before migration migrated_vm_uptime = vm.uptime(connect_uri=dest_uri) logging.info( "Check VM uptime in destination after " "migration: %s", migrated_vm_uptime) if not migrated_vm_uptime: result_check_pass = False logging.error("Failed to check vm uptime after migration") elif vm_uptime > migrated_vm_uptime: result_check_pass = False logging.error( "VM went for a reboot while migrating to destination") # Check dst VM network connectivity after migration logging.debug("Check VM network connectivity after migrating") obj_migration.ping_vm(vm, params, uri=dest_uri) # Restore vm.connect_uri as it is set to src_uri in ping_vm() logging.debug( "Restore vm.connect_uri as it is set to src_uri in ping_vm()") vm.connect_uri = dest_uri # Check dst vm status after migration: persistency logging.debug("Check vm persistency on target after migration") if persistent: if not vm.is_persistent(): result_check_pass = False logging.error("Dst vm should be persistent after migration " "with --persistent") time.sleep(10) # Destroy vm and check vm state should be shutoff. BZ#1076354 vm.destroy() if not libvirt.check_vm_state( vm.name, "shut off", uri=vm.connect_uri): result_check_pass = False logging.error( "Dst vm with name %s should exist and be shutoff", vm.name) elif vm.is_persistent(): result_check_pass = False logging.error("Dst vm should not be persistent after migration " "without --persistent") finally: logging.debug("Start to clean up env") # Clean up vm on dest and src host for vm in vms: cleanup_vm(vm, vm_name=dname, uri=dest_uri) cleanup_vm(vm, vm_name=vm_name_backup, uri=src_uri) # Recover source vm defination (just in case). logging.info("Recover vm defination on source") if vm_xml_backup: vm_xml_backup.define() # Clean up SSH, TCP, TLS test env if objs_list and len(objs_list) > 0: logging.debug("Clean up test env: SSH, TCP, TLS, etc") for obj in objs_list: obj.auto_recover = True obj.__del__() # Disable libvirtd remote connection transport port obj_migration.migrate_pre_setup(dest_uri, params, cleanup=True, ports=transport_port) # Check test result. if not result_check_pass: test.fail("Migration succeed, but some check points didn't pass." "Please check the error log for details")
def run(test, params, env): """ Test watchdog device: 1.Add watchdog device to the guest xml. 2.Start the guest. 3.Trigger the watchdog in the guest. 4.Confirm the guest status. """ def trigger_watchdog(model): """ Trigger watchdog :param model: action when watchdog triggered """ watchdog_device = "device %s" % model if action == "dump": watchdog_action = "watchdog-action pause" else: watchdog_action = "watchdog-action %s" % action vm_pid = vm.get_pid() with open("/proc/%s/cmdline" % vm_pid) as vm_cmdline_file: vm_cmdline = vm_cmdline_file.read() vm_cmdline = vm_cmdline.replace('\x00', ' ') if not all(option in vm_cmdline for option in (watchdog_device, watchdog_action)): test.fail("Can not find %s or %s in qemu cmd line" % (watchdog_device, watchdog_action)) cmd = "gsettings set org.gnome.settings-daemon.plugins.power button-power shutdown" session.cmd(cmd, ignore_all_errors=True) try: if model == "ib700": try: session.cmd("modprobe ib700wdt") except aexpect.ShellCmdError: session.close() test.fail("Failed to load module ib700wdt") session.cmd("dmesg | grep %s && lsmod | grep %s" % (model, model)) session.cmd("echo 1 > /dev/watchdog") except aexpect.ShellCmdError: session.close() test.fail("Failed to trigger watchdog") def confirm_guest_status(): """ Confirm the guest status after watchdog triggered """ def _booting_completed(): session = vm.wait_for_login() output = session.cmd_status_output("last reboot") second_boot_time = output[1].strip().split("\n")[0].split()[-4] logging.debug(second_boot_time) session.close() return second_boot_time > first_boot_time def _inject_nmi(): session = vm.wait_for_login() status, output = session.cmd_status_output("dmesg | grep -i nmi") session.close() if status == 0: logging.debug(output) return True return False def _inject_nmi_event(): virsh_session.send_ctrl("^C") output = virsh_session.get_stripped_output() if "inject-nmi" not in output: return False return True def _check_dump_file(dump_path, domain_id): dump_file = glob.glob('%s%s-*' % (dump_path, domain_id)) if len(dump_file): logging.debug("Find the auto core dump file:\n%s", dump_file[0]) os.remove(dump_file[0]) return True return False if action in ["poweroff", "shutdown"]: if not utils_misc.wait_for(lambda: vm.state() == "shut off", 180, 10): test.fail("Guest not shutdown after watchdog triggered") elif action == "reset": if not utils_misc.wait_for(_booting_completed, 600, 10): test.fail("Guest not reboot after watchdog triggered") elif action == "pause": if utils_misc.wait_for(lambda: vm.state() == "paused", 180, 10): cmd_output = virsh.domstate(vm_name, '--reason').stdout.strip() logging.debug("Check guest status: %s\n", cmd_output) if cmd_output != "paused (watchdog)": test.fail( "The domstate is not correct after dump by watchdog") else: test.fail("Guest not pause after watchdog triggered") elif action == "none" and utils_misc.wait_for( lambda: vm.state() == "shut off", 180, 10): test.fail("Guest shutdown unexpectedly") elif action == "inject-nmi": if not utils_misc.wait_for(_inject_nmi, 180, 10): test.fail( "Guest not receive inject-nmi after watchdog triggered\n") elif not utils_misc.wait_for(_inject_nmi_event, 180, 10): test.fail("No inject-nmi watchdog event caught") virsh_session.close() elif action == "dump": domain_id = vm.get_id() dump_path = "/var/lib/libvirt/qemu/dump/" if not utils_misc.wait_for( lambda: _check_dump_file(dump_path, domain_id), 180, 10): test.fail( "No auto core dump file found after watchdog triggered") name_length = params.get("name_length", "default") vm_name = params.get("main_vm", "avocado-vt-vm1") vm = env.get_vm(params["main_vm"]) model = params.get("model") action = params.get("action") # Backup xml file vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) backup_xml = vmxml.copy() # Rename the guest name to the length defined in the config file if name_length != "default": origin_name = vm_name name_length = int(params.get("name_length", "1")) vm_name = ''.join([ random.choice(string.ascii_letters + string.digits) for _ in range(name_length) ]) vm_xml.VMXML.vm_rename(vm, vm_name) # Generate the renamed xml file vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name) # Add watchdog device to domain vmxml.remove_all_device_by_type('watchdog') watchdog_dev = Watchdog() watchdog_dev.model_type = model watchdog_dev.action = action chars = string.ascii_letters + string.digits + '-_' alias_name = 'ua-' + ''.join(random.choice(chars) for _ in list(range(64))) watchdog_dev.alias = {'name': alias_name} vmxml.add_device(watchdog_dev) vmxml.sync() try: vm.start() session = vm.wait_for_login() if action == "reset": output = session.cmd_status_output("last reboot") first_boot_time = output[1].strip().split("\n")[0].split()[-4] logging.info("The first boot time is %s\n", first_boot_time) if action == "inject-nmi": virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True) event_cmd = "event --event watchdog --all --loop" virsh_session.sendline(event_cmd) trigger_watchdog(model) confirm_guest_status() finally: if vm.is_alive(): vm.destroy(gracefully=False) backup_xml.sync()