def monitor_event(params):
"""
Monitor event on source/target host
:param params: dict, used to setup the connection
:return: virsh session and remote virsh session to catch events
"""
expected_event_src = params.get("expected_event_src")
expected_event_target = params.get("expected_event_target")
remote_pwd = params.get("migrate_dest_pwd")
remote_ip = params.get("migrate_dest_host")
remote_user = params.get("remote_user", "root")
cmd = "event --loop --all"
if expected_event_src:
logging.debug("Running virsh command on source: %s", cmd)
virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC,
auto_close=True)
virsh_session.sendline(cmd)
if expected_event_target:
logging.debug("Running virsh command on target: %s", cmd)
virsh_dargs = {
'remote_ip': remote_ip,
'remote_user': remote_user,
'remote_pwd': remote_pwd,
'unprivileged_user': None,
'virsh_exec': virsh.VIRSH_EXEC,
'auto_close': True,
'uri': 'qemu+ssh://%s/system' % remote_ip
}
remote_virsh_session = virsh.VirshSession(**virsh_dargs)
remote_virsh_session.sendline(cmd)
return virsh_session, remote_virsh_session
def set_migratepostcopy(self, vm_name, uri=None):
"""
Switch to postcopy during migration.
:param vm_name: vm's name
:param uri: target virsh uri
:raise: test.error when command fails
"""
cmd = "event --loop --all"
virsh_event_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC,
auto_close=True,
uri=uri)
virsh_event_session.sendline(cmd)
if not utils_misc.wait_for(
lambda: not virsh.migrate_postcopy(
vm_name, uri=uri, debug=True).exit_status, 10):
raise exceptions.TestError("Failed to set migration postcopy.")
exp_str = "Suspended Post-copy"
if not utils_misc.wait_for(
lambda: re.findall(
exp_str, virsh_event_session.get_stripped_output()), 30):
raise exceptions.TestError(
"Unalbe to find event {}".format(exp_str))
def _run_collect_event_cmd():
"""
To execute virsh event command to collect the domain events
:return: VirshSession to retrieve the events
"""
cmd = "event --loop --all"
virsh_event_session = virsh.VirshSession(
virsh_exec=virsh.VIRSH_EXEC, auto_close=True, uri=srcuri)
virsh_event_session.sendline(cmd)
logging.debug("Begin to collect domain events...")
return virsh_event_session
def set_secret_value(test, params):
"""
Set the secet value
:params: the parameter dictionary
"""
uuid = params.get("secret_ref")
options = params.get("set_secret_options")
status_error = params.get("status_error", "no")
secret_string = params.get("secret_base64_no_encoded")
secret_encode = "yes" == params.get("secret_string_base64_encode", "yes")
secret_file = "yes" == params.get("secret_file", "no")
if options and "interactive" in options:
cmd = "secret-set-value %s --%s" % (uuid, options)
virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC,
auto_close=True)
virsh_session.sendline(cmd)
# Wait for 5s
time.sleep(5)
virsh_session.sendline(secret_string)
# Wait for 5s to get stripped output
time.sleep(5)
output = virsh_session.get_stripped_output()
exit_status = 0 if "Secret value set" in output else 1
result = process.CmdResult(cmd, output, output, exit_status)
else:
result = virsh.secret_set_value(uuid,
secret_string,
options=options,
encode=secret_encode,
use_file=secret_file,
debug=True)
status = result.exit_status
# Don't check result if we don't need to.
if params.get("check_set_status", "yes") == "no":
return
# Check status_error
if status_error == "yes":
if status:
logging.info("It's an expected %s", result.stderr)
else:
test.fail("%d not a expected command " "return value", status)
elif status_error == "no":
if status:
test.fail(result.stderr)
else:
# Check secret value
if check_secret(params):
logging.info(result.stdout.strip())
else:
test.fail("The secret value " "mismatch with result")
def setup_blockjob_raw():
"""
Prepare running domain and do blockcopy
"""
if not vm.is_alive():
vm.start()
if os.path.exists(tmp_copy_path):
process.run('rm -rf %s' % tmp_copy_path)
cmd = "blockcopy %s %s %s --wait --verbose --transient-job " \
"--bandwidth 1000 " % (vm_name, dev, tmp_copy_path)
virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC,
auto_close=True)
virsh_session.sendline(cmd)
def run(test, params, env):
"""
Test watchdog device:
1.Add watchdog device to the guest xml.
2.Start the guest.
3.Trigger the watchdog in the guest.
4.Confirm the guest status.
"""
def trigger_watchdog(model):
"""
Trigger watchdog
:param model: action when watchdog triggered
"""
watchdog_device = "device %s" % model
if action == "dump":
watchdog_action = "watchdog-action pause"
else:
watchdog_action = "watchdog-action %s" % action
if not hotplug_test:
vm_pid = vm.get_pid()
with open("/proc/%s/cmdline" % vm_pid) as vm_cmdline_file:
vm_cmdline = vm_cmdline_file.read()
vm_cmdline = vm_cmdline.replace('\x00', ' ')
if not all(option in vm_cmdline for option in (watchdog_device, watchdog_action)):
test.fail("Can not find %s or %s in qemu cmd line"
% (watchdog_device, watchdog_action))
cmd = "gsettings set org.gnome.settings-daemon.plugins.power button-power shutdown"
session.cmd(cmd, ignore_all_errors=True)
try:
if model == "ib700":
try:
session.cmd("modprobe ib700wdt")
except aexpect.ShellCmdError:
session.close()
test.fail("Failed to load module ib700wdt")
session.cmd("dmesg | grep -i %s && lsmod | grep %s" % (model, model))
session.cmd("echo 1 > /dev/watchdog")
except aexpect.ShellCmdError as e:
session.close()
test.fail("Failed to trigger watchdog: %s" % e)
def watchdog_attached(vm_name):
"""
Confirm whether watchdog device is attached to vm by checking domain dumpxml
:param vm_name: vm name
"""
vmxml = vm_xml.VMXML.new_from_dumpxml(vm_name)
if vmxml.xmltreefile.find('devices/watchdog'):
return True
else:
return False
def confirm_guest_status():
"""
Confirm the guest status after watchdog triggered
"""
def _booting_completed():
session = vm.wait_for_login()
status = None
second_boot_time = None
try:
status, second_boot_time = session.cmd_status_output("uptime --since")
logging.debug("The second boot time is %s", second_boot_time)
except (aexpect.ShellStatusError, aexpect.ShellProcessTerminatedError) as e:
logging.error("Exception caught:%s", e)
session.close()
return second_boot_time > first_boot_time
def _inject_nmi():
session = vm.wait_for_login()
status, output = session.cmd_status_output("dmesg | grep -i nmi")
session.close()
if status == 0:
logging.debug(output)
return True
return False
def _inject_nmi_event():
virsh_session.send_ctrl("^C")
output = virsh_session.get_stripped_output()
if "inject-nmi" not in output:
return False
return True
def _check_dump_file(dump_path, domain_id):
dump_file = glob.glob('%s%s-*' % (dump_path, domain_id))
if len(dump_file):
logging.debug("Find the auto core dump file:\n%s", dump_file[0])
os.remove(dump_file[0])
return True
return False
if action in ["poweroff", "shutdown"]:
if not utils_misc.wait_for(lambda: vm.state() == "shut off", 180, 10):
test.fail("Guest not shutdown after watchdog triggered")
else:
logging.debug("Guest is in shutdown state after watchdog triggered")
elif action == "reset":
if not utils_misc.wait_for(_booting_completed, 600, 10):
test.fail("Guest not reboot after watchdog triggered")
else:
logging.debug("Guest is rebooted after watchdog triggered")
elif action == "pause":
if utils_misc.wait_for(lambda: vm.state() == "paused", 180, 10):
logging.debug("Guest is in paused status after watchdog triggered.")
cmd_output = virsh.domstate(vm_name, '--reason').stdout.strip()
logging.debug("Check guest status: %s\n", cmd_output)
if cmd_output != "paused (watchdog)":
test.fail("The domstate is not correct after dump by watchdog")
else:
test.fail("Guest not pause after watchdog triggered")
elif action == "none":
if utils_misc.wait_for(lambda: vm.state() == "shut off", 180, 10):
test.fail("Guest shutdown unexpectedly")
else:
logging.debug("Guest is not in shutoff state since watchdog action is none.")
elif action == "inject-nmi":
if not utils_misc.wait_for(_inject_nmi, 180, 10):
test.fail("Guest not receive inject-nmi after watchdog triggered\n")
elif not utils_misc.wait_for(_inject_nmi_event, 180, 10):
test.fail("No inject-nmi watchdog event caught")
else:
logging.debug("Guest received inject-nmi and inject-nmi watchdog event "
" has been caught.")
virsh_session.close()
elif action == "dump":
domain_id = vm.get_id()
dump_path = "/var/lib/libvirt/qemu/dump/"
if not utils_misc.wait_for(lambda: _check_dump_file(dump_path, domain_id), 180, 10):
test.fail("No auto core dump file found after watchdog triggered")
else:
logging.debug("VM core has been dumped after watchdog triggered.")
name_length = params.get("name_length", "default")
vm_name = params.get("main_vm", "avocado-vt-vm1")
vm = env.get_vm(params["main_vm"])
model = params.get("model")
action = params.get("action")
model_test = params.get("model_test") == "yes"
hotplug_test = params.get("hotplug_test") == "yes"
hotunplug_test = params.get("hotunplug_test") == "yes"
machine_type = params.get("machine_type")
if machine_type == "q35" and model == "ib700":
test.cancel("ib700wdt watchdog device is not supported "
"on guest with q35 machine type")
# Backup xml file
vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
backup_xml = vmxml.copy()
# Rename the guest name to the length defined in the config file
if name_length != "default":
origin_name = vm_name
name_length = int(params.get("name_length", "1"))
vm_name = ''.join([random.choice(string.ascii_letters+string.digits)
for _ in range(name_length)])
vm_xml.VMXML.vm_rename(vm, vm_name)
# Generate the renamed xml file
vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
# Coldplug pcie-to-pci-bridge to vm xml for q35 guest as i6300esb watchdog
# device can only be plugged to conventional PCI slot
if (machine_type == 'q35'
and not vmxml.get_controllers(controller_type='pci', model='pcie-to-pci-bridge')):
logging.debug("Add pcie-root-port and pcie-to-pci-bridge controller to vm")
pcie_root_port = Controller("pci")
pcie_pci_bridge = Controller("pci")
pcie_root_port.model = "pcie-root-port"
pcie_pci_bridge.model = "pcie-to-pci-bridge"
pcie_root_port.model_name = {'name': 'pcie-root-port'}
pcie_pci_bridge.model_name = {'name': 'pcie-pci-bridge'}
vmxml.add_device(pcie_root_port)
vmxml.add_device(pcie_pci_bridge)
vmxml.sync()
if hotplug_test:
vm.start()
session = vm.wait_for_login()
# Add watchdog device to domain
vmxml.remove_all_device_by_type('watchdog')
watchdog_dev = Watchdog()
watchdog_dev.model_type = model
watchdog_dev.action = action
chars = string.ascii_letters + string.digits + '-_'
alias_name = 'ua-' + ''.join(random.choice(chars) for _ in list(range(64)))
watchdog_dev.alias = {'name': alias_name}
try:
if model_test or hotunplug_test:
vmxml.add_device(watchdog_dev)
vmxml.sync()
try:
vm.start()
except Exception:
test.fail("VM startup after adding watchdog device failed!")
elif hotplug_test:
watchdog_xml = watchdog_dev.xml
attach_result = virsh.attach_device(vm_name, watchdog_xml,
ignore_status=False, debug=True)
if not utils_misc.wait_for(lambda: watchdog_attached(vm.name), 60):
test.fail("Failed to hotplug watchdog device.")
session = vm.wait_for_login()
# No need to trigger watchdog after hotunplug
if hotunplug_test:
cur_xml = vm_xml.VMXML.new_from_dumpxml(vm_name)
cur_watchdog = cur_xml.xmltreefile.find('devices/watchdog')
cur_watchdog_xml = Watchdog.new_from_element(cur_watchdog).xml
detach_result = virsh.detach_device(vm_name, cur_watchdog_xml,
ignore_status=True, debug=True)
if detach_result.exit_status:
test.fail("i6300esb watchdog device can NOT be detached successfully, "
"result:\n%s" % detach_result)
elif not utils_misc.wait_for(lambda: not watchdog_attached(vm.name), 60):
test.fail("Failed to hotunplug watchdog device.")
return
if action == "reset":
status, first_boot_time = session.cmd_status_output("uptime --since")
logging.info("The first boot time is %s\n", first_boot_time)
if action == "inject-nmi":
virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC, auto_close=True)
event_cmd = "event --event watchdog --all --loop"
virsh_session.sendline(event_cmd)
trigger_watchdog(model)
confirm_guest_status()
finally:
if vm.is_alive():
vm.destroy(gracefully=False)
if name_length != "default":
vm_xml.VMXML.vm_rename(vm, origin_name)
backup_xml.sync()
def _migrate(self):
"""
1.Set selinux state
2.Record vm uptime
3.For postcopy migration:
1) Set migration speed to low value
2) Monitor postcopy event
4.Do live migration
5.Check migration result: succeed or fail with expected error
6.For postcopy migration: check postcopy event
7.Do post migration check: check vm state, uptime, network
"""
# Set selinux state before migration
# NOTE: if selinux state is set too early, it may be changed
# in other methods unexpectedly, so set it just before migration
logging.debug("Set selinux to enforcing before migration")
utils_selinux.set_status(self.selinux_state)
# TODO: Set selinux on migrate_dest_host
# Check vm uptime before migration
logging.debug("Check vm uptime before migration")
self.uptime = {}
for vm in self.vms:
self.uptime[vm.name] = vm.uptime(connect_uri=vm.connect_uri)
# Do postcopy/precopy related operations/setting
if self.migrate_flags & VIR_MIGRATE_POSTCOPY:
# Set migration speed to low value in case it finished too early
# before postcopy mode starts
for vm in self.vms:
virsh.migrate_setspeed(vm.name, 1, uri=vm.connect_uri)
# Monitor event "Suspended Post-copy" for postcopy migration
logging.debug("Monitor the event for postcopy migration")
virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC,
auto_close=True)
self.objs_list.append(virsh_session)
cmd = "event %s --loop --all --timestamp" % self.main_vm.name
virsh_session.sendline(cmd)
# Set func to be executed during postcopy migration
func = virsh.migrate_postcopy
else:
# Set func to be executed during precopy migration
func = None
# Start to do migration
logging.debug("Start to do migration")
thread_timeout = self.migrate_thread_timeout
self.obj_migration.do_migration(self.vms,
self.src_uri,
self.dest_uri,
"orderly",
options=self.virsh_migrate_options,
thread_timeout=thread_timeout,
ignore_status=True,
virsh_uri=self.src_uri,
func=func,
shell=True)
logging.info("Check migration result: succeed or"
" fail with expected error")
self.obj_migration.check_result(self.obj_migration.ret, self.params)
# Check "suspended post-copy" event after postcopy migration
if self.migrate_flags & VIR_MIGRATE_POSTCOPY:
logging.debug("Check event after postcopy migration")
virsh_session.send_ctrl("^c")
events_output = virsh_session.get_stripped_output()
logging.debug("Events_output are %s", events_output)
pattern = "Suspended Post-copy"
if pattern not in events_output:
self.test.error("Migration didn't switch to postcopy mode")
logging.debug("Do post migration check after migrate to dest")
self.params["migrate_options"] = self.virsh_migrate_options
self.obj_migration.post_migration_check(self.vms,
self.params,
self.uptime,
uri=self.dest_uri)
def run(test, params, env):
"""
Test command: virsh start.
1) Get the params from params.
2) Prepare libvirtd's status.
3) Do the start operation.
4) Result check.
5) clean up.
"""
# get the params from params
vm_name = params.get("main_vm", "avocado-vt-vm1")
vm_ref = params.get("vm_ref", "vm1")
opt = params.get("vs_opt", "")
# Backup for recovery.
vmxml_backup = libvirt_xml.vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
backup_name = vm_ref
vm = None
if vm_ref is not "":
vm = env.get_vm(vm_ref)
vmxml = libvirt_xml.VMXML()
libvirtd_state = params.get("libvirtd", "on")
pre_operation = params.get("vs_pre_operation", "")
status_error = params.get("status_error", "no")
try:
# prepare before start vm
if libvirtd_state == "on":
utils_libvirtd.libvirtd_start()
elif libvirtd_state == "off":
utils_libvirtd.libvirtd_stop()
if pre_operation == "rename":
new_vm_name = params.get("vs_new_vm_name", "virsh_start_vm1")
vm = libvirt_xml.VMXML.vm_rename(vm, new_vm_name)
vm_ref = new_vm_name
elif pre_operation == "undefine":
vmxml = vmxml.new_from_dumpxml(vm_ref)
vmxml.undefine()
# do the start operation
try:
if pre_operation == "remote":
# get the params for remote test
remote_ip = params.get("remote_ip", "ENTER.YOUR.REMOTE.IP")
remote_user = params.get("remote_user", "root")
remote_pwd = params.get("remote_pwd",
"ENTER.YOUR.REMOTE.PASSWORD")
if pre_operation == "remote" and remote_ip.count(
"ENTER.YOUR."):
test.cancel("Remote test parameters not configured")
ssh_key.setup_ssh_key(remote_ip, remote_user, remote_pwd)
remote_uri = "qemu+ssh://%s/system" % remote_ip
cmd_result = virsh.start(vm_ref,
ignore_status=True,
debug=True,
uri=remote_uri)
if cmd_result.exit_status:
test.fail("Start vm failed.\n Detail: %s" % cmd_result)
elif opt.count("console"):
# With --console, start command will print the
# dmesg of guest in starting and turn into the
# login prompt. In this case, we start it with
# --console and login vm in console by
# remote.handle_prompts().
cmd = "start %s --console" % vm_ref
virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC,
auto_close=True)
virsh_session.sendline(cmd)
remote.handle_prompts(virsh_session,
params.get("username", ""),
params.get("password", ""),
r"[\#\$]\s*$",
timeout=60,
debug=True)
elif opt.count("autodestroy"):
# With --autodestroy, vm will be destroyed when
# virsh session closed. Then we execute start
# command in a virsh session and start vm with
# --autodestroy. Then we closed the virsh session,
# and check the vm is destroyed or not.
virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC,
auto_close=True)
cmd = "start %s --autodestroy" % vm_ref
status = virsh_session.cmd_status(cmd)
if status:
test.fail("Failed to start vm with --autodestroy.")
# Close the session, then the vm shoud be destroyed.
virsh_session.close()
elif opt.count("force-boot"):
# With --force-boot, VM will be stared from boot
# even we have saved it with virsh managedsave.
# In this case, we start vm and execute sleep 1000&,
# then save it with virsh managedsave. At last, we
# start vm with --force-boot. To verify the result,
# we check the sleep process. If the process exists,
# force-boot failed, else case pass.
vm.start()
session = vm.wait_for_login()
status = session.cmd_status("sleep 1000&")
if status:
test.error("Can not execute command in guest.")
sleep_pid = session.cmd_output("echo $!").strip()
virsh.managedsave(vm_ref)
virsh.start(vm_ref, options=opt)
else:
cmd_result = virsh.start(vm_ref, options=opt)
if cmd_result.exit_status:
if status_error == "no":
test.fail("Start vm failed.\n Detail: %s" % cmd_result)
else:
# start vm successfully
if status_error == "yes":
test.fail("Run successfully with wrong "
"command!\n Detail:%s" % cmd_result)
if opt.count("paused"):
if not (vm.state() == "paused"):
test.fail("VM is not paused when started with "
"--paused.")
elif opt.count("autodestroy"):
if vm.is_alive():
test.fail("VM was started with --autodestroy,"
"but not destroyed when virsh session "
"closed.")
elif opt.count("force-boot"):
session = vm.wait_for_login()
status = session.cmd_status("ps %s |grep '[s]leep 1000'" %
sleep_pid)
if not status:
test.fail("VM was started with --force-boot,"
"but it is restored from a"
" managedsave.")
else:
if status_error == "no" and not vm.is_alive(
) and pre_operation != "remote":
test.fail("VM was started but it is not alive.")
except remote.LoginError as detail:
test.fail("Failed to login guest.")
finally:
# clean up
if libvirtd_state == "off":
utils_libvirtd.libvirtd_start()
elif pre_operation == "rename":
libvirt_xml.VMXML.vm_rename(vm, backup_name)
elif pre_operation == "remote":
virsh.destroy(vm_ref,
ignore_status=False,
debug=True,
uri=remote_uri)
if vm and vm.is_paused():
vm.resume()
# Restore VM
vmxml_backup.sync()
def run(test, params, env):
"""
Test virsh migrate command.
"""
def cleanup_vm(vm, vm_name='', uri=''):
"""
Clean up vm in the src or destination host environment
when doing the uni-direction migration.
"""
# Backup vm name and uri
uri_bak = vm.connect_uri
vm_name_bak = vm.name
# Destroy and undefine vm
vm.connect_uri = uri if uri else uri_bak
vm.name = vm_name if vm_name else vm_name_bak
logging.info("Cleaning up VM %s on %s", vm.name, vm.connect_uri)
if vm.is_alive():
vm.destroy()
if vm.is_persistent():
vm.undefine()
# Restore vm connect_uri
vm.connect_uri = uri_bak
vm.name = vm_name_bak
# Check whether there are unset parameters
for v in list(itervalues(params)):
if isinstance(v, string_types) and v.count("EXAMPLE"):
test.cancel("Please set real value for %s" % v)
# Params for virsh migrate options:
live_migration = params.get("live_migration") == "yes"
offline_migration = params.get("offline_migration") == "yes"
persistent = params.get("persistent") == "yes"
undefinesource = params.get("undefinesource") == "yes"
p2p = params.get("p2p") == "yes"
tunnelled = params.get("tunnelled") == "yes"
postcopy = params.get("postcopy") == "yes"
dname = params.get("dname")
xml_option = params.get("xml_option") == "yes"
persistent_xml_option = params.get("persistent_xml_option") == "yes"
extra_options = params.get("virsh_migrate_extra", "")
if live_migration and not extra_options.count("--live"):
extra_options = "%s --live" % extra_options
if offline_migration and not extra_options.count("--offline"):
extra_options = "%s --offline" % extra_options
if persistent and not extra_options.count("--persistent"):
extra_options = "%s --persistent" % extra_options
if undefinesource and not extra_options.count("--undefinesource"):
extra_options = "%s --undefinesource" % extra_options
if p2p and not extra_options.count("--p2p"):
extra_options = "%s --p2p" % extra_options
if tunnelled and not extra_options.count("--tunnelled"):
extra_options = "%s --tunnelled" % extra_options
if tunnelled and not extra_options.count("--p2p"):
extra_options = "%s --p2p" % extra_options
if postcopy and not extra_options.count("--postcopy"):
extra_options = "%s --postcopy" % extra_options
if dname and not extra_options.count("--dname"):
extra_options = "%s --dname %s" % (extra_options, dname)
if xml_option:
pass
if persistent_xml_option and not extra_options.count("--persistent"):
extra_options = "%s --persistent" % extra_options
if persistent_xml_option:
pass
# Set param migrate_options in case it is used somewhere:
params.setdefault("migrate_options", extra_options)
# Params for postcopy migration
postcopy_timeout = int(params.get("postcopy_migration_timeout", "180"))
# Params for migrate hosts:
server_cn = params.get("server_cn")
client_cn = params.get("client_cn")
migrate_source_host = client_cn if client_cn else params.get(
"migrate_source_host")
migrate_dest_host = server_cn if server_cn else params.get(
"migrate_dest_host")
# Params for migrate uri
transport = params.get("transport", "tls")
transport_port = params.get("transport_port")
uri_port = ":%s" % transport_port if transport_port else ''
hypervisor_driver = params.get("hypervisor_driver", "qemu")
hypervisor_mode = params.get("hypervisor_mode", 'system')
if "virsh_migrate_desturi" not in list(params.keys()):
params["virsh_migrate_desturi"] = "%s+%s://%s%s/%s" % (
hypervisor_driver, transport, migrate_dest_host, uri_port,
hypervisor_mode)
if "virsh_migrate_srcuri" not in list(params.keys()):
params["virsh_migrate_srcuri"] = "%s:///%s" % (hypervisor_driver,
hypervisor_mode)
dest_uri = params.get("virsh_migrate_desturi")
src_uri = params.get("virsh_migrate_srcuri")
# Params for src vm cfg:
src_vm_cfg = params.get("src_vm_cfg")
src_vm_status = params.get("src_vm_status")
with_graphic_passwd = params.get("with_graphic_passwd")
graphic_passwd = params.get("graphic_passwd")
# For test result check
cancel_exception = False
fail_exception = False
exception = False
result_check_pass = True
# Objects(SSH, TLS and TCP, etc) to be cleaned up in finally
objs_list = []
# VM objects for migration test
vms = []
try:
# Get a MigrationTest() Object
logging.debug("Get a MigrationTest() object")
obj_migration = migration.MigrationTest()
# Setup libvirtd remote connection TLS connection env
if transport == "tls":
tls_obj = TLSConnection(params)
# Setup CA, server(on dest host) and client(on src host)
tls_obj.conn_setup()
# Add tls_obj to objs_list
objs_list.append(tls_obj)
# Enable libvirtd remote connection transport port
if transport == 'tls':
transport_port = '16514'
elif transport == 'tcp':
transport_port = '16509'
obj_migration.migrate_pre_setup(dest_uri, params, ports=transport_port)
# Back up vm name for recovery in finally
vm_name_backup = params.get("migrate_main_vm")
# Get a vm object for migration
logging.debug("Get a vm object for migration")
vm = env.get_vm(vm_name_backup)
# Back up vm xml for recovery in finally
logging.debug("Backup vm xml before migration")
vm_xml_backup = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name)
if not vm_xml_backup:
test.error("Backing up xmlfile failed.")
# Prepare shared disk in vm xml for live migration:
# Change the source of the first disk of vm to shared disk
if live_migration:
logging.debug("Prepare shared disk in vm xml for live migration")
storage_type = params.get("storage_type")
if storage_type == 'nfs':
logging.debug("Prepare nfs shared disk in vm xml")
nfs_mount_dir = params.get("nfs_mount_dir")
libvirt.update_vm_disk_source(vm.name, nfs_mount_dir)
libvirt.update_vm_disk_driver_cache(vm.name,
driver_cache="none")
else:
# TODO:Other storage types
test.cancel("Other storage type is not supported for now")
pass
# Prepare graphic password in vm xml
if with_graphic_passwd in ["yes", "no"]:
logging.debug("Set VM graphic passwd in vm xml")
# Get graphics list in vm xml
vmxml_tmp = vm_xml.VMXML.new_from_inactive_dumpxml(vm.name)
graphics_list = vmxml_tmp.get_graphics_devices
if not graphics_list:
# Add spice graphic with passwd to vm xml
logging.debug("Add spice graphic to vm xml")
graphics.Graphics.add_graphic(vm.name, graphic_passwd, "spice")
elif graphic_passwd:
# Graphics already exist in vm xml and passwd is required
# Add passwd to the first graphic device in vm xml
logging.debug("Add graphic passwd to vm xml")
vm_xml.VMXML.add_security_info(vmxml_tmp, graphic_passwd)
vmxml_tmp.sync()
else:
# Graphics already exist in vm xml and non-passwd is required
# Do nothing here as passwd has been removed by new_from_inactive_dumpxml()
pass
# Prepare for required src vm status.
logging.debug("Turning %s into certain state.", vm.name)
if src_vm_status == "running" and not vm.is_alive():
vm.start()
elif src_vm_status == "shut off" and not vm.is_dead():
vm.destroy()
# Prepare for required src vm persistency.
logging.debug("Prepare for required src vm persistency")
if src_vm_cfg == "persistent" and not vm.is_persistent():
logging.debug("Make src vm persistent")
vm_xml_backup.define()
elif src_vm_cfg == "transient" and vm.is_persistent():
logging.debug("Make src vm transient")
vm.undefine()
# Prepare for postcopy migration: install and run stress in VM
if postcopy and src_vm_status == "running":
logging.debug(
"Install and run stress in vm for postcopy migration")
pkg_name = 'stress'
# Get a vm session
logging.debug("Get a vm session")
vm_session = vm.wait_for_login()
if not vm_session:
test.error("Can't get a vm session successfully")
# Install package stress if it is not installed in vm
logging.debug(
"Check if stress tool is installed for postcopy migration")
pkg_mgr = utils_package.package_manager(vm_session, pkg_name)
if not pkg_mgr.is_installed(pkg_name):
logging.debug("Stress tool will be installed")
if not pkg_mgr.install():
test.error("Package '%s' installation fails" % pkg_name)
# Run stress in vm
logging.debug("Run stress in vm")
stress_args = params.get("stress_args")
vm_session.cmd('stress %s' % stress_args)
# Prepare for --xml <updated_xml_file>.
if xml_option:
logging.debug("Preparing new xml file for --xml option.")
# Get the vm xml
vmxml_tmp = vm_xml.VMXML.new_from_dumpxml(
vm.name, "--security-info --migratable")
# Update something in the xml file: e.g. title
# Note: VM ABI shall not be broken when migrating with updated_xml
updated_title = "VM Title in updated xml"
vmxml_tmp.title = updated_title
# Add --xml to migrate extra_options
extra_options = ("%s --xml=%s" % (extra_options, vmxml_tmp.xml))
# Prepare for --persistent-xml <updated_xml_file>.
if persistent_xml_option:
logging.debug(
"Preparing new xml file for --persistent-xml option.")
# Get the vm xml
vmxml_persist_tmp = vm_xml.VMXML.new_from_inactive_dumpxml(
vm.name, "--security-info")
# Update something in the xml file: e.g. title
# Note: VM ABI shall not be broken when migrating with updated_xml
updated_persist_title = "VM Title in updated persist xml"
vmxml_persist_tmp.title = updated_persist_title
# Add --persistent-xml to migrate extra_options
extra_options = ("%s --persistent-xml=%s" %
(extra_options, vmxml_persist_tmp.xml))
# Prepare host env: clean up vm on dest host
logging.debug("Clean up vm on dest host before migration")
if dname:
cleanup_vm(vm, dname, dest_uri)
else:
cleanup_vm(vm, vm.name, dest_uri)
# Prepare host env: set selinux state before migration
logging.debug("Set selinux to enforcing before migration")
utils_selinux.set_status(params.get("selinux_state", "enforcing"))
# Check vm network connectivity by ping before migration
logging.debug("Check vm network before migration")
if src_vm_status == "running":
obj_migration.ping_vm(vm, params)
# Get VM uptime before migration
if src_vm_status == "running":
vm_uptime = vm.uptime()
logging.info("Check VM uptime before migration: %s", vm_uptime)
# Print vm active xml before migration
process.system_output("virsh dumpxml %s --security-info" % vm.name,
shell=True)
# Print vm inactive xml before migration
process.system_output("virsh dumpxml %s --security-info --inactive" %
vm.name,
shell=True)
# Do uni-direction migration.
# NOTE: vm.connect_uri will be set to dest_uri once migration is complete successfully
logging.debug("Start to do migration test.")
vms.append(vm)
if postcopy:
# Monitor the qemu monitor event of "postcopy-active" for postcopy migration
logging.debug(
"Monitor the qemu monitor event for postcopy migration")
virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC,
auto_close=True)
cmd = "qemu-monitor-event --loop --domain %s --event MIGRATION" % vm.name
virsh_session.sendline(cmd)
# Do live migration and switch to postcopy by "virsh migrate-postcopy"
logging.debug("Start to do postcopy migration")
obj_migration.do_migration(vms,
src_uri,
dest_uri,
"orderly",
options="",
thread_timeout=postcopy_timeout,
ignore_status=True,
func=virsh.migrate_postcopy,
extra_opts=extra_options,
shell=True)
# Check migration result
obj_migration.check_result(obj_migration.ret, params)
# Check "postcopy-active" event after postcopy migration
logging.debug(
"Check postcopy-active event after postcopy migration")
virsh_session.send_ctrl("^C")
events_output = virsh_session.get_stripped_output()
logging.debug("events_output are %s", events_output)
pattern = "postcopy-active"
if not re.search(pattern, events_output):
test.fail("Migration didn't switch to postcopy mode")
virsh_session.close()
virsh_session.close()
else:
logging.debug("Start to do precopy migration")
obj_migration.do_migration(vms,
src_uri,
dest_uri,
"orderly",
options="",
ignore_status=True,
extra_opts=extra_options)
# Check migration result
obj_migration.check_result(obj_migration.ret, params)
"""
# Check src vm after migration
# First, update vm name and connect_uri to src vm's
"""
vm.name = vm_name_backup
vm.connect_uri = src_uri
logging.debug("Start to check %s state on src %s after migration.",
vm.name, src_uri)
# Check src vm status after migration: existence, running, shutoff, etc
logging.debug("Check vm status on source after migration")
if offline_migration:
if src_vm_status == "shut off" and undefinesource:
if vm.exists():
result_check_pass = False
logging.error(
"Src vm should not exist after offline migration"
" with --undefinesource")
logging.debug("Src vm state is %s" % vm.state())
elif not libvirt.check_vm_state(
vm.name, src_vm_status, uri=vm.connect_uri):
result_check_pass = False
logging.error("Src vm should be %s after offline migration" %
src_vm_status)
logging.debug("Src vm state is %s" % vm.state())
if live_migration:
if not undefinesource and src_vm_cfg == "persistent":
if not libvirt.check_vm_state(
vm.name, "shut off", uri=vm.connect_uri):
result_check_pass = False
logging.error(
"Src vm should be shutoff after live migration")
logging.debug("Src vm state is %s" % vm.state())
elif vm.exists():
result_check_pass = False
logging.error("Src vm should not exist after live migration")
logging.debug("Src vm state is %s" % vm.state())
# Check src vm status after migration: persistency
logging.debug("Check vm persistency on source after migration")
if src_vm_cfg == "persistent" and not undefinesource:
if not vm.is_persistent():
# Src vm should be persistent after migration without --undefinesource
result_check_pass = False
logging.error("Src vm should be persistent after migration")
elif vm.is_persistent():
result_check_pass = False
logging.error("Src vm should be not be persistent after migration")
"""
# Check dst vm after migration
# First, update vm name and connect_uri to dst vm's
"""
vm.name = dname if dname else vm.name
vm.connect_uri = dest_uri
logging.debug("Start to check %s state on target %s after migration.",
vm.name, vm.connect_uri)
# Check dst vm status after migration: running, shutoff, etc
logging.debug("Check vm status on target after migration")
if live_migration:
if not libvirt.check_vm_state(
vm.name, src_vm_status, uri=vm.connect_uri):
result_check_pass = False
logging.error("Dst vm should be %s after live migration",
src_vm_status)
elif vm.is_alive():
result_check_pass = False
logging.error("Dst vm should not be alive after offline migration")
# Print vm active xml after migration
process.system_output("virsh -c %s dumpxml %s --security-info" %
(vm.connect_uri, vm.name),
shell=True)
# Print vm inactive xml after migration
process.system_output(
"virsh -c %s dumpxml %s --security-info --inactive" %
(vm.connect_uri, vm.name),
shell=True)
# Check dst vm xml after migration
logging.debug("Check vm xml on target after migration")
remote_virsh = virsh.Virsh(uri=vm.connect_uri)
vmxml_active_tmp = vm_xml.VMXML.new_from_dumpxml(
vm.name, "--security-info", remote_virsh)
vmxml_inactive_tmp = vm_xml.VMXML.new_from_inactive_dumpxml(
vm.name, "--security-info", remote_virsh)
# Check dst vm xml after migration: --xml <updated_xml_file>
if xml_option and not offline_migration:
logging.debug("Check vm active xml for --xml")
if not vmxml_active_tmp.title == updated_title:
print("vmxml active tmp title is %s" % vmxml_active_tmp.title)
result_check_pass = False
logging.error("--xml doesn't take effect in migration")
if xml_option and offline_migration:
logging.debug("Check vm inactive xml for --xml")
if not vmxml_active_tmp.title == updated_title:
result_check_pass = False
logging.error("--xml doesn't take effect in migration")
# Check dst vm xml after migration: --persistent-xml <updated_xml_file>
if persistent_xml_option:
logging.debug("Check vm inactive xml for --persistent-xml")
if not offline_migration and not vmxml_inactive_tmp.title == updated_persist_title:
print("vmxml inactive tmp title is %s" %
vmxml_inactive_tmp.title)
result_check_pass = False
logging.error(
"--persistent-xml doesn't take effect in live migration")
elif offline_migration and vmxml_inactive_tmp.title == updated_persist_title:
result_check_pass = False
logging.error(
"--persistent-xml should not take effect in offline "
"migration")
# Check dst vm xml after migration: graphic passwd
if with_graphic_passwd == "yes":
logging.debug("Check graphic passwd in vm xml after migration")
graphic_active = vmxml_active_tmp.devices.by_device_tag(
'graphics')[0]
graphic_inactive = vmxml_inactive_tmp.devices.by_device_tag(
'graphics')[0]
try:
logging.debug("Check graphic passwd in active vm xml")
if graphic_active.passwd != graphic_passwd:
result_check_pass = False
logging.error(
"Graphic passwd in active xml of dst vm should be %s",
graphic_passwd)
logging.debug("Check graphic passwd in inactive vm xml")
if graphic_inactive.passwd != graphic_passwd:
result_check_pass = False
logging.error(
"Graphic passwd in inactive xml of dst vm should be %s",
graphic_passwd)
except LibvirtXMLNotFoundError:
result_check_pass = False
logging.error("Graphic passwd lost in dst vm xml")
# Check dst vm uptime, network, etc after live migration
if live_migration:
# Check dst VM uptime after migration
# Note: migrated_vm_uptime should be greater than the vm_uptime got
# before migration
migrated_vm_uptime = vm.uptime(connect_uri=dest_uri)
logging.info(
"Check VM uptime in destination after "
"migration: %s", migrated_vm_uptime)
if not migrated_vm_uptime:
result_check_pass = False
logging.error("Failed to check vm uptime after migration")
elif vm_uptime > migrated_vm_uptime:
result_check_pass = False
logging.error(
"VM went for a reboot while migrating to destination")
# Check dst VM network connectivity after migration
logging.debug("Check VM network connectivity after migrating")
obj_migration.ping_vm(vm, params, uri=dest_uri)
# Restore vm.connect_uri as it is set to src_uri in ping_vm()
logging.debug(
"Restore vm.connect_uri as it is set to src_uri in ping_vm()")
vm.connect_uri = dest_uri
# Check dst vm status after migration: persistency
logging.debug("Check vm persistency on target after migration")
if persistent:
if not vm.is_persistent():
result_check_pass = False
logging.error("Dst vm should be persistent after migration "
"with --persistent")
time.sleep(10)
# Destroy vm and check vm state should be shutoff. BZ#1076354
vm.destroy()
if not libvirt.check_vm_state(
vm.name, "shut off", uri=vm.connect_uri):
result_check_pass = False
logging.error(
"Dst vm with name %s should exist and be shutoff", vm.name)
elif vm.is_persistent():
result_check_pass = False
logging.error("Dst vm should not be persistent after migration "
"without --persistent")
finally:
logging.debug("Start to clean up env")
# Clean up vm on dest and src host
for vm in vms:
cleanup_vm(vm, vm_name=dname, uri=dest_uri)
cleanup_vm(vm, vm_name=vm_name_backup, uri=src_uri)
# Recover source vm defination (just in case).
logging.info("Recover vm defination on source")
if vm_xml_backup:
vm_xml_backup.define()
# Clean up SSH, TCP, TLS test env
if objs_list and len(objs_list) > 0:
logging.debug("Clean up test env: SSH, TCP, TLS, etc")
for obj in objs_list:
obj.auto_recover = True
obj.__del__()
# Disable libvirtd remote connection transport port
obj_migration.migrate_pre_setup(dest_uri,
params,
cleanup=True,
ports=transport_port)
# Check test result.
if not result_check_pass:
test.fail("Migration succeed, but some check points didn't pass."
"Please check the error log for details")
def run(test, params, env):
"""
Test watchdog device:
1.Add watchdog device to the guest xml.
2.Start the guest.
3.Trigger the watchdog in the guest.
4.Confirm the guest status.
"""
def trigger_watchdog(model):
"""
Trigger watchdog
:param model: action when watchdog triggered
"""
watchdog_device = "device %s" % model
if action == "dump":
watchdog_action = "watchdog-action pause"
else:
watchdog_action = "watchdog-action %s" % action
vm_pid = vm.get_pid()
with open("/proc/%s/cmdline" % vm_pid) as vm_cmdline_file:
vm_cmdline = vm_cmdline_file.read()
vm_cmdline = vm_cmdline.replace('\x00', ' ')
if not all(option in vm_cmdline
for option in (watchdog_device, watchdog_action)):
test.fail("Can not find %s or %s in qemu cmd line" %
(watchdog_device, watchdog_action))
cmd = "gsettings set org.gnome.settings-daemon.plugins.power button-power shutdown"
session.cmd(cmd, ignore_all_errors=True)
try:
if model == "ib700":
try:
session.cmd("modprobe ib700wdt")
except aexpect.ShellCmdError:
session.close()
test.fail("Failed to load module ib700wdt")
session.cmd("dmesg | grep %s && lsmod | grep %s" % (model, model))
session.cmd("echo 1 > /dev/watchdog")
except aexpect.ShellCmdError:
session.close()
test.fail("Failed to trigger watchdog")
def confirm_guest_status():
"""
Confirm the guest status after watchdog triggered
"""
def _booting_completed():
session = vm.wait_for_login()
output = session.cmd_status_output("last reboot")
second_boot_time = output[1].strip().split("\n")[0].split()[-4]
logging.debug(second_boot_time)
session.close()
return second_boot_time > first_boot_time
def _inject_nmi():
session = vm.wait_for_login()
status, output = session.cmd_status_output("dmesg | grep -i nmi")
session.close()
if status == 0:
logging.debug(output)
return True
return False
def _inject_nmi_event():
virsh_session.send_ctrl("^C")
output = virsh_session.get_stripped_output()
if "inject-nmi" not in output:
return False
return True
def _check_dump_file(dump_path, domain_id):
dump_file = glob.glob('%s%s-*' % (dump_path, domain_id))
if len(dump_file):
logging.debug("Find the auto core dump file:\n%s",
dump_file[0])
os.remove(dump_file[0])
return True
return False
if action in ["poweroff", "shutdown"]:
if not utils_misc.wait_for(lambda: vm.state() == "shut off", 180,
10):
test.fail("Guest not shutdown after watchdog triggered")
elif action == "reset":
if not utils_misc.wait_for(_booting_completed, 600, 10):
test.fail("Guest not reboot after watchdog triggered")
elif action == "pause":
if utils_misc.wait_for(lambda: vm.state() == "paused", 180, 10):
cmd_output = virsh.domstate(vm_name, '--reason').stdout.strip()
logging.debug("Check guest status: %s\n", cmd_output)
if cmd_output != "paused (watchdog)":
test.fail(
"The domstate is not correct after dump by watchdog")
else:
test.fail("Guest not pause after watchdog triggered")
elif action == "none" and utils_misc.wait_for(
lambda: vm.state() == "shut off", 180, 10):
test.fail("Guest shutdown unexpectedly")
elif action == "inject-nmi":
if not utils_misc.wait_for(_inject_nmi, 180, 10):
test.fail(
"Guest not receive inject-nmi after watchdog triggered\n")
elif not utils_misc.wait_for(_inject_nmi_event, 180, 10):
test.fail("No inject-nmi watchdog event caught")
virsh_session.close()
elif action == "dump":
domain_id = vm.get_id()
dump_path = "/var/lib/libvirt/qemu/dump/"
if not utils_misc.wait_for(
lambda: _check_dump_file(dump_path, domain_id), 180, 10):
test.fail(
"No auto core dump file found after watchdog triggered")
name_length = params.get("name_length", "default")
vm_name = params.get("main_vm", "avocado-vt-vm1")
vm = env.get_vm(params["main_vm"])
model = params.get("model")
action = params.get("action")
# Backup xml file
vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
backup_xml = vmxml.copy()
# Rename the guest name to the length defined in the config file
if name_length != "default":
origin_name = vm_name
name_length = int(params.get("name_length", "1"))
vm_name = ''.join([
random.choice(string.ascii_letters + string.digits)
for _ in range(name_length)
])
vm_xml.VMXML.vm_rename(vm, vm_name)
# Generate the renamed xml file
vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
# Add watchdog device to domain
vmxml.remove_all_device_by_type('watchdog')
watchdog_dev = Watchdog()
watchdog_dev.model_type = model
watchdog_dev.action = action
chars = string.ascii_letters + string.digits + '-_'
alias_name = 'ua-' + ''.join(random.choice(chars) for _ in list(range(64)))
watchdog_dev.alias = {'name': alias_name}
vmxml.add_device(watchdog_dev)
vmxml.sync()
try:
vm.start()
session = vm.wait_for_login()
if action == "reset":
output = session.cmd_status_output("last reboot")
first_boot_time = output[1].strip().split("\n")[0].split()[-4]
logging.info("The first boot time is %s\n", first_boot_time)
if action == "inject-nmi":
virsh_session = virsh.VirshSession(virsh_exec=virsh.VIRSH_EXEC,
auto_close=True)
event_cmd = "event --event watchdog --all --loop"
virsh_session.sendline(event_cmd)
trigger_watchdog(model)
confirm_guest_status()
finally:
if vm.is_alive():
vm.destroy(gracefully=False)
backup_xml.sync()
