def copied_migration(vms, params):
"""
Migrate vms with storage copied.
"""
dest_uri = params.get("migrate_dest_uri")
remote_host = params.get("remote_ip")
copy_option = params.get("copy_storage_option", "")
username = params.get("remote_user")
password = params.get("remote_pwd")
timeout = int(params.get("thread_timeout", 1200))
options = "--live %s" % copy_option
# Get vm ip for remote checking
vms_ip = {}
for vm in vms:
if vm.is_dead():
vm.start()
vm.wait_for_login()
vms_ip[vm.name] = vm.get_address()
cp_mig = utlv.MigrationTest()
cp_mig.do_migration(vms, None, dest_uri, "orderly", options, timeout)
check_ip_failures = []
if cp_mig.RET_MIGRATION:
for vm in vms:
try:
utils_test.check_dest_vm_network(vm, vms_ip[vm.name],
remote_host, username,
password)
except error.TestFail, detail:
check_ip_failures.append(str(detail))
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
def copied_migration(test, vms, vms_ip, params):
"""
Migrate vms with storage copied.
:param test: The test object
:param vms: VM list
:param vms_ip: Dict for VM's IP
:param params: The parameters for the migration
:return: MigrationTest object
:raise: test.fail if anything goes wrong
"""
dest_uri = params.get("migrate_dest_uri")
remote_host = params.get("migrate_dest_host")
copy_option = params.get("copy_storage_option", "")
username = params.get("migrate_dest_user", "root")
password = params.get("migrate_dest_pwd")
timeout = int(params.get("thread_timeout", 1200))
status_error = params.get("status_error", "no")
options = "--live %s" % copy_option
cp_mig = migration.MigrationTest()
cp_mig.do_migration(vms,
None,
dest_uri,
"orderly",
options,
timeout,
ignore_status=True,
status_error=status_error)
check_ip_failures = []
if cp_mig.RET_MIGRATION:
for vm in vms:
try:
utils_test.check_dest_vm_network(vm, vms_ip[vm.name],
remote_host, username,
password)
except exceptions.TestFail as detail:
check_ip_failures.append(str(detail))
else:
for vm in vms:
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
check_output(test, str(cp_mig.ret), params)
test.fail("Migrate vms with storage copied failed.")
if len(check_ip_failures):
test.fail("Check IP failed:%s" % check_ip_failures)
return cp_mig
def copied_migration(test, vms, params):
"""
Migrate vms with storage copied.
"""
dest_uri = params.get("migrate_dest_uri")
remote_host = params.get("migrate_dest_host")
copy_option = params.get("copy_storage_option", "")
username = params.get("migrate_dest_user", "root")
password = params.get("migrate_dest_pwd")
timeout = int(params.get("thread_timeout", 1200))
options = "--live %s" % copy_option
# Get vm ip for remote checking
vms_ip = {}
for vm in vms:
if vm.is_dead():
vm.start()
vm.wait_for_login()
vms_ip[vm.name] = vm.get_address()
cp_mig = utlv.MigrationTest()
cp_mig.do_migration(vms,
None,
dest_uri,
"orderly",
options,
timeout,
ignore_status=True)
check_ip_failures = []
if cp_mig.RET_MIGRATION:
for vm in vms:
try:
utils_test.check_dest_vm_network(vm, vms_ip[vm.name],
remote_host, username,
password)
except exceptions.TestFail as detail:
check_ip_failures.append(str(detail))
else:
for vm in vms:
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
check_output(test, str(cp_mig.ret), params)
test.fail("Migrate vms with storage copied failed.")
if len(check_ip_failures):
test.fail("Check IP failed:%s" % check_ip_failures)
return cp_mig
def copied_migration(test, vms, params):
"""
Migrate vms with storage copied.
"""
dest_uri = params.get("migrate_dest_uri")
remote_host = params.get("migrate_dest_host")
copy_option = params.get("copy_storage_option", "")
username = params.get("migrate_dest_user", "root")
password = params.get("migrate_dest_pwd")
timeout = int(params.get("thread_timeout", 1200))
options = "--live %s" % copy_option
# Get vm ip for remote checking
vms_ip = {}
for vm in vms:
if vm.is_dead():
vm.start()
vm.wait_for_login()
vms_ip[vm.name] = vm.get_address()
cp_mig = utlv.MigrationTest()
cp_mig.do_migration(vms, None, dest_uri, "orderly", options, timeout,
ignore_status=True)
check_ip_failures = []
if cp_mig.RET_MIGRATION:
for vm in vms:
try:
utils_test.check_dest_vm_network(vm, vms_ip[vm.name],
remote_host, username,
password)
except exceptions.TestFail as detail:
check_ip_failures.append(str(detail))
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
else:
for vm in vms:
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
check_output(test, str(cp_mig.ret), params)
test.fail("Migrate vms with storage copied failed.")
if len(check_ip_failures):
test.fail("Check IP failed:%s" % check_ip_failures)
if migration_thread.isAlive():
logging.error("Migrate %s timeout.", migration_thread)
cp_mig.RET_LOCK.acquire()
cp_mig.RET_MIGRATION = False
cp_mig.RET_LOCK.release()
if len(blockjob_failures):
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
raise error.TestFail("Run qemu monitor command failed %s"
% blockjob_failures)
check_ip_failures = []
if cp_mig.RET_MIGRATION:
try:
utils_test.check_dest_vm_network(vm, vms_ip[vm.name],
remote_host, username,
password)
except error.TestFail, detail:
check_ip_failures.append(str(detail))
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
raise error.TestFail("Storage migration passed even after "
"cancellation.")
else:
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
logging.error("Expected Migration Error for %s", blockjob_type)
return
else:
raise error.TestFail("Command blockjob does not work well under "
"storage copied migration.")
migration_thread.join(timeout)
if migration_thread.isAlive():
logging.error("Migrate %s timeout.", migration_thread)
cp_mig.RET_LOCK.acquire()
cp_mig.RET_MIGRATION = False
cp_mig.RET_LOCK.release()
if len(blockjob_failures):
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
raise error.TestFail("Run qemu monitor command failed %s" %
blockjob_failures)
check_ip_failures = []
if cp_mig.RET_MIGRATION:
try:
utils_test.check_dest_vm_network(vm, vms_ip[vm.name], remote_host,
username, password)
except error.TestFail, detail:
check_ip_failures.append(str(detail))
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
raise error.TestFail("Storage migration passed even after "
"cancellation.")
else:
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
logging.error("Expected Migration Error for %s", blockjob_type)
return
else:
raise error.TestFail("Command blockjob does not work well under "
"storage copied migration.")
if target != "vda":
disks.append("/dev/%s" % target)
checked_count = int(params.get("checked_count", 0))
disks_before = disks[:(checked_count / 2)]
disks_after = disks[(checked_count / 2):checked_count]
logging.debug("Disks to be checked:\nBefore migration:%s\n"
"After migration:%s", disks_before, disks_after)
options = "--live --unsafe"
if not migrate_in_advance:
cleanup_ssh_config(vm)
mig.do_migration(vms, None, dsturi, "orderly", options, 120)
if mig.RET_MIGRATION:
utils_test.check_dest_vm_network(vm, vm_ip, remote_ip,
username, host_pwd)
runner = remote.RemoteRunner(host=remote_ip, username=username,
password=host_pwd)
# After migration, config autologin to vm
set_remote_vm_autologin(vm_ip, vm_pwd, runner)
check_disks_in_vm(vm, vm_ip, disks_after, runner)
if migrate_in_advance:
raise error.TestFail("Migration before attaching successfully, "
"but not expected.")
finally:
# Cleanup remote vm
if srcuri != dsturi:
mig.cleanup_dest_vm(vm, srcuri, dsturi)
# Cleanup created vm anyway
def run(test, params, env):
"""
Test migration under stress.
"""
vm_names = params.get("migration_vms").split()
if len(vm_names) < 2:
raise exceptions.TestSkipError("Provide enough vms for migration")
src_uri = libvirt_vm.complete_uri(params.get("migrate_source_host",
"EXAMPLE"))
if src_uri.count('///') or src_uri.count('EXAMPLE'):
raise exceptions.TestSkipError("The src_uri '%s' is invalid" % src_uri)
dest_uri = libvirt_vm.complete_uri(params.get("migrate_dest_host",
"EXAMPLE"))
if dest_uri.count('///') or dest_uri.count('EXAMPLE'):
raise exceptions.TestSkipError("The dest_uri '%s' is invalid" %
dest_uri)
# Params for NFS and SSH setup
params["server_ip"] = params.get("migrate_dest_host")
params["server_user"] = "******"
params["server_pwd"] = params.get("migrate_dest_pwd")
params["client_ip"] = params.get("migrate_source_host")
params["client_user"] = "******"
params["client_pwd"] = params.get("migrate_source_pwd")
params["nfs_client_ip"] = params.get("migrate_dest_host")
params["nfs_server_ip"] = params.get("migrate_source_host")
# Configure NFS client on remote host
nfs_client = nfs.NFSClient(params)
nfs_client.setup()
# Migrated vms' instance
vms = []
for vm_name in vm_names:
vms.append(libvirt_vm.VM(vm_name, params, test.bindir,
env.get("address_cache")))
load_vm_names = params.get("load_vms").split()
# vms for load
load_vms = []
for vm_name in load_vm_names:
load_vms.append(libvirt_vm.VM(vm_name, params, test.bindir,
env.get("address_cache")))
params['load_vms'] = load_vms
cpu = int(params.get("smp", 1))
memory = int(params.get("mem")) * 1024
stress_type = params.get("migration_stress_type")
vm_bytes = params.get("stress_vm_bytes")
stress_args = params.get("stress_args")
migration_type = params.get("migration_type")
start_migration_vms = "yes" == params.get("start_migration_vms", "yes")
thread_timeout = int(params.get("thread_timeout", 120))
remote_host = params.get("migrate_dest_host")
username = params.get("migrate_dest_user", "root")
password = params.get("migrate_dest_pwd")
prompt = params.get("shell_prompt", r"[\#\$]")
# Set vm_bytes for start_cmd
mem_total = utils_memory.memtotal()
vm_reserved = len(vms) * memory
if vm_bytes == "half":
vm_bytes = (mem_total - vm_reserved) / 2
elif vm_bytes == "shortage":
vm_bytes = mem_total - vm_reserved + 524288
if vm_bytes is not None:
params["stress_args"] = stress_args % vm_bytes
for vm in vms:
# Keep vm dead for edit
if vm.is_alive():
vm.destroy()
set_cpu_memory(vm.name, cpu, memory)
try:
vm_ipaddr = {}
if start_migration_vms:
for vm in vms:
vm.start()
vm.wait_for_login()
vm_ipaddr[vm.name] = vm.get_address()
# TODO: recover vm if start failed?
# Config ssh autologin for remote host
ssh_key.setup_ssh_key(remote_host, username, password, port=22)
do_stress_migration(vms, src_uri, dest_uri, stress_type,
migration_type, params, thread_timeout)
# Check network of vms on destination
if start_migration_vms and migration_type != "cross":
for vm in vms:
utils_test.check_dest_vm_network(vm, vm_ipaddr[vm.name],
remote_host,
username, password, prompt)
finally:
logging.debug("Cleanup vms...")
for vm_name in vm_names:
vm = libvirt_vm.VM(vm_name, params, test.bindir,
env.get("address_cache"))
utlv.MigrationTest().cleanup_dest_vm(vm, None, dest_uri)
if vm.is_alive():
vm.destroy(gracefully=False)
if nfs_client:
logging.info("Cleanup NFS client environment...")
nfs_client.cleanup()
env.clean_objects()
try:
obj_migration.do_migration(vms,
src_uri,
dest_uri,
"orderly",
options=migrate_options,
thread_timeout=postcopy_timeout,
ignore_status=False,
func=process.run,
func_params=cmd,
shell=True)
except Exception, info:
raise exceptions.TestFail(info)
if obj_migration.RET_MIGRATION:
utils_test.check_dest_vm_network(vm, vm.get_address(),
server_ip, server_user,
server_pwd)
ret_migrate = True
else:
ret_migrate = False
if not asynch_migration:
ret_migrate = do_migration(delay, vm, dest_uri, options, extra)
dest_state = params.get("virsh_migrate_dest_state", "running")
if ret_migrate and dest_state == "running":
server_session = remote.wait_for_login('ssh', server_ip, '22',
server_user, server_pwd,
r"[\#\$]\s*$")
logging.info("Check VM network connectivity after migrating")
s_ping, o_ping = utils_test.ping(vm_ip,
count=ping_count,
migrate_options = "%s %s" % (options, extra)
cmd = "sleep 5 && virsh %s %s" % (postcopy_cmd, vm_name)
logging.info("Starting migration in thread")
try:
obj_migration.do_migration(vms, src_uri, dest_uri, "orderly",
options=migrate_options,
thread_timeout=postcopy_timeout,
ignore_status=False,
func=process.run,
func_params=cmd,
shell=True)
except Exception, info:
raise exceptions.TestFail(info)
if obj_migration.RET_MIGRATION:
utils_test.check_dest_vm_network(vm, vm.get_address(),
server_ip, server_user,
server_pwd)
ret_migrate = True
else:
ret_migrate = False
if not asynch_migration:
ret_migrate = do_migration(delay, vm, dest_uri, options, extra)
dest_state = params.get("virsh_migrate_dest_state", "running")
if ret_migrate and dest_state == "running":
server_session = remote.wait_for_login('ssh', server_ip, '22',
server_user, server_pwd,
r"[\#\$]\s*$")
logging.info("Check VM network connectivity after migrating")
s_ping, o_ping = utils_test.ping(vm_ip, count=ping_count,
timeout=ping_timeout,
def copied_migration(test, vm, params, blockjob_type=None, block_target="vda"):
"""
Migrate vms with storage copied under some stress.
And during it, some qemu-monitor-command will be sent.
"""
dest_uri = params.get("migrate_dest_uri")
remote_host = params.get("migrate_dest_host")
copy_option = params.get("copy_storage_option", "")
username = params.get("remote_user")
password = params.get("migrate_dest_pwd")
timeout = int(params.get("thread_timeout", 1200))
options = "--live %s --unsafe" % copy_option
# Get vm ip for remote checking
if vm.is_dead():
vm.start()
vm.wait_for_login()
vms_ip = {}
vms_ip[vm.name] = vm.get_address()
logging.debug("VM %s IP: %s", vm.name, vms_ip[vm.name])
# Start to load stress
stress_type = params.get("migrate_stress_type")
if stress_type == "cpu":
params['stress_args'] = "--cpu 2 --quiet --timeout 60"
elif stress_type == "memory":
params['stress_args'] = "--vm 2 --vm-bytes 256M --vm-keep --timeout 60"
if stress_type is not None:
utils_test.load_stress("stress_in_vms", params=params, vms=[vm])
cp_mig = migration.MigrationTest()
migration_thread = threading.Thread(target=cp_mig.thread_func_migration,
args=(vm, dest_uri, options))
migration_thread.start()
# Wait for migration launched
time.sleep(5)
job_ret = virsh.domjobinfo(vm.name, debug=True)
if job_ret.exit_status:
test.error("Prepare migration for blockjob failed.")
# Execute some qemu monitor commands
pause_cmd = "block-job-pause %s" % block_target
resume_cmd = "block-job-resume %s" % block_target
cancel_cmd = "block-job-cancel %s" % block_target
complete_cmd = "block-job-complete %s" % block_target
blockjob_failures = []
try:
if blockjob_type == "cancel":
virsh.qemu_monitor_command(vm.name,
cancel_cmd,
debug=True,
ignore_status=False)
elif blockjob_type == "pause_resume":
virsh.qemu_monitor_command(vm.name,
pause_cmd,
debug=True,
ignore_status=False)
# TODO: Check whether it is paused.
virsh.qemu_monitor_command(vm.name,
resume_cmd,
debug=True,
ignore_status=False)
elif blockjob_type == "complete":
virsh.qemu_monitor_command(vm.name,
complete_cmd,
debug=True,
ignore_status=False)
except process.CmdError as detail:
blockjob_failures.append(str(detail))
# Job info FYI
virsh.domjobinfo(vm.name, debug=True)
if len(blockjob_failures):
timeout = 30
migration_thread.join(timeout)
if migration_thread.isAlive():
logging.error("Migrate %s timeout.", migration_thread)
cp_mig.RET_LOCK.acquire()
cp_mig.RET_MIGRATION = False
cp_mig.RET_LOCK.release()
if len(blockjob_failures):
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
test.fail("Run qemu monitor command failed %s" % blockjob_failures)
check_ip_failures = []
if cp_mig.RET_MIGRATION:
try:
utils_test.check_dest_vm_network(vm, vms_ip[vm.name], remote_host,
username, password)
except exceptions.TestFail as detail:
check_ip_failures.append(str(detail))
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
test.fail("Storage migration passed even after " "cancellation.")
else:
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
logging.error("Expected Migration Error for %s", blockjob_type)
return
else:
test.fail("Command blockjob does not work well under "
"storage copied migration.")
if len(check_ip_failures):
test.fail("Check IP failed:%s" % check_ip_failures)
disks.append("/dev/%s" % target)
checked_count = int(params.get("checked_count", 0))
disks_before = disks[:(checked_count / 2)]
disks_after = disks[(checked_count / 2):checked_count]
logging.debug(
"Disks to be checked:\nBefore migration:%s\n"
"After migration:%s", disks_before, disks_after)
options = "--live --unsafe"
if not migrate_in_advance:
cleanup_ssh_config(vm)
mig.do_migration(vms, None, dsturi, "orderly", options, 120)
if mig.RET_MIGRATION:
utils_test.check_dest_vm_network(vm, vm_ip, remote_ip, username,
host_pwd)
runner = remote.RemoteRunner(host=remote_ip,
username=username,
password=host_pwd)
# After migration, config autologin to vm
set_remote_vm_autologin(vm_ip, vm_pwd, runner)
check_disks_in_vm(vm, vm_ip, disks_after, runner)
if migrate_in_advance:
raise error.TestFail(
"Migration before attaching successfully, "
"but not expected.")
finally:
# Cleanup remote vm
if srcuri != dsturi:
def run(test, params, env):
"""
Test virsh migrate when disks are virtio-scsi.
"""
def check_vm_state(vm, state):
"""
Return True if vm is in the correct state.
"""
try:
actual_state = vm.state()
except process.CmdError:
return False
if actual_state == state:
return True
else:
return False
def check_disks_in_vm(vm, vm_ip, disks_list=[], runner=None):
"""
Check disks attached to vm.
"""
fail_list = []
while len(disks_list):
disk = disks_list.pop()
if runner:
check_cmd = ("ssh %s \"dd if=/dev/urandom of=%s bs=1 "
"count=1024\"" % (vm_ip, disk))
try:
logging.debug(runner.run(check_cmd))
continue
except process.CmdError as detail:
logging.debug("Remote checking failed:%s", detail)
fail_list.append(disk)
else:
check_cmd = "dd if=/dev/urandom of=%s bs=1 count=1024"
session = vm.wait_for_login()
cs = session.cmd_status(check_cmd)
if cs:
fail_list.append(disk)
session.close()
if len(fail_list):
test.fail("Checking attached devices failed:%s" % fail_list)
def get_disk_id(device):
"""
Show disk by id.
"""
output = process.run("ls /dev/disk/by-id/", shell=True).stdout_text
for line in output.splitlines():
disk_ids = line.split()
for disk_id in disk_ids:
disk = os.path.basename(
process.run("readlink %s" % disk_id,
shell=True).stdout_text)
if disk == os.path.basename(device):
return disk_id
return None
def cleanup_ssh_config(vm):
session = vm.wait_for_login()
session.cmd("rm -f ~/.ssh/authorized_keys")
session.cmd("rm -f ~/.ssh/id_rsa*")
session.close()
vm = env.get_vm(params.get("migrate_main_vm"))
source_type = params.get("disk_source_type", "file")
device_type = params.get("disk_device_type", "disk")
disk_format = params.get("disk_format_type", "raw")
if source_type == "file":
params['added_disk_type'] = "file"
else:
params['added_disk_type'] = "block"
block_device = params.get("disk_block_device", "/dev/EXAMPLE")
if block_device.count("EXAMPLE"):
# Prepare host parameters
local_host = params.get("migrate_source_host", "LOCAL.EXAMPLE")
remote_host = params.get("migrate_dest_host", "REMOTE.EXAMPLE")
remote_user = params.get("migrate_dest_user", "root")
remote_passwd = params.get("migrate_dest_pwd")
if remote_host.count("EXAMPLE") or local_host.count("EXAMPLE"):
test.cancel("Config remote or local host first.")
rdm_params = {
'remote_ip': remote_host,
'remote_user': remote_user,
'remote_pwd': remote_passwd
}
rdm = utils_test.RemoteDiskManager(rdm_params)
# Try to build an iscsi device
# For local, target is a device name
target = utlv.setup_or_cleanup_iscsi(
is_setup=True, is_login=True, emulated_image="emulated-iscsi")
logging.debug("Created target: %s", target)
try:
# Attach this iscsi device both local and remote
remote_device = rdm.iscsi_login_setup(local_host, target)
except Exception as detail:
utlv.setup_or_cleanup_iscsi(is_setup=False)
test.error("Attach iscsi device on remote failed:%s" % detail)
# Use id to get same path on local and remote
block_device = get_disk_id(target)
if block_device is None:
rdm.iscsi_login_setup(local_host, target, is_login=False)
utlv.setup_or_cleanup_iscsi(is_setup=False)
test.error("Set iscsi device couldn't find id?")
srcuri = params.get("virsh_migrate_srcuri")
dsturi = params.get("virsh_migrate_dsturi")
remote_ip = params.get("remote_ip")
username = params.get("remote_user", "root")
host_pwd = params.get("remote_pwd")
# Connection to remote, init here for cleanup
runner = None
# Identify easy config. mistakes early
warning_text = ("Migration VM %s URI %s appears problematic "
"this may lead to migration problems. "
"Consider specifying vm.connect_uri using "
"fully-qualified network-based style.")
if srcuri.count('///') or srcuri.count('EXAMPLE'):
test.cancel(warning_text % ('source', srcuri))
if dsturi.count('///') or dsturi.count('EXAMPLE'):
test.cancel(warning_text % ('destination', dsturi))
# Config auto-login to remote host for migration
ssh_key.setup_ssh_key(remote_ip, username, host_pwd)
sys_image = vm.get_first_disk_devices()
sys_image_source = sys_image["source"]
sys_image_info = utils_misc.get_image_info(sys_image_source)
logging.debug("System image information:\n%s", sys_image_info)
sys_image_fmt = sys_image_info["format"]
created_img_path = os.path.join(os.path.dirname(sys_image_source),
"vsmimages")
migrate_in_advance = "yes" == params.get("migrate_in_advance", "no")
status_error = "yes" == params.get("status_error", "no")
if source_type == "file" and device_type == "lun":
status_error = True
try:
# For safety and easily reasons, we'd better define a new vm
new_vm_name = "%s_vsmtest" % vm.name
mig = utlv.MigrationTest()
if vm.is_alive():
vm.destroy()
utlv.define_new_vm(vm.name, new_vm_name)
vm = libvirt_vm.VM(new_vm_name, vm.params, vm.root_dir,
vm.address_cache)
# Change the disk of the vm to shared disk
# Detach exist devices
devices = vm.get_blk_devices()
for device in devices:
s_detach = virsh.detach_disk(vm.name,
device,
"--config",
debug=True)
if not s_detach:
test.error("Detach %s failed before test.", device)
# Attach system image as vda
# Then added scsi disks will be sda,sdb...
attach_args = "--subdriver %s --config" % sys_image_fmt
virsh.attach_disk(vm.name,
sys_image_source,
"vda",
attach_args,
debug=True)
vms = [vm]
def start_check_vm(vm):
try:
vm.start()
except virt_vm.VMStartError as detail:
if status_error:
logging.debug("Expected failure:%s", detail)
return None, None
else:
raise
vm.wait_for_login()
# Confirm VM can be accessed through network.
# And this ip will be used on remote after migration
vm_ip = vm.get_address()
vm_pwd = params.get("password")
s_ping, o_ping = utils_test.ping(vm_ip, count=2, timeout=60)
logging.info(o_ping)
if s_ping != 0:
test.fail("%s did not respond after several "
"seconds with attaching new devices." % vm.name)
return vm_ip, vm_pwd
options = "--live --unsafe"
# Do migration before attaching new devices
if migrate_in_advance:
vm_ip, vm_pwd = start_check_vm(vm)
cleanup_ssh_config(vm)
mig_thread = threading.Thread(target=mig.thread_func_migration,
args=(vm, dsturi, options))
mig_thread.start()
# Make sure migration is running
time.sleep(2)
# Attach other disks
params['added_disk_target'] = "scsi"
params['target_bus'] = "scsi"
params['device_type'] = device_type
params['type_name'] = source_type
params['added_disk_format'] = disk_format
if migrate_in_advance:
params["attach_disk_config"] = "no"
attach_disk_config = False
else:
params["attach_disk_config"] = "yes"
attach_disk_config = True
try:
if source_type == "file":
utlv.attach_disks(vm, "%s/image" % created_img_path, None,
params)
else:
ret = utlv.attach_additional_device(vm.name,
"sda",
block_device,
params,
config=attach_disk_config)
if ret.exit_status:
test.fail(ret)
except (exceptions.TestFail, process.CmdError) as detail:
if status_error:
logging.debug("Expected failure:%s", detail)
return
else:
raise
if migrate_in_advance:
mig_thread.join(60)
if mig_thread.isAlive():
mig.RET_LOCK.acquire()
mig.MIGRATION = False
mig.RET_LOCK.release()
else:
vm_ip, vm_pwd = start_check_vm(vm)
# Have got expected failures when starting vm, end the test
if vm_ip is None and status_error:
return
# Start checking before migration and go on checking after migration
disks = []
for target in list(vm.get_disk_devices().keys()):
if target != "vda":
disks.append("/dev/%s" % target)
checked_count = int(params.get("checked_count", 0))
disks_before = disks[:(checked_count // 2)]
disks_after = disks[(checked_count // 2):checked_count]
logging.debug(
"Disks to be checked:\nBefore migration:%s\n"
"After migration:%s", disks_before, disks_after)
options = "--live --unsafe"
if not migrate_in_advance:
cleanup_ssh_config(vm)
mig.do_migration(vms, None, dsturi, "orderly", options, 120)
if mig.RET_MIGRATION:
utils_test.check_dest_vm_network(vm, vm_ip, remote_ip, username,
host_pwd)
runner = remote.RemoteRunner(host=remote_ip,
username=username,
password=host_pwd)
# After migration, config autologin to vm
ssh_key.setup_remote_ssh_key(vm_ip, "root", vm_pwd)
check_disks_in_vm(vm, vm_ip, disks_after, runner)
if migrate_in_advance:
test.fail("Migration before attaching successfully, "
"but not expected.")
finally:
# Cleanup remote vm
if srcuri != dsturi:
mig.cleanup_dest_vm(vm, srcuri, dsturi)
# Cleanup created vm anyway
if vm.is_alive():
vm.destroy(gracefully=False)
virsh.undefine(new_vm_name)
# Cleanup iscsi device for block if it is necessary
if source_type == "block":
if params.get("disk_block_device",
"/dev/EXAMPLE").count("EXAMPLE"):
rdm.iscsi_login_setup(local_host, target, is_login=False)
utlv.setup_or_cleanup_iscsi(is_setup=False,
emulated_image="emulated-iscsi")
if runner:
runner.session.close()
process.run("rm -f %s/*vsmtest" % created_img_path, shell=True)
def run(test, params, env):
"""
Test migration under stress.
"""
vm_names = params.get("migration_vms").split()
if len(vm_names) < 2:
raise exceptions.TestSkipError("Provide enough vms for migration")
src_uri = libvirt_vm.complete_uri(
params.get("migrate_source_host", "EXAMPLE"))
if src_uri.count('///') or src_uri.count('EXAMPLE'):
raise exceptions.TestSkipError("The src_uri '%s' is invalid" % src_uri)
dest_uri = libvirt_vm.complete_uri(
params.get("migrate_dest_host", "EXAMPLE"))
if dest_uri.count('///') or dest_uri.count('EXAMPLE'):
raise exceptions.TestSkipError("The dest_uri '%s' is invalid" %
dest_uri)
# Params for NFS and SSH setup
params["server_ip"] = params.get("migrate_dest_host")
params["server_user"] = "******"
params["server_pwd"] = params.get("migrate_dest_pwd")
params["client_ip"] = params.get("migrate_source_host")
params["client_user"] = "******"
params["client_pwd"] = params.get("migrate_source_pwd")
params["nfs_client_ip"] = params.get("migrate_dest_host")
params["nfs_server_ip"] = params.get("migrate_source_host")
# Configure NFS client on remote host
nfs_client = nfs.NFSClient(params)
nfs_client.setup()
# Migrated vms' instance
vms = []
for vm_name in vm_names:
vms.append(
libvirt_vm.VM(vm_name, params, test.bindir,
env.get("address_cache")))
load_vm_names = params.get("load_vms").split()
# vms for load
load_vms = []
for vm_name in load_vm_names:
load_vms.append(
libvirt_vm.VM(vm_name, params, test.bindir,
env.get("address_cache")))
params['load_vms'] = load_vms
cpu = int(params.get("smp", 1))
memory = int(params.get("mem")) * 1024
stress_type = params.get("migration_stress_type")
vm_bytes = params.get("stress_vm_bytes")
stress_args = params.get("stress_args")
migration_type = params.get("migration_type")
start_migration_vms = "yes" == params.get("start_migration_vms", "yes")
thread_timeout = int(params.get("thread_timeout", 120))
remote_host = params.get("migrate_dest_host")
username = params.get("migrate_dest_user", "root")
password = params.get("migrate_dest_pwd")
prompt = params.get("shell_prompt", r"[\#\$]")
# Set vm_bytes for start_cmd
mem_total = utils_memory.memtotal()
vm_reserved = len(vms) * memory
if vm_bytes == "half":
vm_bytes = (mem_total - vm_reserved) / 2
elif vm_bytes == "shortage":
vm_bytes = mem_total - vm_reserved + 524288
if vm_bytes is not None:
params["stress_args"] = stress_args % vm_bytes
for vm in vms:
# Keep vm dead for edit
if vm.is_alive():
vm.destroy()
set_cpu_memory(vm.name, cpu, memory)
try:
vm_ipaddr = {}
if start_migration_vms:
for vm in vms:
vm.start()
vm.wait_for_login()
vm_ipaddr[vm.name] = vm.get_address()
# TODO: recover vm if start failed?
# Config ssh autologin for remote host
ssh_key.setup_ssh_key(remote_host, username, password, port=22)
do_stress_migration(vms, src_uri, dest_uri, stress_type,
migration_type, params, thread_timeout)
# Check network of vms on destination
if start_migration_vms and migration_type != "cross":
for vm in vms:
utils_test.check_dest_vm_network(vm, vm_ipaddr[vm.name],
remote_host, username,
password, prompt)
finally:
logging.debug("Cleanup vms...")
for vm_name in vm_names:
vm = libvirt_vm.VM(vm_name, params, test.bindir,
env.get("address_cache"))
utlv.MigrationTest().cleanup_dest_vm(vm, None, dest_uri)
if vm.is_alive():
vm.destroy(gracefully=False)
if nfs_client:
logging.info("Cleanup NFS client environment...")
nfs_client.cleanup()
env.clean_objects()
def run(test, params, env):
"""
Test virsh migrate when disks are virtio-scsi.
"""
def check_vm_state(vm, state):
"""
Return True if vm is in the correct state.
"""
try:
actual_state = vm.state()
except process.CmdError:
return False
if actual_state == state:
return True
else:
return False
def check_disks_in_vm(vm, vm_ip, disks_list=[], runner=None):
"""
Check disks attached to vm.
"""
fail_list = []
while len(disks_list):
disk = disks_list.pop()
if runner:
check_cmd = ("ssh %s \"dd if=/dev/urandom of=%s bs=1 "
"count=1024\"" % (vm_ip, disk))
try:
logging.debug(runner.run(check_cmd))
continue
except process.CmdError as detail:
logging.debug("Remote checking failed:%s", detail)
fail_list.append(disk)
else:
check_cmd = "dd if=/dev/urandom of=%s bs=1 count=1024"
session = vm.wait_for_login()
cs = session.cmd_status(check_cmd)
if cs:
fail_list.append(disk)
session.close()
if len(fail_list):
test.fail("Checking attached devices failed:%s"
% fail_list)
def get_disk_id(device):
"""
Show disk by id.
"""
output = process.run("ls /dev/disk/by-id/", shell=True).stdout_text
for line in output.splitlines():
disk_ids = line.split()
for disk_id in disk_ids:
disk = os.path.basename(
process.run("readlink %s" % disk_id, shell=True).stdout_text)
if disk == os.path.basename(device):
return disk_id
return None
def cleanup_ssh_config(vm):
session = vm.wait_for_login()
session.cmd("rm -f ~/.ssh/authorized_keys")
session.cmd("rm -f ~/.ssh/id_rsa*")
session.close()
vm = env.get_vm(params.get("migrate_main_vm"))
source_type = params.get("disk_source_type", "file")
device_type = params.get("disk_device_type", "disk")
disk_format = params.get("disk_format_type", "raw")
if source_type == "file":
params['added_disk_type'] = "file"
else:
params['added_disk_type'] = "block"
block_device = params.get("disk_block_device", "/dev/EXAMPLE")
if block_device.count("EXAMPLE"):
# Prepare host parameters
local_host = params.get("migrate_source_host", "LOCAL.EXAMPLE")
remote_host = params.get("migrate_dest_host", "REMOTE.EXAMPLE")
remote_user = params.get("migrate_dest_user", "root")
remote_passwd = params.get("migrate_dest_pwd")
if remote_host.count("EXAMPLE") or local_host.count("EXAMPLE"):
test.cancel("Config remote or local host first.")
rdm_params = {'remote_ip': remote_host,
'remote_user': remote_user,
'remote_pwd': remote_passwd}
rdm = utils_test.RemoteDiskManager(rdm_params)
# Try to build an iscsi device
# For local, target is a device name
target = utlv.setup_or_cleanup_iscsi(is_setup=True, is_login=True,
emulated_image="emulated-iscsi")
logging.debug("Created target: %s", target)
try:
# Attach this iscsi device both local and remote
remote_device = rdm.iscsi_login_setup(local_host, target)
except Exception as detail:
utlv.setup_or_cleanup_iscsi(is_setup=False)
test.error("Attach iscsi device on remote failed:%s"
% detail)
# Use id to get same path on local and remote
block_device = get_disk_id(target)
if block_device is None:
rdm.iscsi_login_setup(local_host, target, is_login=False)
utlv.setup_or_cleanup_iscsi(is_setup=False)
test.error("Set iscsi device couldn't find id?")
srcuri = params.get("virsh_migrate_srcuri")
dsturi = params.get("virsh_migrate_dsturi")
remote_ip = params.get("remote_ip")
username = params.get("remote_user", "root")
host_pwd = params.get("remote_pwd")
# Connection to remote, init here for cleanup
runner = None
# Identify easy config. mistakes early
warning_text = ("Migration VM %s URI %s appears problematic "
"this may lead to migration problems. "
"Consider specifying vm.connect_uri using "
"fully-qualified network-based style.")
if srcuri.count('///') or srcuri.count('EXAMPLE'):
test.cancel(warning_text % ('source', srcuri))
if dsturi.count('///') or dsturi.count('EXAMPLE'):
test.cancel(warning_text % ('destination', dsturi))
# Config auto-login to remote host for migration
ssh_key.setup_ssh_key(remote_ip, username, host_pwd)
sys_image = vm.get_first_disk_devices()
sys_image_source = sys_image["source"]
sys_image_info = utils_misc.get_image_info(sys_image_source)
logging.debug("System image information:\n%s", sys_image_info)
sys_image_fmt = sys_image_info["format"]
created_img_path = os.path.join(os.path.dirname(sys_image_source),
"vsmimages")
migrate_in_advance = "yes" == params.get("migrate_in_advance", "no")
status_error = "yes" == params.get("status_error", "no")
if source_type == "file" and device_type == "lun":
status_error = True
try:
# For safety and easily reasons, we'd better define a new vm
new_vm_name = "%s_vsmtest" % vm.name
mig = utlv.MigrationTest()
if vm.is_alive():
vm.destroy()
utlv.define_new_vm(vm.name, new_vm_name)
vm = libvirt_vm.VM(new_vm_name, vm.params, vm.root_dir,
vm.address_cache)
# Change the disk of the vm to shared disk
# Detach exist devices
devices = vm.get_blk_devices()
for device in devices:
s_detach = virsh.detach_disk(vm.name, device, "--config",
debug=True)
if not s_detach:
test.error("Detach %s failed before test.", device)
# Attach system image as vda
# Then added scsi disks will be sda,sdb...
attach_args = "--subdriver %s --config" % sys_image_fmt
virsh.attach_disk(vm.name, sys_image_source, "vda",
attach_args, debug=True)
vms = [vm]
def start_check_vm(vm):
try:
vm.start()
except virt_vm.VMStartError as detail:
if status_error:
logging.debug("Expected failure:%s", detail)
return None, None
else:
raise
vm.wait_for_login()
# Confirm VM can be accessed through network.
# And this ip will be used on remote after migration
vm_ip = vm.get_address()
vm_pwd = params.get("password")
s_ping, o_ping = utils_test.ping(vm_ip, count=2, timeout=60)
logging.info(o_ping)
if s_ping != 0:
test.fail("%s did not respond after several "
"seconds with attaching new devices."
% vm.name)
return vm_ip, vm_pwd
options = "--live --unsafe"
# Do migration before attaching new devices
if migrate_in_advance:
vm_ip, vm_pwd = start_check_vm(vm)
cleanup_ssh_config(vm)
mig_thread = threading.Thread(target=mig.thread_func_migration,
args=(vm, dsturi, options))
mig_thread.start()
# Make sure migration is running
time.sleep(2)
# Attach other disks
params['added_disk_target'] = "scsi"
params['target_bus'] = "scsi"
params['device_type'] = device_type
params['type_name'] = source_type
params['added_disk_format'] = disk_format
if migrate_in_advance:
params["attach_disk_config"] = "no"
attach_disk_config = False
else:
params["attach_disk_config"] = "yes"
attach_disk_config = True
try:
if source_type == "file":
utlv.attach_disks(vm, "%s/image" % created_img_path,
None, params)
else:
ret = utlv.attach_additional_device(vm.name, "sda", block_device,
params, config=attach_disk_config)
if ret.exit_status:
test.fail(ret)
except (exceptions.TestFail, process.CmdError) as detail:
if status_error:
logging.debug("Expected failure:%s", detail)
return
else:
raise
if migrate_in_advance:
mig_thread.join(60)
if mig_thread.isAlive():
mig.RET_LOCK.acquire()
mig.MIGRATION = False
mig.RET_LOCK.release()
else:
vm_ip, vm_pwd = start_check_vm(vm)
# Have got expected failures when starting vm, end the test
if vm_ip is None and status_error:
return
# Start checking before migration and go on checking after migration
disks = []
for target in list(vm.get_disk_devices().keys()):
if target != "vda":
disks.append("/dev/%s" % target)
checked_count = int(params.get("checked_count", 0))
disks_before = disks[:(checked_count // 2)]
disks_after = disks[(checked_count // 2):checked_count]
logging.debug("Disks to be checked:\nBefore migration:%s\n"
"After migration:%s", disks_before, disks_after)
options = "--live --unsafe"
if not migrate_in_advance:
cleanup_ssh_config(vm)
mig.do_migration(vms, None, dsturi, "orderly", options, 120)
if mig.RET_MIGRATION:
utils_test.check_dest_vm_network(vm, vm_ip, remote_ip,
username, host_pwd)
runner = remote.RemoteRunner(host=remote_ip, username=username,
password=host_pwd)
# After migration, config autologin to vm
ssh_key.setup_remote_ssh_key(vm_ip, "root", vm_pwd)
check_disks_in_vm(vm, vm_ip, disks_after, runner)
if migrate_in_advance:
test.fail("Migration before attaching successfully, "
"but not expected.")
finally:
# Cleanup remote vm
if srcuri != dsturi:
mig.cleanup_dest_vm(vm, srcuri, dsturi)
# Cleanup created vm anyway
if vm.is_alive():
vm.destroy(gracefully=False)
virsh.undefine(new_vm_name)
# Cleanup iscsi device for block if it is necessary
if source_type == "block":
if params.get("disk_block_device",
"/dev/EXAMPLE").count("EXAMPLE"):
rdm.iscsi_login_setup(local_host, target, is_login=False)
utlv.setup_or_cleanup_iscsi(is_setup=False,
emulated_image="emulated-iscsi")
if runner:
runner.session.close()
process.run("rm -f %s/*vsmtest" % created_img_path, shell=True)
def copied_migration(test, vm, params, blockjob_type=None, block_target="vda"):
"""
Migrate vms with storage copied under some stress.
And during it, some qemu-monitor-command will be sent.
"""
dest_uri = params.get("migrate_dest_uri")
remote_host = params.get("migrate_dest_host")
copy_option = params.get("copy_storage_option", "")
username = params.get("remote_user")
password = params.get("migrate_dest_pwd")
timeout = int(params.get("thread_timeout", 1200))
options = "--live %s --unsafe" % copy_option
# Get vm ip for remote checking
if vm.is_dead():
vm.start()
vm.wait_for_login()
vms_ip = {}
vms_ip[vm.name] = vm.get_address()
logging.debug("VM %s IP: %s", vm.name, vms_ip[vm.name])
# Start to load stress
stress_type = params.get("migrate_stress_type")
if stress_type == "cpu":
params['stress_args'] = "--cpu 2 --quiet --timeout 60"
elif stress_type == "memory":
params['stress_args'] = "--vm 2 --vm-bytes 256M --vm-keep --timeout 60"
if stress_type is not None:
utils_test.load_stress("stress_in_vms", params=params, vms=[vm])
cp_mig = utlv.MigrationTest()
migration_thread = threading.Thread(target=cp_mig.thread_func_migration,
args=(vm, dest_uri, options))
migration_thread.start()
# Wait for migration launched
time.sleep(5)
job_ret = virsh.domjobinfo(vm.name, debug=True)
if job_ret.exit_status:
test.error("Prepare migration for blockjob failed.")
# Execute some qemu monitor commands
pause_cmd = "block-job-pause %s" % block_target
resume_cmd = "block-job-resume %s" % block_target
cancel_cmd = "block-job-cancel %s" % block_target
complete_cmd = "block-job-complete %s" % block_target
blockjob_failures = []
try:
if blockjob_type == "cancel":
virsh.qemu_monitor_command(vm.name, cancel_cmd, debug=True,
ignore_status=False)
elif blockjob_type == "pause_resume":
virsh.qemu_monitor_command(vm.name, pause_cmd, debug=True,
ignore_status=False)
# TODO: Check whether it is paused.
virsh.qemu_monitor_command(vm.name, resume_cmd, debug=True,
ignore_status=False)
elif blockjob_type == "complete":
virsh.qemu_monitor_command(vm.name, complete_cmd, debug=True,
ignore_status=False)
except process.CmdError as detail:
blockjob_failures.append(str(detail))
# Job info FYI
virsh.domjobinfo(vm.name, debug=True)
if len(blockjob_failures):
timeout = 30
migration_thread.join(timeout)
if migration_thread.isAlive():
logging.error("Migrate %s timeout.", migration_thread)
cp_mig.RET_LOCK.acquire()
cp_mig.RET_MIGRATION = False
cp_mig.RET_LOCK.release()
if len(blockjob_failures):
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
test.fail("Run qemu monitor command failed %s"
% blockjob_failures)
check_ip_failures = []
if cp_mig.RET_MIGRATION:
try:
utils_test.check_dest_vm_network(vm, vms_ip[vm.name],
remote_host, username,
password)
except exceptions.TestFail as detail:
check_ip_failures.append(str(detail))
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
test.fail("Storage migration passed even after "
"cancellation.")
else:
cp_mig.cleanup_dest_vm(vm, None, dest_uri)
if blockjob_type in ["cancel", "complete"]:
logging.error("Expected Migration Error for %s", blockjob_type)
return
else:
test.fail("Command blockjob does not work well under "
"storage copied migration.")
if len(check_ip_failures):
test.fail("Check IP failed:%s" % check_ip_failures)
