def copy_file_from_nfs(src, dst, mount_point, image_name):
logging.info("Test failed before the install process start."
" So just copy a good image from nfs for following tests.")
utils_misc.mount(src, mount_point, "nfs", perm="ro")
image_src = utils_misc.get_path(mount_point, image_name)
shutil.copy(image_src, dst)
utils_misc.umount(src, mount_point, "nfs")
def copy_file_from_nfs(src, dst, mount_point, image_name):
logging.info("Test failed before the install process start."
" So just copy a good image from nfs for following tests.")
utils_misc.mount(src, mount_point, "nfs", perm="ro")
image_src = utils_misc.get_path(mount_point, image_name)
shutil.copy(image_src, dst)
utils_misc.umount(src, mount_point, "nfs")
def test_has_run_againt_vddk7_0():
"""
check if nbdkit --run + vddk + esx7.0 works.
"""
from virttest.utils_pyvmomi import VSphereConnection, vim
vm_name = params_get(params, "main_vm")
if not vm_name:
test.error('No VM specified')
# vsphere server's host name or IP address
vsphere_host = params_get(params, "vsphere_host")
vsphere_user = params_get(params, "vsphere_user", 'root')
# vsphere password
vsphere_pwd = params_get(params, "vsphere_pwd")
vsphere_passwd_file = params_get(params, "vpx_passwd_file",
'/tmp/v2v_vpx_passwd')
with open(vsphere_passwd_file, 'w') as fd:
fd.write(vsphere_pwd)
# get vm and file's value
connect_args = {
'host': vsphere_host,
'user': vsphere_user,
'pwd': vsphere_pwd
}
with VSphereConnection(**connect_args) as conn:
conn.target_vm = vm_name
nbdkit_vm_name = 'moref=' + \
str(conn.target_vm).strip('\'').split(':')[1]
nbdkit_file = conn.get_hardware_devices(
dev_type=vim.vm.device.VirtualDisk)[0].backing.fileName
# vddk_libdir
vddk_libdir_src = params_get(params, "vddk_libdir_src")
with tempfile.TemporaryDirectory(prefix='vddklib_') as vddk_libdir:
utils_misc.mount(vddk_libdir_src, vddk_libdir, 'nfs')
vddk_thumbprint = '11'
nbdkit_cmd = """
nbdkit -rfv -U - --exportname / \
--filter=cacheextents --filter=retry vddk server=%s user=%s password=+%s vm=%s \
file='%s' libdir=%s --run 'qemu-img info $nbd' thumbprint=%s
""" % (vsphere_host, vsphere_user, vsphere_passwd_file, nbdkit_vm_name,
nbdkit_file, vddk_libdir, vddk_thumbprint)
# get thumbprint by a trick
cmd_result = process.run(nbdkit_cmd,
shell=True,
ignore_status=True)
output = cmd_result.stdout_text + cmd_result.stderr_text
vddk_thumbprint = re.search(r'PeerThumbprint:\s+(.*)',
output).group(1)
# replace thumbprint with correct value
nbdkit_cmd = nbdkit_cmd.strip()[:-2] + vddk_thumbprint
logging.info('nbdkit command:\n%s', nbdkit_cmd)
# Run the finnal nbdkit command
output = process.run(nbdkit_cmd, shell=True).stdout_text
utils_misc.umount(vddk_libdir_src, vddk_libdir, 'nfs')
if not re.search(r'virtual size', output):
test.fail('failed to test has_run_againt_vddk7_0')
def cleanup(self):
"""
Cleanup NFS mount records
"""
for src, dst, fstype in self.mount_records.values():
utils_misc.umount(src, dst, fstype)
self.cleanup_authorized_keys_in_vmx()
def cleanup(self):
"""
Clean up the host env.
Umount NFS from the mount point. If there has some change for exported
file system in host when setup, also clean up that.
"""
utils_misc.umount(self.mount_src, self.mount_dir, "nfs")
if self.nfs_setup and self.unexportfs_in_clean:
self.exportfs.reset_export()
def cleanup(self):
"""
Clean up the host env.
Umount NFS from the mount point. If there has some change for exported
file system in host when setup, also clean up that.
"""
utils_misc.umount(self.mount_src, self.mount_dir, "nfs")
if self.nfs_setup and self.unexportfs_in_clean:
self.exportfs.reset_export()
def cleanup(self):
self.close_virsh_session()
try:
self.checker.cleanup()
except Exception as e:
logging.debug("Exception during cleanup:\n%s", e)
pass
if len(self.mount_records) != 0:
for src, dst, fstype in self.mount_records.values():
utils_misc.umount(src, dst, fstype)
def cleanup(self):
self.close_virsh_session()
try:
if self.checker.session:
self.checker.session.close()
self.checker.cleanup()
except Exception:
pass
if len(self.mount_records) != 0:
for src, dst, fstype in self.mount_records.values():
utils_misc.umount(src, dst, fstype)
def remove(self, name):
info = name
# ugly workaround for nfs which unable to umount
#os.system('systemctl restart nfs')
if not umount(info['src'], info['mount_point'], info['fstype'],
verbose=False):
raise Exception("Failed to unmount %s" % info['mount_point'])
def file_exists(params, filename_path):
sg_uri = create_gluster_uri(params, stripped=True)
g_uri = create_gluster_uri(params, stripped=False)
# Using directly /tmp dir because directory should be really temporary and
# should be deleted immediately when no longer needed and
# created directory don't file tmp dir by any data.
tmpdir = "gmount-%s" % (utils_misc.generate_random_string(6))
tmpdir_path = os.path.join(data_dir.get_tmp_dir(), tmpdir)
while os.path.exists(tmpdir_path):
tmpdir = "gmount-%s" % (utils_misc.generate_random_string(6))
tmpdir_path = os.path.join(data_dir.get_tmp_dir(), tmpdir)
ret = False
try:
try:
os.mkdir(tmpdir_path)
glusterfs_mount(sg_uri, tmpdir_path)
mount_filename_path = os.path.join(tmpdir_path,
filename_path[len(g_uri):])
if os.path.exists(mount_filename_path):
ret = True
except Exception as e:
logging.error("Failed to mount gluster volume %s to"
" mount dir %s: %s" % (sg_uri, tmpdir_path, e))
finally:
if utils_misc.umount(sg_uri, tmpdir_path, "glusterfs", False,
"fuse.glusterfs"):
try:
os.rmdir(tmpdir_path)
except OSError:
pass
else:
logging.warning("Unable to unmount tmp directory %s with glusterfs"
" mount.", tmpdir_path)
return ret
def file_exists(params, filename_path):
sg_uri = create_gluster_uri(params, stripped=True)
g_uri = create_gluster_uri(params, stripped=False)
# Using directly /tmp dir because directory should be really temporary and
# should be deleted immediately when no longer needed and
# created directory don't file tmp dir by any data.
tmpdir = "gmount-%s" % (utils_misc.generate_random_string(6))
tmpdir_path = os.path.join(data_dir.get_tmp_dir(), tmpdir)
while os.path.exists(tmpdir_path):
tmpdir = "gmount-%s" % (utils_misc.generate_random_string(6))
tmpdir_path = os.path.join(data_dir.get_tmp_dir(), tmpdir)
ret = False
try:
try:
os.mkdir(tmpdir_path)
glusterfs_mount(sg_uri, tmpdir_path)
mount_filename_path = os.path.join(tmpdir_path,
filename_path[len(g_uri):])
if os.path.exists(mount_filename_path):
ret = True
except Exception as e:
logging.error("Failed to mount gluster volume %s to"
" mount dir %s: %s" % (sg_uri, tmpdir_path, e))
finally:
if utils_misc.umount(sg_uri, tmpdir_path, "glusterfs", False,
"fuse.glusterfs"):
try:
os.rmdir(tmpdir_path)
except OSError:
pass
else:
logging.warning(
"Unable to unmount tmp directory %s with glusterfs"
" mount.", tmpdir_path)
return ret
def test_virt_tar_out(test, vm, params):
"""
1) Write a tempfile to guest
2) Copy file to host with tar-out
3) Delete created file
"""
content = "This is file for test of virt-tar-out."
path = params.get("vt_temp_file", "/tmp/test_virt_tar_out")
file_dir = os.path.dirname(path)
path_on_host = os.path.join(data_dir.get_tmp_dir(),
"test_virt_tar_out.tar")
vt = utils_test.libguestfs.VirtTools(vm, params)
mountpoint = params.get("vt_mountpoint")
if mountpoint is None:
tmpdir = "gmount-%s" % (utils_misc.generate_random_string(6))
mountpoint = "/tmp/%s" % tmpdir
if not os.path.exists(mountpoint):
os.mkdir(mountpoint)
writes, writeo = vt.write_file_with_guestmount(mountpoint, path, content,
cleanup=False)
if utils_misc.umount("", mountpoint, "") is False:
logging.error("Umount vm's filesytem failed.")
if writes is False:
test.fail("Write file to mounted filesystem failed.")
logging.info("Create %s successfully.", path)
# Copy file to host
tar_out_result = vt.tar_out(file_dir, path_on_host)
logging.debug(tar_out_result)
if tar_out_result.exit_status:
test.fail("Tar out failed.")
logging.info("Tar out successfully.")
# uncompress file and check file in it.
uc_result = process.run("cd %s && tar xf %s" % (file_dir, path_on_host),
shell=True)
logging.debug(uc_result)
try:
os.remove(path_on_host)
except IOError as detail:
test.fail(str(detail))
if uc_result.exit_status:
test.fail("uncompress file on host failed.")
logging.info("uncompress file on host successfully.")
# Check file
cat_result = process.run("cat %s" % path, ignore_status=True, shell=True)
logging.debug(cat_result)
try:
os.remove(path)
except IOError as detail:
logging.error(detail)
if cat_result.exit_status:
test.fail("Cat file failed.")
else:
if not re.search(content, cat_result.stdout_text):
test.fail("Catted file do not match.")
def mount_hugepages(page_size):
"""
To mount hugepages
:param page_size: unit is kB, it can be 4,2048,1048576,etc
"""
if page_size == 4:
perm = ""
else:
perm = "pagesize=%dK" % page_size
tlbfs_status = utils_misc.is_mounted("hugetlbfs", "/dev/hugepages",
"hugetlbfs")
if tlbfs_status:
utils_misc.umount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
utils_misc.mount("hugetlbfs", "/dev/hugepages", "hugetlbfs", perm)
def mount_hugepages(page_size):
"""
To mount hugepages
:param page_size: unit is kB, it can be 4,2048,1048576,etc
"""
if page_size == 4:
perm = ""
else:
perm = "pagesize=%dK" % page_size
tlbfs_status = utils_misc.is_mounted("hugetlbfs", "/dev/hugepages",
"hugetlbfs")
if tlbfs_status:
utils_misc.umount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
utils_misc.mount("hugetlbfs", "/dev/hugepages", "hugetlbfs", perm)
def test_virt_tar_out(test, vm, params):
"""
1) Write a tempfile to guest
2) Copy file to host with tar-out
3) Delete created file
"""
content = "This is file for test of virt-tar-out."
path = params.get("vt_temp_file", "/tmp/test_virt_tar_out")
file_dir = os.path.dirname(path)
path_on_host = os.path.join(data_dir.get_tmp_dir(),
"test_virt_tar_out.tar")
vt = utils_test.libguestfs.VirtTools(vm, params)
mountpoint = params.get("vt_mountpoint")
if mountpoint is None:
tmpdir = "gmount-%s" % (utils_misc.generate_random_string(6))
mountpoint = "/tmp/%s" % tmpdir
if not os.path.exists(mountpoint):
os.mkdir(mountpoint)
writes, writeo = vt.write_file_with_guestmount(mountpoint, path, content,
cleanup=False)
if utils_misc.umount("", mountpoint, "") is False:
logging.error("Umount vm's filesystem failed.")
if writes is False:
test.fail("Write file to mounted filesystem failed.")
logging.info("Create %s successfully.", path)
# Copy file to host
tar_out_result = vt.tar_out(file_dir, path_on_host)
logging.debug(tar_out_result)
if tar_out_result.exit_status:
test.fail("Tar out failed.")
logging.info("Tar out successfully.")
# uncompress file and check file in it.
uc_result = process.run("cd %s && tar xf %s" % (file_dir, path_on_host),
shell=True)
logging.debug(uc_result)
try:
os.remove(path_on_host)
except IOError as detail:
test.fail(str(detail))
if uc_result.exit_status:
test.fail("uncompress file on host failed.")
logging.info("uncompress file on host successfully.")
# Check file
cat_result = process.run("cat %s" % path, ignore_status=True, shell=True)
logging.debug(cat_result)
try:
os.remove(path)
except IOError as detail:
logging.error(detail)
if cat_result.exit_status:
test.fail("Cat file failed.")
else:
if not re.search(content, cat_result.stdout_text):
test.fail("Catted file do not match.")
def backup_img_chain(image_file):
"""
Backup whole image in a image chain;
"""
mount_point = tempfile.mkdtemp(dir=test.resultsdir)
qemu_img = utils_misc.get_qemu_img_binary(params)
if enable_gluster:
g_uri = gluster.create_gluster_uri(params)
gluster.glusterfs_mount(g_uri, mount_point)
image_name = os.path.basename(image_file)
image_file = os.path.join(mount_point, image_name)
logging.warn("backup %s to %s" % (image_file, test.resultsdir))
shutil.copy(image_file, test.resultsdir)
backing_file = _info(qemu_img, image_file, "backing file", None)
if backing_file:
backup_img_chain(backing_file)
elif enable_gluster:
utils_misc.umount(g_uri, mount_point, "glusterfs", False, "fuse.glusterfs")
shutil.rmtree(mount_point)
return None
def backup_img_chain(image_file):
"""
Backup whole image in a image chain;
"""
mount_point = tempfile.mkdtemp(dir=test.resultsdir)
qemu_img = utils_misc.get_qemu_img_binary(params)
if enable_gluster:
g_uri = gluster.create_gluster_uri(params)
gluster.glusterfs_mount(g_uri, mount_point)
image_name = os.path.basename(image_file)
image_file = os.path.join(mount_point, image_name)
logging.warn("backup %s to %s" % (image_file, test.resultsdir))
shutil.copy(image_file, test.resultsdir)
backing_file = _info(qemu_img, image_file, "backing file", None)
if backing_file:
backup_img_chain(backing_file)
elif enable_gluster:
utils_misc.umount(g_uri, mount_point, "glusterfs", False,
"fuse.glusterfs")
shutil.rmtree(mount_point)
return None
def recover(self, params):
"""
Recover test environment
"""
abnormal_type = params.get("abnormal_type")
cpu_enable = True if self.cpu_status else False
utils_misc.set_cpu_status(self.cpu_num, cpu_enable)
if virsh.domain_exists(self.vm_new_name):
virsh.remove_domain(self.vm_new_name)
if os.path.exists(self.new_image_file):
os.remove(self.new_image_file)
if self.twice_execute:
if virsh.domain_exists(self.vm_new_name1):
virsh.remove_domain(self.vm_new_name1)
if os.path.exists(self.new_image_file1):
os.remove(self.new_image_file1)
if abnormal_type == "memory_lack":
if params.has_key('memory_pid'):
pid = params.get('memory_pid')
if isinstance(pid,str):
pid = int(pid)
utils_misc.safe_kill(pid, signal.SIGKILL)
utils.run("swapon -a")
tmp_c_file = params.get("tmp_c_file", "/tmp/test.c")
tmp_exe_file = params.get("tmp_exe_file", "/tmp/test")
if os.path.exists(tmp_c_file):
os.remove(tmp_c_file)
if os.path.exists(tmp_exe_file):
os.remove(tmp_exe_file)
elif abnormal_type in ["disk_lack", ""]:
if self.selinux_enforcing:
utils_selinux.set_status("enforcing")
tmp_file = os.path.join(self.mount_dir, "tmp")
if os.path.exists(tmp_file):
os.remove(tmp_file)
# Sometimes one umount action is not enough
utils_misc.wait_for(lambda: utils_misc.umount(self.partition,
self.mount_dir,
self.fs_type), 120)
if self.iscsi_dev:
self.iscsi_dev.cleanup()
os.rmdir(self.mount_dir)
elif abnormal_type == "cpu_lack":
os.system("cat /sys/fs/cgroup/cpuset/cpuset.cpus > /sys/fs/cgroup/cpuset/machine.slice/cpuset.cpus")
remove_machine_cgroup()
def test_virt_copy_out(vm, params):
"""
1) Write a tempfile to guest
2) Copy file to host with copy-out
3) Delete created file
4) Check file on host
"""
content = "This is file for test of virt-copy-out."
path = params.get("vt_temp_file", "/tmp/test_virt_copy_out")
path_dir = os.path.dirname(path)
vt = utils_test.libguestfs.VirtTools(vm, params)
mountpoint = params.get("vt_mountpoint")
if mountpoint is None:
tmpdir = "gmount-%s" % (utils_misc.generate_random_string(6))
mountpoint = "/tmp/%s" % tmpdir
if not os.path.exists(mountpoint):
os.mkdir(mountpoint)
writes, writeo = vt.write_file_with_guestmount(mountpoint,
path,
content,
cleanup=False)
if utils_misc.umount("", mountpoint, "") is False:
logging.error("Umount vm's filesytem failed.")
if writes is False:
raise error.TestFail("Write file to mounted filesystem failed.")
logging.info("Create %s successfully.", path)
# Copy file to host
copy_out_result = vt.copy_out(path, path_dir)
logging.debug(copy_out_result)
if copy_out_result.exit_status:
raise error.TestFail("Copy out failed.")
logging.info("Copy out successfully.")
# Check file
cat_result = utils.run("cat %s" % path, ignore_status=True)
logging.debug(cat_result.stdout)
try:
os.remove(path)
except IOError, detail:
logging.error(detail)
def recover(self, params):
"""
Recover test environment
"""
abnormal_type = params.get("abnormal_type")
if self.cpu_status:
cpu.offline(self.cpu_num)
else:
cpu.online(self.cpu_num)
if virsh.domain_exists(self.vm_new_name):
virsh.remove_domain(self.vm_new_name)
if os.path.exists(self.new_image_file):
os.remove(self.new_image_file)
if self.twice_execute:
if virsh.domain_exists(self.vm_new_name1):
virsh.remove_domain(self.vm_new_name1)
if os.path.exists(self.new_image_file1):
os.remove(self.new_image_file1)
if abnormal_type == "memory_lack":
if 'memory_pid' in params:
pid = params.get('memory_pid')
utils_misc.safe_kill(pid, signal.SIGKILL)
process.run("swapon -a", shell=True)
tmp_c_file = params.get("tmp_c_file", "/tmp/test.c")
tmp_exe_file = params.get("tmp_exe_file", "/tmp/test")
if os.path.exists(tmp_c_file):
os.remove(tmp_c_file)
if os.path.exists(tmp_exe_file):
os.remove(tmp_exe_file)
elif abnormal_type in ["disk_lack", ""]:
if self.selinux_enforcing:
utils_selinux.set_status("enforcing")
tmp_file = os.path.join(self.mount_dir, "tmp")
if os.path.exists(tmp_file):
os.remove(tmp_file)
# Sometimes one umount action is not enough
utils_misc.wait_for(
lambda: utils_misc.umount(self.partition, self.mount_dir, self.
fs_type), 120)
if self.iscsi_dev:
self.iscsi_dev.cleanup()
os.rmdir(self.mount_dir)
remove_machine_cgroup()
def test_virt_copy_out(vm, params):
"""
1) Write a tempfile to guest
2) Copy file to host with copy-out
3) Delete created file
4) Check file on host
"""
content = "This is file for test of virt-copy-out."
path = params.get("vt_temp_file", "/tmp/test_virt_copy_out")
path_dir = os.path.dirname(path)
vt = utils_test.libguestfs.VirtTools(vm, params)
mountpoint = params.get("vt_mountpoint")
if mountpoint is None:
tmpdir = "gmount-%s" % (utils_misc.generate_random_string(6))
mountpoint = "/tmp/%s" % tmpdir
if not os.path.exists(mountpoint):
os.mkdir(mountpoint)
writes, writeo = vt.write_file_with_guestmount(mountpoint, path, content,
cleanup=False)
if utils_misc.umount("", mountpoint, "") is False:
logging.error("Umount vm's filesytem failed.")
if writes is False:
raise error.TestFail("Write file to mounted filesystem failed.")
logging.info("Create %s successfully.", path)
# Copy file to host
copy_out_result = vt.copy_out(path, path_dir)
logging.debug(copy_out_result)
if copy_out_result.exit_status:
raise error.TestFail("Copy out failed.")
logging.info("Copy out successfully.")
# Check file
cat_result = utils.run("cat %s" % path, ignore_status=True)
logging.debug(cat_result.stdout)
try:
os.remove(path)
except IOError, detail:
logging.error(detail)
def recover(self, params):
"""
Recover test environment
"""
abnormal_type = params.get("abnormal_type")
cpu_enable = True if self.cpu_status else False
utils_misc.set_cpu_status(self.cpu_num, cpu_enable)
if virsh.domain_exists(self.vm_new_name):
virsh.remove_domain(self.vm_new_name)
if os.path.exists(self.new_image_file):
os.remove(self.new_image_file)
if self.twice_execute:
if virsh.domain_exists(self.vm_new_name1):
virsh.remove_domain(self.vm_new_name1)
if os.path.exists(self.new_image_file1):
os.remove(self.new_image_file1)
if abnormal_type == "memory_lack":
if 'memory_pid' in params:
pid = params.get('memory_pid')
utils_misc.safe_kill(pid, signal.SIGKILL)
process.run("swapon -a", shell=True)
tmp_c_file = params.get("tmp_c_file", "/tmp/test.c")
tmp_exe_file = params.get("tmp_exe_file", "/tmp/test")
if os.path.exists(tmp_c_file):
os.remove(tmp_c_file)
if os.path.exists(tmp_exe_file):
os.remove(tmp_exe_file)
elif abnormal_type in ["disk_lack", ""]:
if self.selinux_enforcing:
utils_selinux.set_status("enforcing")
tmp_file = os.path.join(self.mount_dir, "tmp")
if os.path.exists(tmp_file):
os.remove(tmp_file)
# Sometimes one umount action is not enough
utils_misc.wait_for(lambda: utils_misc.umount(self.partition,
self.mount_dir,
self.fs_type), 120)
if self.iscsi_dev:
self.iscsi_dev.cleanup()
os.rmdir(self.mount_dir)
remove_machine_cgroup()
def test_unformatted_part(vm, params):
"""
1) Do some necessary check
2) Format additional disk without filesystem type
3) Try to mount device
"""
add_device = params.get("gf_additional_device", "/dev/vdb")
device_in_lgf = utils.run("echo %s | sed -e 's/vd/sd/g'" % add_device,
ignore_status=True).stdout.strip()
device_part = "%s1" % device_in_lgf
# Mount specific partition
params['special_mountpoints'] = [device_part]
vt = utils_test.libguestfs.VirtTools(vm, params)
# Create a new vm with additional disk
vt.update_vm_disk()
# Default vm_ref is oldvm, so switch it.
vm_ref = vt.newvm.name
# Format disk
format_result = vt.format_disk()
if format_result.exit_status:
raise error.TestFail("Format added disk failed.")
logging.info("Format added disk successfully.")
# List filesystems detail
list_fs_detail = vt.get_filesystems_info(vm_ref)
if list_fs_detail.exit_status:
raise error.TestFail("List filesystems detail failed:"
"%s" % list_fs_detail)
logging.info("List filesystems detail successfully.")
mountpoint = params.get("vt_mountpoint", "/mnt")
mounts, mounto = vt.guestmount(mountpoint, vm_ref)
if utils_misc.umount("", mountpoint, "") and mounts:
raise error.TestFail("Mount vm's filesytem successfully, "
"but not expected.")
logging.info("Mount vm's filesystem failed as expected.")
def test_unformatted_part(vm, params):
"""
1) Do some necessary check
2) Format additional disk without filesystem type
3) Try to mount device
"""
add_device = params.get("gf_additional_device", "/dev/vdb")
device_in_lgf = utils.run("echo %s | sed -e 's/vd/sd/g'" % add_device,
ignore_status=True).stdout.strip()
device_part = "%s1" % device_in_lgf
# Mount specific partition
params['special_mountpoints'] = [device_part]
vt = utils_test.libguestfs.VirtTools(vm, params)
# Create a new vm with additional disk
vt.update_vm_disk()
# Default vm_ref is oldvm, so switch it.
vm_ref = vt.newvm.name
# Format disk
format_result = vt.format_disk()
if format_result.exit_status:
raise error.TestFail("Format added disk failed.")
logging.info("Format added disk successfully.")
# List filesystems detail
list_fs_detail = vt.get_filesystems_info(vm_ref)
if list_fs_detail.exit_status:
raise error.TestFail("List filesystems detail failed:"
"%s" % list_fs_detail)
logging.info("List filesystems detail successfully.")
mountpoint = params.get("vt_mountpoint", "/mnt")
mounts, mounto = vt.guestmount(mountpoint, vm_ref)
if utils_misc.umount("", mountpoint, "") and mounts:
raise error.TestFail("Mount vm's filesytem successfully, "
"but not expected.")
logging.info("Mount vm's filesystem failed as expected.")
def umount(self):
"""
Umount the given mount point.
"""
return utils_misc.umount(self.mount_src, self.mount_dir, "nfs")
def run(test, params, env):
"""
Test steps:
1) Get the params from params.
2) check the environment
3) Strat the VM and check whether the VM been started successfully
4) Compare the Hugepage memory size to the Guest memory setted.
5) Check the hugepage memory usage.
6) Clean up
"""
test_type = params.get("test_type", 'normal')
tlbfs_enable = 'yes' == params.get("hugetlbfs_enable", 'no')
shp_num = int(params.get("static_hugepage_num", 1024))
thp_enable = 'yes' == params.get("trans_hugepage_enable", 'no')
mb_enable = 'yes' == params.get("mb_enable", 'yes')
delay = int(params.get("delay_time", 10))
# Skip cases early
vm_names = []
if test_type == "contrast":
vm_names = params.get("vms").split()[:2]
if len(vm_names) < 2:
test.cancel("This test requires two VMs")
# confirm no VM running
allvms = virsh.dom_list('--name').stdout.strip()
if allvms != '':
test.cancel("one or more VMs are alive")
err_range = float(params.get("mem_error_range", 1.25))
else:
vm_names.append(params.get("main_vm"))
if test_type == "stress":
target_path = params.get("target_path", "/tmp/test.out")
elif test_type == "unixbench":
unixbench_control_file = params.get("unixbench_controle_file",
"unixbench5.control")
# backup orignal setting
shp_orig_num = utils_memory.get_num_huge_pages()
thp_orig_status = utils_memory.get_transparent_hugepage()
page_size = utils_memory.get_huge_page_size()
# mount/umount hugetlbfs
tlbfs_status = utils_misc.is_mounted("hugetlbfs", "/dev/hugepages",
"hugetlbfs")
if tlbfs_enable is True:
if tlbfs_status is not True:
utils_misc.mount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
else:
if tlbfs_status is True:
utils_misc.umount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
# set static hugepage
utils_memory.set_num_huge_pages(shp_num)
# enable/disable transparent hugepage
if thp_enable:
utils_memory.set_transparent_hugepage('always')
else:
utils_memory.set_transparent_hugepage('never')
# set/del memoryBacking tag
for vm_name in vm_names:
if mb_enable:
vm_xml.VMXML.set_memoryBacking_tag(vm_name)
else:
vm_xml.VMXML.del_memoryBacking_tag(vm_name)
utils_libvirtd.libvirtd_restart()
non_started_free = utils_memory.get_num_huge_pages_free()
vms = []
sessions = []
try:
for vm_name in vm_names:
# try to start vm and login
try:
vm = env.get_vm(vm_name)
vm.start()
except VMError as e:
if mb_enable and not tlbfs_enable:
# if hugetlbfs not be mounted,
# VM start with memoryBacking tag will fail
logging.debug(e)
else:
error_msg = "Test failed in positive case. error: %s\n" % e
test.fail(error_msg)
if vm.is_alive() is not True:
break
vms.append(vm)
# try to login and run some program
try:
session = vm.wait_for_login()
except (LoginError, ShellError) as e:
error_msg = "Test failed in positive case.\n error: %s\n" % e
test.fail(error_msg)
sessions.append(session)
if test_type == "stress":
# prepare file for increasing stress
stress_path = prepare_c_file()
remote.scp_to_remote(vm.get_address(), 22,
'root', params.get('password'),
stress_path, "/tmp/")
# Try to install gcc on guest first
utils_package.package_install(["gcc"], session, 360)
# increasing workload
session.cmd("gcc %s -o %s" % (stress_path, target_path))
session.cmd("%s &" % target_path)
if test_type == "unixbench":
params["main_vm"] = vm_name
params["test_control_file"] = unixbench_control_file
control_path = os.path.join(test.virtdir, "control",
unixbench_control_file)
# unixbench test need 'patch' and 'perl' commands installed
utils_package.package_install(["patch", "perl"], session, 360)
command = utils_test.run_autotest(vm, session, control_path,
None, None, params,
copy_only=True)
session.cmd("%s &" % command, ignore_all_errors=True)
# wait for autotest running on vm
time.sleep(delay)
def _is_unixbench_running():
cmd = "ps -ef | grep perl | grep Run"
return not session.cmd_status(cmd)
if not utils_misc.wait_for(_is_unixbench_running, timeout=240):
test.cancel("Failed to run unixbench in guest,"
" please make sure some necessary"
" packages are installed in guest,"
" such as gcc, tar, bzip2")
logging.debug("Unixbench test is running in VM")
if test_type == "contrast":
# wait for vm finish starting completely
time.sleep(delay)
if not (mb_enable and not tlbfs_enable):
logging.debug("starting analyzing the hugepage usage...")
pid = vms[-1].get_pid()
started_free = utils_memory.get_num_huge_pages_free()
# Get the thp usage from /proc/pid/smaps
started_anon = utils_memory.get_num_anon_huge_pages(pid)
static_used = non_started_free - started_free
hugepage_used = static_used * page_size
if test_type == "contrast":
# get qemu-kvm memory consumption by top
cmd = "top -b -n 1|awk '$1 == %s {print $10}'" % pid
rate = process.run(cmd, ignore_status=False,
verbose=True, shell=True).stdout_text.strip()
qemu_kvm_used = (utils_memory.memtotal() * float(rate)) / 100
logging.debug("rate: %s, used-by-qemu-kvm: %f, used-by-vm: %d",
rate, qemu_kvm_used, hugepage_used)
if abs(qemu_kvm_used - hugepage_used) > hugepage_used * (err_range - 1):
test.fail("Error for hugepage usage")
if test_type == "stress":
if non_started_free <= started_free:
logging.debug("hugepage usage:%d -> %d", non_started_free,
started_free)
test.fail("Error for hugepage usage with stress")
if mb_enable is not True:
if static_used > 0:
test.fail("VM use static hugepage without"
" memoryBacking element")
if thp_enable is not True and started_anon > 0:
test.fail("VM use transparent hugepage, while"
" it's disabled")
else:
if tlbfs_enable is not True:
if static_used > 0:
test.fail("VM use static hugepage without tlbfs"
" mounted")
if thp_enable and started_anon <= 0:
test.fail("VM doesn't use transparent"
" hugepage")
else:
if shp_num > 0:
if static_used <= 0:
test.fail("VM doesn't use static"
" hugepage")
else:
if static_used > 0:
test.fail("VM use static hugepage,"
" while it's set to zero")
if thp_enable is not True:
if started_anon > 0:
test.fail("VM use transparent hugepage,"
" while it's disabled")
else:
if shp_num == 0 and started_anon <= 0:
test.fail("VM doesn't use transparent"
" hugepage, while static"
" hugepage is disabled")
finally:
# end up session
for session in sessions:
session.close()
for vm in vms:
if vm.is_alive():
vm.destroy()
for vm_name in vm_names:
if mb_enable:
vm_xml.VMXML.del_memoryBacking_tag(vm_name)
else:
vm_xml.VMXML.set_memoryBacking_tag(vm_name)
utils_libvirtd.libvirtd_restart()
if tlbfs_enable is True:
if tlbfs_status is not True:
utils_misc.umount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
else:
if tlbfs_status is True:
utils_misc.mount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
utils_memory.set_num_huge_pages(shp_orig_num)
utils_memory.set_transparent_hugepage(thp_orig_status)
def run(test, params, env):
"""
Test steps:
1) Get the params from params.
2) check the environment
3) Strat the VM and check whether the VM been started successfully
4) Compare the Hugepage memory size to the Guest memory setted.
5) Check the hugepage memory usage.
6) Clean up
"""
test_type = params.get("test_type", 'normal')
tlbfs_enable = 'yes' == params.get("hugetlbfs_enable", 'no')
shp_num = int(params.get("static_hugepage_num", 1024))
thp_enable = 'yes' == params.get("trans_hugepage_enable", 'no')
mb_enable = 'yes' == params.get("mb_enable", 'yes')
delay = int(params.get("delay_time", 10))
# backup orignal setting
shp_orig_num = utils_memory.get_num_huge_pages()
thp_orig_status = utils_memory.get_transparent_hugepage()
page_size = utils_memory.get_huge_page_size()
if test_type == "contrast":
range = float(params.get("mem_error_range", 1.25))
elif test_type == "stress":
target_path = params.get("target_path", "/tmp/test.out")
elif test_type == "unixbench":
unixbench_control_file = params.get("unixbench_controle_file",
"unixbench5.control")
vm_names = []
if test_type == "contrast":
vm_names = params.get("vms").split()[:2]
if len(vm_names) < 2:
raise error.TestNAError("Hugepage Stress Test need two VM(s).")
# confirm no VM(s) running
allvms = virsh.dom_list('--name').stdout.strip()
if allvms != '':
raise error.TestNAError("one or more VM(s) is living.")
else:
vm_names.append(params.get("main_vm"))
# mount/umount hugetlbfs
tlbfs_status = utils_misc.is_mounted("hugetlbfs", "/dev/hugepages", "hugetlbfs")
if tlbfs_enable is True:
if tlbfs_status is not True:
utils_misc.mount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
else:
if tlbfs_status is True:
utils_misc.umount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
# set static hugepage
utils_memory.set_num_huge_pages(shp_num)
# enable/disable transparent hugepage
if thp_enable:
utils_memory.set_transparent_hugepage('always')
else:
utils_memory.set_transparent_hugepage('never')
# set/del memoryBacking tag
for vm_name in vm_names:
if mb_enable:
vm_xml.VMXML.set_memoryBacking_tag(vm_name)
else:
vm_xml.VMXML.del_memoryBacking_tag(vm_name)
utils_libvirtd.libvirtd_restart()
non_started_free = utils_memory.get_num_huge_pages_free()
vms = []
sessions = []
for vm_name in vm_names:
# try to start vm and login
try:
vm = env.get_vm(vm_name)
vm.start()
except VMError, e:
if mb_enable and not tlbfs_enable:
# if hugetlbfs not be mounted,
# VM start with memoryBacking tag will fail
logging.debug(e)
pass # jump out of for-loop
else:
error_msg = "Test failed in positive case. error: %s\n" % e
raise error.TestFail(error_msg)
if vm.is_alive() is not True:
break
vms.append(vm)
# try to login and run some program
try:
session = vm.wait_for_login()
except (LoginError, ShellError), e:
error_msg = "Test failed in positive case.\n error: %s\n" % e
raise error.TestFail(error_msg)
def run(test, params, env):
"""
Test hpt resizing
"""
vm_name = params.get('main_vm')
vm = env.get_vm(vm_name)
status_error = 'yes' == params.get('status_error', 'no')
error_msg = eval(params.get('error_msg', '[]'))
hpt_attrs = eval(params.get('hpt_attrs', '{}'))
hpt_order_path = params.get('hpt_order_path', '')
cpu_attrs = eval(params.get('cpu_attrs', '{}'))
numa_cell = eval(params.get('numa_cell', '{}'))
hugepage = 'yes' == params.get('hugepage', 'no')
maxpagesize = int(params.get('maxpagesize', 0))
check_hp = 'yes' == params.get('check_hp', 'no')
qemu_check = params.get('qemu_check', '')
skip_p8 = 'yes' == params.get('skip_p8', 'no')
def set_hpt(vmxml, sync, **attrs):
"""
Set resizing value to vm xml
:param vmxml: xml of vm to be manipulated
:param sync: whether to sync vmxml after
:param attrs: attrs to set to hpt xml
"""
if vmxml.xmltreefile.find('/features'):
features_xml = vmxml.features
else:
features_xml = vm_xml.VMFeaturesXML()
hpt_xml = vm_xml.VMFeaturesHptXML()
for attr in attrs:
setattr(hpt_xml, attr, attrs[attr])
features_xml.hpt = hpt_xml
vmxml.features = features_xml
logging.debug(vmxml)
if sync:
vmxml.sync()
def set_cpu(vmxml, **attrs):
"""
Set cpu attrs for vmxml according to given attrs
:param vmxml: xml of vm to be manipulated
:param attrs: attrs to set to cpu xml
"""
if vmxml.xmltreefile.find('cpu'):
cpu = vmxml.cpu
else:
cpu = vm_xml.VMCPUXML()
if 'numa_cell' in attrs:
cpu.xmltreefile.create_by_xpath('/numa')
cpu.numa_cell = attrs['numa_cell']
for key in attrs:
setattr(cpu, key, attrs[key])
vmxml.cpu = cpu
vmxml.sync()
def set_memory(vmxml):
"""
Set memory attributes in vm xml
"""
vmxml.max_mem_rt = int(params.get('max_mem_rt', 30670848))
vmxml.max_mem_rt_slots = int(params.get('max_mem_rt_slots', 16))
vmxml.max_mem_rt_unit = params.get('max_mem_rt_unit', 'KiB')
logging.debug(numa_cell)
if numa_cell:
# Remove cpu topology to avoid that it doesn't match vcpu count
if vmxml.get_cpu_topology():
new_cpu = vmxml.cpu
new_cpu.del_topology()
vmxml.cpu = new_cpu
vmxml.vcpu = max([int(cell['cpus'][-1]) for cell in numa_cell]) + 1
vmxml.sync()
def check_hpt_order(session, resizing=''):
"""
Return htp order in hpt_order file by default
If 'resizing' is disabled, test updating htp_order
"""
if not hpt_order_path:
test.cancel('No hpt order path provided.')
hpt_order = session.cmd_output('cat %s' % hpt_order_path).strip()
hpt_order = int(hpt_order)
logging.info('Current hpt_order is %d', hpt_order)
if resizing == 'disabled':
cmd_result = session.cmd_status_output(
'echo %d > %s' % (hpt_order + 1, hpt_order_path))
result = process.CmdResult(stderr=cmd_result[1],
exit_status=cmd_result[0])
libvirt.check_exit_status(result, True)
libvirt.check_result(result, error_msg)
return hpt_order
def check_hp_in_vm(session, page_size):
"""
Check if hugepage size is correct inside vm
:param session: the session of the running vm
:param page_size: the expected pagesize to be checked inside vm
"""
expect = False if int(page_size) == 65536 else True
meminfo = session.cmd_output('cat /proc/meminfo|grep Huge')
logging.info('meminfo: \n%s', meminfo)
pattern = 'Hugepagesize:\s+%d\s+kB' % int(page_size / 1024)
logging.info('"%s" should %s be found in meminfo output', pattern,
'' if expect else 'not')
result = expect == bool(re.search(pattern, meminfo))
if not result:
test.fail('meminfo output not meet expectation')
# Check PAGE_SIZE in another way
if not expect:
conf_page_size = session.cmd_output('getconf PAGE_SIZE')
logging.debug('Output of "getconf PAGE_SIZE": %s', conf_page_size)
if int(conf_page_size) != int(page_size):
test.fail(
'PAGE_SIZE not correct, should be %r, actually is %r' %
(page_size, conf_page_size))
bk_xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
try:
arch = platform.machine()
vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
resizing = hpt_attrs.get('resizing')
# Test on ppc64le hosts
if arch.lower() == 'ppc64le':
cpu_arch = cpu.get_cpu_arch()
logging.debug('cpu_arch is: %s', cpu_arch)
if skip_p8 and cpu_arch == 'power8':
test.cancel('This case is not for POWER8')
if maxpagesize and not utils_misc.compare_qemu_version(3, 1, 0):
test.cancel('Qemu version is too low, '
'does not support maxpagesize setting')
if maxpagesize == 16384 and cpu_arch == 'power9':
test.cancel('Power9 does not support 16M pagesize.')
set_hpt(vmxml, True, **hpt_attrs)
if cpu_attrs or numa_cell:
if numa_cell:
cpu_attrs['numa_cell'] = numa_cell
set_cpu(vmxml, **cpu_attrs)
if hugepage:
vm_mem = vmxml.max_mem
host_hp_size = utils_memory.get_huge_page_size()
# Make 100m extra memory just to be safe
hp_count = max((vm_mem + 102400) // host_hp_size, 1200)
vm_xml.VMXML.set_memoryBacking_tag(vm_name, hpgs=True)
# Set up hugepage env
mnt_source, hp_path, fstype = 'hugetlbfs', '/dev/hugepages', 'hugetlbfs'
if not os.path.isdir(hp_path):
process.run('mkdir %s' % hp_path, verbose=True)
utils_memory.set_num_huge_pages(hp_count)
if utils_misc.is_mounted(mnt_source,
hp_path,
fstype,
verbose=True):
utils_misc.umount(mnt_source,
hp_path,
fstype,
verbose=True)
utils_misc.mount(mnt_source, hp_path, fstype, verbose=True)
# Restart libvirtd service to make sure mounted hugepage
# be recognized
utils_libvirtd.libvirtd_restart()
if resizing == 'enabled':
set_memory(vmxml)
logging.debug('vmxml: \n%s', vmxml)
# Start vm and check if start succeeds
result = virsh.start(vm_name, debug=True)
libvirt.check_exit_status(result, expect_error=status_error)
# if vm is not suposed to start, terminate test
if status_error:
libvirt.check_result(result, error_msg)
return
libvirt.check_qemu_cmd_line(qemu_check)
session = vm.wait_for_login()
hpt_order = check_hpt_order(session, resizing)
# Check hugepage inside vm
if check_hp:
check_hp_in_vm(session, maxpagesize * 1024)
if resizing == 'enabled':
mem_xml = utils_hotplug.create_mem_xml(
tg_size=int(params.get('mem_size', 2048000)),
tg_sizeunit=params.get('size_unit', 'KiB'),
tg_node=int(params.get('mem_node', 0)),
mem_model=params.get('mem_model', 'dimm'))
logging.debug(mem_xml)
# Attach memory device to the guest for 12 times
# that will reach the maxinum memory limitation
for i in range(12):
virsh.attach_device(vm_name,
mem_xml.xml,
debug=True,
ignore_status=False)
xml_after_attach = vm_xml.VMXML.new_from_dumpxml(vm_name)
logging.debug(xml_after_attach)
# Check dumpxml of the guest,
# check if each device has its alias
for i in range(12):
pattern = "alias\s+name=[\'\"]dimm%d[\'\"]" % i
logging.debug('Searching for %s', pattern)
if not re.search(pattern, str(
xml_after_attach.xmltreefile)):
test.fail('Missing memory alias: %s' % pattern)
# Test on non-ppc64le hosts
else:
set_hpt(vmxml, sync=False, **hpt_attrs)
result = virsh.define(vmxml.xml)
libvirt.check_exit_status(result, status_error)
libvirt.check_result(result, error_msg)
finally:
bk_xml.sync()
if hugepage:
utils_misc.umount('hugetlbfs', '/dev/hugepages', 'hugetlbfs')
utils_memory.set_num_huge_pages(0)
def test_created_volume(vm, params):
"""
1) Do some necessary check
2) Format additional disk with new volume
3) Login to check created volume
"""
add_device = params.get("gf_additional_device", "/dev/vdb")
device_in_lgf = utils.run("echo %s | sed -e 's/vd/sd/g'" % add_device,
ignore_status=True).stdout.strip()
if utils_test.libguestfs.primary_disk_virtio(vm):
device_in_vm = add_device
else:
device_in_vm = "/dev/vda"
device_part = "%s1" % device_in_lgf
device_part_in_vm = "%s1" % device_in_vm
# Mount specific partition
params['special_mountpoints'] = [device_part]
vt = utils_test.libguestfs.VirtTools(vm, params)
# Create a new vm with additional disk
vt.update_vm_disk()
# Default vm_ref is oldvm, so switch it.
vm_ref = vt.newvm.name
# Format disk
volume_name = "/dev/VGTest/LVTest"
format_result = vt.format_disk(filesystem="ext3", lvm=volume_name)
if format_result.exit_status:
raise error.TestFail("Format added disk failed.")
logging.info("Format added disk successfully.")
# List filesystems detail
list_fs_detail = vt.get_filesystems_info(vm_ref)
if list_fs_detail.exit_status:
raise error.TestFail("List filesystems detail failed.")
logging.info("List filesystems detail successfully.")
mountpoint = params.get("vt_mountpoint", "/mnt")
path = os.path.join(mountpoint, "test_created_volume.img")
params['special_mountpoints'] = [volume_name]
mounts, mounto = vt.guestmount(mountpoint, vm_ref)
if mounts is False:
raise error.TestFail("Mount vm's filesystem failed.")
else:
dfs, dfo = commands.getstatusoutput("df")
logging.debug(dfo)
if not re.search("/dev/fuse", dfo):
utils_misc.umount("", mountpoint, "")
raise error.TestFail("Did not find mounted filesystem.")
dd_cmd = "dd if=/dev/zero of=%s bs=1M count=5" % path
dds, ddo = commands.getstatusoutput(dd_cmd)
logging.debug(ddo)
if dds:
utils_misc.umount("", mountpoint, "")
raise error.TestFail("Create a image file failed.")
md5s, md5o = commands.getstatusoutput("md5sum %s" % path)
logging.debug(md5o)
if md5s:
utils_misc.umount("", mountpoint, "")
raise error.TestFail("Get md5 value failed.")
if not utils_misc.umount("", mountpoint, ""):
raise error.TestFail("Unmount vm's filesytem failed.")
logging.info("Unmount vm's filesystem successfully.")
attached_vm = vt.newvm
try:
attached_vm.start()
session = attached_vm.wait_for_login()
if session.cmd_status("which lvs"):
attached_vm.destroy()
attached_vm.wait_for_shutdown()
logging.error("Can not use volume in guest, SKIP...")
return
except (virt_vm.VMError, remote.LoginError), detail:
attached_vm.destroy()
raise error.TestFail(str(detail))
def run(test, params, env):
"""
Qemu numa stress test:
1) Boot up a guest and find the node it used
2) Try to allocate memory in that node
3) Run memory heavy stress inside guest
4) Check the memory use status of qemu process
5) Repeat step 2 ~ 4 several times
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
host_numa_node = utils_misc.NumaInfo()
if len(host_numa_node.online_nodes) < 2:
raise error.TestNAError("Host only has one NUMA node, "
"skipping test...")
timeout = float(params.get("login_timeout", 240))
test_count = int(params.get("test_count", 4))
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session = vm.wait_for_login(timeout=timeout)
qemu_pid = vm.get_pid()
if test_count < len(host_numa_node.online_nodes):
test_count = len(host_numa_node.online_nodes)
tmpfs_size = 0
for node in host_numa_node.nodes:
node_mem = int(host_numa_node.read_from_node_meminfo(node, "MemTotal"))
if tmpfs_size < node_mem:
tmpfs_size = node_mem
tmpfs_path = params.get("tmpfs_path", "tmpfs_numa_test")
tmpfs_path = utils_misc.get_path(data_dir.get_tmp_dir(), tmpfs_path)
tmpfs_write_speed = int(params.get("tmpfs_write_speed", 10240))
dd_timeout = tmpfs_size / tmpfs_write_speed * 1.5
mount_fs_size = "size=%dK" % tmpfs_size
memory_file = utils_misc.get_path(tmpfs_path, "test")
dd_cmd = "dd if=/dev/urandom of=%s bs=1k count=%s" % (memory_file,
tmpfs_size)
if not os.path.isdir(tmpfs_path):
os.mkdir(tmpfs_path)
numa_node_malloc = -1
most_used_node, memory_used = max_mem_map_node(host_numa_node, qemu_pid)
for test_round in range(test_count):
if utils_memory.freememtotal() < tmpfs_size:
raise error.TestError("Don't have enough memory to execute this "
"test after %s round" % test_round)
error.context("Executing stress test round: %s" % test_round,
logging.info)
numa_node_malloc = most_used_node
numa_dd_cmd = "numactl -m %s %s" % (numa_node_malloc, dd_cmd)
error.context("Try to allocate memory in node %s" % numa_node_malloc,
logging.info)
try:
utils_misc.mount("none", tmpfs_path, "tmpfs", perm=mount_fs_size)
funcatexit.register(env, params.get("type"), utils_misc.umount,
"none", tmpfs_path, "tmpfs")
utils.system(numa_dd_cmd, timeout=dd_timeout)
except Exception, error_msg:
if "No space" in str(error_msg):
pass
else:
raise error.TestFail("Can not allocate memory in node %s."
" Error message:%s" % (numa_node_malloc,
str(error_msg)))
error.context("Run memory heavy stress in guest", logging.info)
autotest_control.run(test, params, env)
error.context("Get the qemu process memory use status", logging.info)
node_after, memory_after = max_mem_map_node(host_numa_node, qemu_pid)
if node_after == most_used_node and memory_after >= memory_used:
raise error.TestFail("Memory still stick in "
"node %s" % numa_node_malloc)
else:
most_used_node = node_after
memory_used = memory_after
utils_misc.umount("none", tmpfs_path, "tmpfs")
funcatexit.unregister(env, params.get("type"), utils_misc.umount,
"none", tmpfs_path, "tmpfs")
session.cmd("sync; echo 3 > /proc/sys/vm/drop_caches")
utils_memory.drop_caches()
def run(test, params, env):
"""
Convert specific esx guest
"""
for v in list(params.values()):
if "V2V_EXAMPLE" in v:
test.cancel("Please set real value for %s" % v)
if utils_v2v.V2V_EXEC is None:
raise ValueError('Missing command: virt-v2v')
vpx_hostname = params.get('vpx_hostname')
vpx_passwd = params.get("vpx_password")
esxi_host = esx_ip = params.get('esx_hostname')
vpx_dc = params.get('vpx_dc')
vm_name = params.get('main_vm')
output_mode = params.get('output_mode')
pool_name = params.get('pool_name', 'v2v_test')
pool_type = params.get('pool_type', 'dir')
pool_target = params.get('pool_target_path', 'v2v_pool')
pvt = libvirt.PoolVolumeTest(test, params)
v2v_timeout = int(params.get('v2v_timeout', 1200))
status_error = 'yes' == params.get('status_error', 'no')
address_cache = env.get('address_cache')
checkpoint = params.get('checkpoint', '')
error_list = []
remote_host = vpx_hostname
# For VDDK
input_transport = params.get("input_transport")
vddk_libdir = params.get('vddk_libdir')
# nfs mount source
vddk_libdir_src = params.get('vddk_libdir_src')
vddk_thumbprint = params.get('vddk_thumbprint')
src_uri_type = params.get('src_uri_type')
esxi_password = params.get('esxi_password')
# For construct rhv-upload option in v2v cmd
output_method = params.get("output_method")
rhv_upload_opts = params.get("rhv_upload_opts")
storage_name = params.get('storage_name')
# for get ca.crt file from ovirt engine
rhv_passwd = params.get("rhv_upload_passwd")
rhv_passwd_file = params.get("rhv_upload_passwd_file")
ovirt_engine_passwd = params.get("ovirt_engine_password")
ovirt_hostname = params.get("ovirt_engine_url").split(
'/')[2] if params.get("ovirt_engine_url") else None
ovirt_ca_file_path = params.get("ovirt_ca_file_path")
local_ca_file_path = params.get("local_ca_file_path")
def log_fail(msg):
"""
Log error and update error list
"""
logging.error(msg)
error_list.append(msg)
def check_device_exist(check, virsh_session_id):
"""
Check if device exist after convertion
"""
xml = virsh.dumpxml(vm_name, session_id=virsh_session_id).stdout
if check == 'cdrom':
if "device='cdrom'" not in xml:
log_fail('CDROM no longer exists')
def check_vmtools(vmcheck, check):
"""
Check whether vmware tools packages have been removed,
or vmware-tools service has stopped
:param vmcheck: VMCheck object for vm checking
:param check: Checkpoint of different cases
:return: None
"""
if check == 'vmtools':
logging.info('Check if packages been removed')
pkgs = vmcheck.session.cmd('rpm -qa').strip()
removed_pkgs = params.get('removed_pkgs').strip().split(',')
if not removed_pkgs:
test.error('Missing param "removed_pkgs"')
for pkg in removed_pkgs:
if pkg in pkgs:
log_fail('Package "%s" not removed' % pkg)
elif check == 'vmtools_service':
logging.info('Check if service stopped')
vmtools_service = params.get('service_name')
status = utils_misc.get_guest_service_status(
vmcheck.session, vmtools_service)
logging.info('Service %s status: %s', vmtools_service, status)
if status != 'inactive':
log_fail('Service "%s" is not stopped' % vmtools_service)
def check_modprobe(vmcheck):
"""
Check whether content of /etc/modprobe.conf meets expectation
"""
content = vmcheck.session.cmd('cat /etc/modprobe.conf').strip()
logging.debug(content)
cfg_content = params.get('cfg_content')
if not cfg_content:
test.error('Missing content for search')
logging.info('Search "%s" in /etc/modprobe.conf', cfg_content)
pattern = r'\s+'.join(cfg_content.split())
if not re.search(pattern, content):
log_fail('Not found "%s"' % cfg_content)
def check_device_map(vmcheck):
"""
Check if the content of device.map meets expectation.
"""
logging.info(vmcheck.session.cmd('fdisk -l').strip())
device_map = params.get('device_map_path')
content = vmcheck.session.cmd('cat %s' % device_map)
logging.debug('Content of device.map:\n%s', content)
logging.info('Found device: %d', content.count('/dev/'))
logging.info('Found virtio device: %d', content.count('/dev/vd'))
if content.count('/dev/') != content.count('/dev/vd'):
log_fail('Content of device.map not correct')
else:
logging.info('device.map has been remaped to "/dev/vd*"')
def check_resume_swap(vmcheck):
"""
Check the content of grub files meet expectation.
"""
# Only for grub2
chkfiles = [
'/etc/default/grub', '/boot/grub2/grub.cfg', '/etc/grub2.cfg'
]
for file_i in chkfiles:
status, content = vmcheck.run_cmd('cat %s' % file_i)
if status != 0:
log_fail('%s does not exist' % file_i)
resume_dev_count = content.count('resume=/dev/')
if resume_dev_count == 0 or resume_dev_count != content.count(
'resume=/dev/vd'):
reason = 'Maybe the VM\'s swap pariton is lvm'
log_fail('Content of %s is not correct or %s' %
(file_i, reason))
content = vmcheck.session.cmd('cat /proc/cmdline')
logging.debug('Content of /proc/cmdline:\n%s', content)
if 'resume=/dev/vd' not in content:
log_fail('Content of /proc/cmdline is not correct')
def check_rhev_file_exist(vmcheck):
"""
Check if rhev files exist
"""
file_path = {
'rhev-apt.exe': r'C:\rhev-apt.exe',
'rhsrvany.exe':
r'"C:\Program Files\Guestfs\Firstboot\rhsrvany.exe"'
}
for key in file_path:
status = vmcheck.session.cmd_status('dir %s' % file_path[key])
if status == 0:
logging.info('%s exists' % key)
else:
log_fail('%s does not exist after convert to rhv' % key)
def check_file_architecture(vmcheck):
"""
Check the 3rd party module info
:param vmcheck: VMCheck object for vm checking
"""
content = vmcheck.session.cmd('uname -r').strip()
status = vmcheck.session.cmd_status(
'rpm -qf /lib/modules/%s/fileaccess/fileaccess_mod.ko ' % content)
if status == 0:
log_fail('3rd party module info is not correct')
else:
logging.info(
'file /lib/modules/%s/fileaccess/fileaccess_mod.ko is not owned by any package'
% content)
def check_ogac(vmcheck):
"""
Check qemu-guest-agent service in VM
:param vmcheck: VMCheck object for vm checking
"""
def get_pkgs(pkg_path):
"""
Get all qemu-guest-agent pkgs
"""
pkgs = []
for _, _, files in os.walk(pkg_path):
for file_name in files:
pkgs.append(file_name)
return pkgs
def get_pkg_version_vm():
"""
Get qemu-guest-agent version in VM
"""
vender = vmcheck.get_vm_os_vendor()
if vender in ['Ubuntu', 'Debian']:
cmd = 'dpkg -l qemu-guest-agent'
else:
cmd = 'rpm -q qemu-guest-agent'
_, output = vmcheck.run_cmd(cmd)
pkg_ver_ptn = [
r'qemu-guest-agent +[0-9]+:(.*?dfsg.*?) +',
r'qemu-guest-agent-(.*?)\.x86_64'
]
for ptn in pkg_ver_ptn:
if re.search(ptn, output):
return re.search(ptn, output).group(1)
return ''
mount_point = utils_v2v.v2v_mount(os.getenv('VIRTIO_WIN'),
'rhv_tools_setup_iso',
fstype='iso9660')
params['tmp_mount_point'] = mount_point
qemu_guest_agent_dir = os.path.join(mount_point, 'linux')
all_pkgs = get_pkgs(qemu_guest_agent_dir)
logging.debug('All packages in qemu-guest-agent-iso: %s' % all_pkgs)
vm_pkg_ver = get_pkg_version_vm()
logging.debug('qemu-guest-agent verion in vm: %s' % vm_pkg_ver)
# If qemu-guest-agent version in VM is higher than the pkg in qemu-guest-agent-iso,
# v2v will not update the qemu-guest-agent version and report a warning.
#
# e.g.
# virt-v2v: warning: failed to install QEMU Guest Agent: command: package
# qemu-guest-agent-10:2.12.0-3.el7.x86_64 (which is newer than
# qemu-guest-agent-10:2.12.0-2.el7.x86_64) is already installed
if not any([vm_pkg_ver in pkg for pkg in all_pkgs]):
logging.debug(
'Wrong qemu-guest-agent version, maybe it is higher than package version in ISO'
)
logging.info(
'Unexpected qemu-guest-agent version, set v2v log checking')
expect_msg_ptn = r'virt-v2v: warning: failed to install QEMU Guest Agent.*?is newer than.*? is already installed'
params.update({'msg_content': expect_msg_ptn, 'expect_msg': 'yes'})
# Check the service status of qemu-guest-agent in VM
status_ptn = r'Active: active \(running\)|qemu-ga \(pid +[0-9]+\) is running'
cmd = 'service qemu-ga status;systemctl status qemu-guest-agent'
_, output = vmcheck.run_cmd(cmd)
if not re.search(status_ptn, output):
log_fail('qemu-guest-agent service exception')
def check_result(result, status_error):
"""
Check virt-v2v command result
"""
libvirt.check_exit_status(result, status_error)
output = result.stdout + result.stderr
if checkpoint == 'empty_cdrom':
if status_error:
log_fail('Virsh dumpxml failed for empty cdrom image')
elif not status_error:
if output_mode == 'rhev':
if not utils_v2v.import_vm_to_ovirt(
params, address_cache, timeout=v2v_timeout):
test.fail('Import VM failed')
elif output_mode == 'libvirt':
virsh.start(vm_name, debug=True)
# Check guest following the checkpoint document after convertion
logging.info('Checking common checkpoints for v2v')
vmchecker = VMChecker(test, params, env)
params['vmchecker'] = vmchecker
if checkpoint != 'GPO_AV':
ret = vmchecker.run()
if len(ret) == 0:
logging.info("All common checkpoints passed")
# Check specific checkpoints
if checkpoint == 'cdrom':
virsh_session = utils_sasl.VirshSessionSASL(params)
virsh_session_id = virsh_session.get_id()
check_device_exist('cdrom', virsh_session_id)
if checkpoint.startswith('vmtools'):
check_vmtools(vmchecker.checker, checkpoint)
if checkpoint == 'modprobe':
check_modprobe(vmchecker.checker)
if checkpoint == 'device_map':
check_device_map(vmchecker.checker)
if checkpoint == 'resume_swap':
check_resume_swap(vmchecker.checker)
if checkpoint == 'rhev_file':
check_rhev_file_exist(vmchecker.checker)
if checkpoint == 'file_architecture':
check_file_architecture(vmchecker.checker)
if checkpoint == 'ogac':
check_ogac(vmchecker.checker)
# Merge 2 error lists
error_list.extend(vmchecker.errors)
log_check = utils_v2v.check_log(params, output)
if log_check:
log_fail(log_check)
if len(error_list):
test.fail('%d checkpoints failed: %s' %
(len(error_list), error_list))
try:
v2v_params = {
'hostname': remote_host,
'hypervisor': 'esx',
'main_vm': vm_name,
'vpx_dc': vpx_dc,
'esx_ip': esx_ip,
'new_name': vm_name + utils_misc.generate_random_string(4),
'v2v_opts': '-v -x',
'input_mode': 'libvirt',
'storage': params.get('output_storage', 'default'),
'network': params.get('network'),
'bridge': params.get('bridge'),
'target': params.get('target'),
'password': vpx_passwd if src_uri_type != 'esx' else esxi_password,
'input_transport': input_transport,
'vcenter_host': vpx_hostname,
'vcenter_password': vpx_passwd,
'vddk_thumbprint': vddk_thumbprint,
'vddk_libdir': vddk_libdir,
'vddk_libdir_src': vddk_libdir_src,
'src_uri_type': src_uri_type,
'esxi_password': esxi_password,
'esxi_host': esxi_host,
'output_method': output_method,
'storage_name': storage_name,
'rhv_upload_opts': rhv_upload_opts
}
os.environ['LIBGUESTFS_BACKEND'] = 'direct'
v2v_uri = utils_v2v.Uri('esx')
remote_uri = v2v_uri.get_uri(remote_host, vpx_dc, esx_ip)
# Create password file for access to ESX hypervisor
vpx_passwd_file = params.get("vpx_passwd_file")
with open(vpx_passwd_file, 'w') as pwd_f:
if src_uri_type == 'esx':
pwd_f.write(esxi_password)
else:
pwd_f.write(vpx_passwd)
v2v_params['v2v_opts'] += " -ip %s" % vpx_passwd_file
if params.get('output_format'):
v2v_params.update({'output_format': params['output_format']})
# Rename guest with special name while converting to rhev
if '#' in vm_name and output_mode == 'rhev':
v2v_params['new_name'] = v2v_params['new_name'].replace('#', '_')
# Create SASL user on the ovirt host
if output_mode == 'rhev':
# create different sasl_user name for different job
params.update({
'sasl_user':
params.get("sasl_user") + utils_misc.generate_random_string(3)
})
logging.info('sals user name is %s' % params.get("sasl_user"))
user_pwd = "[['%s', '%s']]" % (params.get("sasl_user"),
params.get("sasl_pwd"))
v2v_sasl = utils_sasl.SASL(sasl_user_pwd=user_pwd)
v2v_sasl.server_ip = params.get("remote_ip")
v2v_sasl.server_user = params.get('remote_user')
v2v_sasl.server_pwd = params.get('remote_pwd')
v2v_sasl.setup(remote=True)
if output_method == 'rhv_upload':
# Create password file for '-o rhv_upload' to connect to ovirt
with open(rhv_passwd_file, 'w') as f:
f.write(rhv_passwd)
# Copy ca file from ovirt to local
remote.scp_from_remote(ovirt_hostname, 22, 'root',
ovirt_engine_passwd, ovirt_ca_file_path,
local_ca_file_path)
# Create libvirt dir pool
if output_mode == 'libvirt':
pvt.pre_pool(pool_name, pool_type, pool_target, '')
if checkpoint == 'root_ask':
v2v_params['v2v_opts'] += ' --root ask'
v2v_params['custom_inputs'] = params.get('choice', '2')
if checkpoint.startswith('root_') and checkpoint != 'root_ask':
root_option = params.get('root_option')
v2v_params['v2v_opts'] += ' --root %s' % root_option
if checkpoint == 'with_proxy':
http_proxy = params.get('esx_http_proxy')
https_proxy = params.get('esx_https_proxy')
logging.info('Set http_proxy=%s, https_proxy=%s', http_proxy,
https_proxy)
os.environ['http_proxy'] = http_proxy
os.environ['https_proxy'] = https_proxy
if checkpoint == 'ogac':
rhv_iso_path = '/usr/share/rhv-guest-tools-iso/rhv-tools-setup.iso'
os.environ['VIRTIO_WIN'] = rhv_iso_path
if not os.path.isfile(os.getenv('VIRTIO_WIN')):
test.fail('%s does not exist' % os.getenv('VIRTIO_WIN'))
if checkpoint == 'empty_cdrom':
virsh_dargs = {
'uri': remote_uri,
'remote_ip': remote_host,
'remote_user': '******',
'remote_pwd': vpx_passwd,
'debug': True
}
remote_virsh = virsh.VirshPersistent(**virsh_dargs)
v2v_result = remote_virsh.dumpxml(vm_name)
else:
v2v_result = utils_v2v.v2v_cmd(v2v_params)
if 'new_name' in v2v_params:
vm_name = params['main_vm'] = v2v_params['new_name']
check_result(v2v_result, status_error)
finally:
if checkpoint == 'ogac' and params.get('tmp_mount_point'):
if os.path.exists(params.get('tmp_mount_point')):
utils_misc.umount(os.getenv('VIRTIO_WIN'),
params['tmp_mount_point'], 'iso9660')
os.environ.pop('VIRTIO_WIN')
if params.get('vmchecker'):
params['vmchecker'].cleanup()
if output_mode == 'libvirt':
pvt.cleanup_pool(pool_name, pool_type, pool_target, '')
if checkpoint == 'with_proxy':
logging.info('Unset http_proxy&https_proxy')
os.environ.pop('http_proxy')
os.environ.pop('https_proxy')
# Cleanup constant files
utils_v2v.cleanup_constant_files(params)
def cleanup(self):
# Cleanup NFS mount records
for src, dst, fstype in self.mount_record.values():
utils_misc.umount(src, dst, fstype)
def run(test, params, env):
"""
Test various options of virt-v2v.
"""
if utils_v2v.V2V_EXEC is None:
raise ValueError('Missing command: virt-v2v')
for v in list(params.values()):
if "V2V_EXAMPLE" in v:
test.cancel("Please set real value for %s" % v)
vm_name = params.get("main_vm", "EXAMPLE")
new_vm_name = params.get("new_vm_name")
input_mode = params.get("input_mode")
v2v_options = params.get("v2v_options", "")
hypervisor = params.get("hypervisor", "kvm")
remote_host = params.get("remote_host", "EXAMPLE")
vpx_dc = params.get("vpx_dc", "EXAMPLE")
esx_ip = params.get("esx_ip", "EXAMPLE")
source_user = params.get("username", "root")
output_mode = params.get("output_mode")
output_storage = params.get("output_storage", "default")
disk_img = params.get("input_disk_image", "")
nfs_storage = params.get("storage")
no_root = 'yes' == params.get('no_root', 'no')
mnt_point = params.get("mnt_point")
export_domain_uuid = params.get("export_domain_uuid", "")
fake_domain_uuid = params.get("fake_domain_uuid")
vdsm_image_uuid = params.get("vdsm_image_uuid")
vdsm_vol_uuid = params.get("vdsm_vol_uuid")
vdsm_vm_uuid = params.get("vdsm_vm_uuid")
vdsm_ovf_output = params.get("vdsm_ovf_output")
v2v_user = params.get("unprivileged_user", "")
v2v_timeout = int(params.get("v2v_timeout", 1200))
status_error = "yes" == params.get("status_error", "no")
su_cmd = "su - %s -c " % v2v_user
output_uri = params.get("oc_uri", "")
pool_name = params.get("pool_name", "v2v_test")
pool_type = params.get("pool_type", "dir")
pool_target = params.get("pool_target", "v2v_pool")
emulated_img = params.get("emulated_image_path", "v2v-emulated-img")
pvt = utlv.PoolVolumeTest(test, params)
new_v2v_user = False
address_cache = env.get('address_cache')
params['vmcheck_flag'] = False
checkpoint = params.get('checkpoint', '')
error_flag = 'strict'
def create_pool(user_pool=False,
pool_name=pool_name,
pool_target=pool_target):
"""
Create libvirt pool as the output storage
"""
if output_uri == "qemu:///session" or user_pool:
target_path = os.path.join("/home", v2v_user, pool_target)
cmd = su_cmd + "'mkdir %s'" % target_path
process.system(cmd, verbose=True)
cmd = su_cmd + "'virsh pool-create-as %s dir" % pool_name
cmd += " --target %s'" % target_path
process.system(cmd, verbose=True)
else:
pvt.pre_pool(pool_name, pool_type, pool_target, emulated_img)
def cleanup_pool(user_pool=False,
pool_name=pool_name,
pool_target=pool_target):
"""
Clean up libvirt pool
"""
if output_uri == "qemu:///session" or user_pool:
cmd = su_cmd + "'virsh pool-destroy %s'" % pool_name
process.system(cmd, verbose=True)
target_path = os.path.join("/home", v2v_user, pool_target)
cmd = su_cmd + "'rm -rf %s'" % target_path
process.system(cmd, verbose=True)
else:
pvt.cleanup_pool(pool_name, pool_type, pool_target, emulated_img)
def get_all_uuids(output):
"""
Get export domain uuid, image uuid and vol uuid from command output.
"""
tmp_target = re.findall(r"qemu-img\s'convert'\s.+\s'(\S+)'\n", output)
if len(tmp_target) < 1:
test.error("Fail to find tmp target file name when converting vm"
" disk image")
targets = tmp_target[0].split('/')
return (targets[3], targets[5], targets[6])
def get_ovf_content(output):
"""
Find and read ovf file.
"""
export_domain_uuid, _, vol_uuid = get_all_uuids(output)
export_vm_dir = os.path.join(mnt_point, export_domain_uuid,
'master/vms')
ovf_content = ""
if os.path.isdir(export_vm_dir):
ovf_id = "ovf:id='%s'" % vol_uuid
ret = to_text(
process.system_output("grep -R \"%s\" %s" %
(ovf_id, export_vm_dir)))
ovf_file = ret.split(":")[0]
if os.path.isfile(ovf_file):
ovf_f = open(ovf_file, "r")
ovf_content = ovf_f.read()
ovf_f.close()
else:
logging.error("Can't find ovf file to read")
return ovf_content
def get_img_path(output):
"""
Get the full path of the converted image.
"""
img_name = vm_name + "-sda"
if output_mode == "libvirt":
img_path = virsh.vol_path(img_name, output_storage).stdout.strip()
elif output_mode == "local":
img_path = os.path.join(output_storage, img_name)
elif output_mode in ["rhev", "vdsm"]:
export_domain_uuid, image_uuid, vol_uuid = get_all_uuids(output)
img_path = os.path.join(mnt_point, export_domain_uuid, 'images',
image_uuid, vol_uuid)
return img_path
def check_vmtype(ovf, expected_vmtype):
"""
Verify vmtype in ovf file.
"""
if output_mode != "rhev":
return
if expected_vmtype == "server":
vmtype_int = 1
elif expected_vmtype == "desktop":
vmtype_int = 0
else:
return
if "<VmType>%s</VmType>" % vmtype_int in ovf:
logging.info("Find VmType=%s in ovf file", expected_vmtype)
else:
test.fail("VmType check failed")
def check_image(img_path, check_point, expected_value):
"""
Verify image file allocation mode and format
"""
if not img_path or not os.path.isfile(img_path):
test.error("Image path: '%s' is invalid" % img_path)
img_info = utils_misc.get_image_info(img_path)
logging.debug("Image info: %s", img_info)
if check_point == "allocation":
if expected_value == "sparse":
if img_info['vsize'] > img_info['dsize']:
logging.info("%s is a sparse image", img_path)
else:
test.fail("%s is not a sparse image" % img_path)
elif expected_value == "preallocated":
if img_info['vsize'] <= img_info['dsize']:
logging.info("%s is a preallocated image", img_path)
else:
test.fail("%s is not a preallocated image" % img_path)
if check_point == "format":
if expected_value == img_info['format']:
logging.info("%s format is %s", img_path, expected_value)
else:
test.fail("%s format is not %s" % (img_path, expected_value))
def check_new_name(output, expected_name):
"""
Verify guest name changed to the new name.
"""
found = False
if output_mode == "libvirt":
found = virsh.domain_exists(expected_name)
if output_mode == "local":
found = os.path.isfile(
os.path.join(output_storage, expected_name + "-sda"))
if output_mode in ["rhev", "vdsm"]:
ovf = get_ovf_content(output)
found = "<Name>%s</Name>" % expected_name in ovf
else:
return
if found:
logging.info("Guest name renamed when converting it")
else:
test.fail("Rename guest failed")
def check_nocopy(output):
"""
Verify no image created if convert command use --no-copy option
"""
img_path = get_img_path(output)
if not os.path.isfile(img_path):
logging.info("No image created with --no-copy option")
else:
test.fail("Find %s" % img_path)
def check_connection(output, expected_uri):
"""
Check output connection uri used when converting guest
"""
init_msg = "Initializing the target -o libvirt -oc %s" % expected_uri
if init_msg in output:
logging.info("Find message: %s", init_msg)
else:
test.fail("Not find message: %s" % init_msg)
def check_ovf_snapshot_id(ovf_content):
"""
Check if snapshot id in ovf file consists of '0's
"""
search = re.search("ovf:vm_snapshot_id='(.*?)'", ovf_content)
if search:
snapshot_id = search.group(1)
logging.debug('vm_snapshot_id = %s', snapshot_id)
if snapshot_id.count('0') >= 32:
test.fail('vm_snapshot_id consists with "0"')
else:
test.fail('Fail to find snapshot_id')
def setup_esx_ssh_key(hostname, user, password, port=22):
"""
Setup up remote login in esx server by using public key
"""
logging.debug('Performing SSH key setup on %s:%d as %s.' %
(hostname, port, user))
try:
session = remote.remote_login(client='ssh',
host=hostname,
username=user,
port=port,
password=password,
prompt=r'[ $#%]')
public_key = ssh_key.get_public_key()
session.cmd("echo '%s' >> /etc/ssh/keys-root/authorized_keys; " %
public_key)
logging.debug('SSH key setup complete.')
session.close()
except Exception as err:
logging.debug('SSH key setup has failed. %s', err)
def check_source(output):
"""
Check if --print-source option print the correct info
"""
# Parse source info
source = output.split('\n')[2:]
for i in range(len(source)):
if source[i].startswith('\t'):
source[i - 1] += source[i]
source[i] = ''
source_strip = [x.strip() for x in source if x.strip()]
source_info = {}
for line in source_strip:
source_info[line.split(':')[0]] = line.split(':', 1)[1].strip()
logging.debug('Source info to check: %s', source_info)
checklist = [
'nr vCPUs', 'hypervisor type', 'source name', 'memory', 'disks',
'NICs'
]
if hypervisor in ['kvm', 'xen']:
checklist.extend(['display', 'CPU features'])
for key in checklist:
if key not in source_info:
test.fail('%s info missing' % key)
v2v_virsh = None
close_virsh = False
if hypervisor == 'kvm':
v2v_virsh = virsh
else:
virsh_dargs = {
'uri': ic_uri,
'remote_ip': remote_host,
'remote_user': source_user,
'remote_pwd': source_pwd,
'debug': True
}
v2v_virsh = virsh.VirshPersistent(**virsh_dargs)
close_virsh = True
# Check single values
fail = []
try:
xml = vm_xml.VMXML.new_from_inactive_dumpxml(
vm_name, virsh_instance=v2v_virsh)
finally:
if close_virsh:
v2v_virsh.close_session()
check_map = {}
check_map['nr vCPUs'] = xml.vcpu
check_map['hypervisor type'] = xml.hypervisor_type
check_map['source name'] = xml.vm_name
check_map['memory'] = str(int(xml.max_mem) * 1024) + ' (bytes)'
if hypervisor in ['kvm', 'xen']:
check_map['display'] = xml.get_graphics_devices()[0].type_name
logging.info('KEY:\tSOURCE<-> XML')
for key in check_map:
logging.info('%-15s:%18s <-> %s', key, source_info[key],
check_map[key])
if str(check_map[key]) not in source_info[key]:
fail.append(key)
# Check disk info
disk = list(xml.get_disk_all().values())[0]
def _get_disk_subelement_attr_value(obj, attr, subattr):
if obj.find(attr) is not None:
return obj.find(attr).get(subattr)
bus = _get_disk_subelement_attr_value(disk, 'target', 'bus')
driver_type = _get_disk_subelement_attr_value(disk, 'driver', 'type')
path = _get_disk_subelement_attr_value(disk, 'source', 'file')
# For esx, disk output is like "disks: json: { ... } (raw) [scsi]"
# For xen, disk output is like "disks: json: { ... } [ide]"
# For kvm, disk output is like "/rhel8.0-2.qcow2 (qcow2) [virtio-blk]"
if hypervisor == 'kvm':
disks_info_pattern = "%s \(%s\) \[%s" % (path, driver_type, bus)
elif hypervisor == 'esx':
# replace '.vmdk' with '-flat.vmdk', this is done in v2v
path_pattern1 = path.split()[1].replace('.vmdk', '-flat.vmdk')
# In newer qemu version, '_' is replaced with '%5f'.
path_pattern2 = path_pattern1.replace('_', '%5f')
# For esx, '(raw)' is fixed? Let's see if others will be met.
disks_info_pattern = '|'.join([
"https://%s/folder/%s\?dcPath=data&dsName=esx.*} \(raw\) \[%s"
% (remote_host, i, bus)
for i in [path_pattern1, path_pattern2]
])
elif hypervisor == 'xen':
disks_info_pattern = "file\.path.*%s.*file\.host.*%s.* \[%s" % (
path, remote_host, bus)
source_disks = source_info['disks'].split()
logging.info('disks:%s<->%s', source_info['disks'], disks_info_pattern)
if not re.search(disks_info_pattern, source_info['disks']):
fail.append('disks')
# Check nic info
nic = list(xml.get_iface_all().values())[0]
type = nic.get('type')
mac = nic.find('mac').get('address')
nic_source = nic.find('source')
name = nic_source.get(type)
nic_info = '%s "%s" mac: %s' % (type, name, mac)
logging.info('NICs:%s<->%s', source_info['NICs'], nic_info)
if nic_info.lower() not in source_info['NICs'].lower():
fail.append('NICs')
# Check cpu features
if hypervisor in ['kvm', 'xen']:
feature_list = xml.features.get_feature_list()
logging.info('CPU features:%s<->%s', source_info['CPU features'],
feature_list)
if sorted(source_info['CPU features'].split(',')) != sorted(
feature_list):
fail.append('CPU features')
if fail:
test.fail('Source info not correct for: %s' % fail)
def check_man_page(in_man, not_in_man):
"""
Check if content of man page or help info meets expectation
"""
man_page = process.run('man virt-v2v',
verbose=False).stdout_text.strip()
if in_man:
logging.info('Checking man page of virt-v2v for "%s"', in_man)
if in_man not in man_page:
test.fail('"%s" not in man page' % in_man)
if not_in_man:
logging.info('Checking man page of virt-v2v for "%s"', not_in_man)
if not_in_man in man_page:
test.fail('"%s" not removed from man page' % not_in_man)
def check_result(cmd, result, status_error):
"""
Check virt-v2v command result
"""
utils_v2v.check_exit_status(result, status_error, error_flag)
output = to_text(result.stdout + result.stderr, errors=error_flag)
output_stdout = to_text(result.stdout, errors=error_flag)
if status_error:
if checkpoint == 'length_of_error':
log_lines = output.split('\n')
v2v_start = False
for line in log_lines:
if line.startswith('virt-v2v:'):
v2v_start = True
if line.startswith('libvirt:'):
v2v_start = False
if v2v_start and len(line) > 72:
test.fail('Error log longer than 72 charactors: %s' %
line)
if checkpoint == 'disk_not_exist':
vol_list = virsh.vol_list(pool_name)
logging.info(vol_list)
if vm_name in vol_list.stdout:
test.fail('Disk exists for vm %s' % vm_name)
else:
if output_mode == "rhev" and checkpoint != 'quiet':
ovf = get_ovf_content(output)
logging.debug("ovf content: %s", ovf)
check_ovf_snapshot_id(ovf)
if '--vmtype' in cmd:
expected_vmtype = re.findall(r"--vmtype\s(\w+)", cmd)[0]
check_vmtype(ovf, expected_vmtype)
if '-oa' in cmd and '--no-copy' not in cmd:
expected_mode = re.findall(r"-oa\s(\w+)", cmd)[0]
img_path = get_img_path(output)
def check_alloc():
try:
check_image(img_path, "allocation", expected_mode)
return True
except exceptions.TestFail:
pass
if not utils_misc.wait_for(check_alloc, timeout=600,
step=10.0):
test.fail('Allocation check failed.')
if '-of' in cmd and '--no-copy' not in cmd and '--print-source' not in cmd and checkpoint != 'quiet':
expected_format = re.findall(r"-of\s(\w+)", cmd)[0]
img_path = get_img_path(output)
check_image(img_path, "format", expected_format)
if '-on' in cmd:
expected_name = re.findall(r"-on\s(\w+)", cmd)[0]
check_new_name(output, expected_name)
if '--no-copy' in cmd:
check_nocopy(output)
if '-oc' in cmd:
expected_uri = re.findall(r"-oc\s(\S+)", cmd)[0]
check_connection(output, expected_uri)
if output_mode == "rhev":
if not utils_v2v.import_vm_to_ovirt(params, address_cache):
test.fail("Import VM failed")
else:
params['vmcheck_flag'] = True
if output_mode == "libvirt":
if "qemu:///session" not in v2v_options and not no_root:
virsh.start(vm_name, debug=True, ignore_status=False)
if checkpoint == ['vmx', 'vmx_ssh']:
vmchecker = VMChecker(test, params, env)
params['vmchecker'] = vmchecker
params['vmcheck_flag'] = True
ret = vmchecker.run()
if len(ret) == 0:
logging.info("All common checkpoints passed")
if checkpoint == 'quiet':
if len(output.strip().splitlines()) > 10:
test.fail('Output is not empty in quiet mode')
if checkpoint == 'dependency':
if 'libguestfs-winsupport' not in output:
test.fail('libguestfs-winsupport not in dependency')
if all(pkg_pattern not in output
for pkg_pattern in ['VMF', 'edk2-ovmf']):
test.fail('OVMF/AAVMF not in dependency')
if 'qemu-kvm-rhev' in output:
test.fail('qemu-kvm-rhev is in dependency')
if 'libX11' in output:
test.fail('libX11 is in dependency')
if 'kernel-rt' in output:
test.fail('kernel-rt is in dependency')
win_img = params.get('win_image')
command = 'guestfish -a %s -i'
if process.run(command % win_img,
ignore_status=True).exit_status == 0:
test.fail('Command "%s" success' % command % win_img)
if checkpoint == 'no_dcpath':
if '--dcpath' in output:
test.fail('"--dcpath" is not removed')
if checkpoint == 'debug_overlays':
search = re.search('Overlay saved as(.*)', output)
if not search:
test.fail('Not find log of saving overlays')
overlay_path = search.group(1).strip()
logging.debug('Overlay file location: %s' % overlay_path)
if os.path.isfile(overlay_path):
logging.info('Found overlay file: %s' % overlay_path)
else:
test.fail('Overlay file not saved')
if checkpoint.startswith('empty_nic_source'):
target_str = '%s "eth0" mac: %s' % (params[checkpoint][0],
params[checkpoint][1])
logging.info('Expect log: %s', target_str)
if target_str not in output_stdout.lower():
test.fail('Expect log not found: %s' % target_str)
if checkpoint == 'print_source':
check_source(output_stdout)
if checkpoint == 'machine_readable':
if os.path.exists(params.get('example_file', '')):
# Checking items in example_file exist in latest
# output regardless of the orders and new items.
with open(params['example_file']) as f:
for line in f:
if line.strip() not in output_stdout.strip():
test.fail(
'%s not in --machine-readable output' %
line.strip())
else:
test.error('No content to compare with')
if checkpoint == 'compress':
img_path = get_img_path(output)
logging.info('Image path: %s', img_path)
qemu_img_cmd = 'qemu-img check %s' % img_path
qemu_img_locking_feature_support = libvirt_storage.check_qemu_image_lock_support(
)
if qemu_img_locking_feature_support:
qemu_img_cmd = 'qemu-img check %s -U' % img_path
disk_check = process.run(qemu_img_cmd).stdout_text
logging.info(disk_check)
compress_info = disk_check.split(',')[-1].split('%')[0].strip()
compress_rate = float(compress_info)
logging.info('%s%% compressed', compress_rate)
if compress_rate < 0.1:
test.fail('Disk image NOT compressed')
if checkpoint == 'tail_log':
messages = params['tail'].get_output()
logging.info('Content of /var/log/messages during conversion:')
logging.info(messages)
msg_content = params['msg_content']
if msg_content in messages:
test.fail('Found "%s" in /var/log/messages' % msg_content)
log_check = utils_v2v.check_log(params, output)
if log_check:
test.fail(log_check)
check_man_page(params.get('in_man'), params.get('not_in_man'))
backup_xml = None
vdsm_domain_dir, vdsm_image_dir, vdsm_vm_dir = ("", "", "")
try:
if checkpoint.startswith('empty_nic_source'):
xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
iface = xml.get_devices('interface')[0]
disks = xml.get_devices('disk')
del iface.source
iface.type_name = checkpoint.split('_')[-1]
iface.source = {iface.type_name: ''}
params[checkpoint] = [iface.type_name, iface.mac_address]
logging.debug(iface.source)
devices = vm_xml.VMXMLDevices()
devices.extend(disks)
devices.append(iface)
xml.set_devices(devices)
logging.info(xml.xmltreefile)
params['input_xml'] = xml.xmltreefile.name
# Build input options
input_option = ""
if input_mode is None:
pass
elif input_mode == "libvirt":
uri_obj = utils_v2v.Uri(hypervisor)
ic_uri = uri_obj.get_uri(remote_host, vpx_dc, esx_ip)
if checkpoint == 'with_ic':
ic_uri = 'qemu:///session'
input_option = "-i %s -ic %s %s" % (input_mode, ic_uri, vm_name)
if checkpoint == 'without_ic':
input_option = '-i %s %s' % (input_mode, vm_name)
# Build network&bridge option to avoid network error
v2v_options += " -b %s -n %s" % (params.get("output_bridge"),
params.get("output_network"))
elif input_mode == "disk":
input_option += "-i %s %s" % (input_mode, disk_img)
elif input_mode == 'libvirtxml':
input_xml = params.get('input_xml')
input_option += '-i %s %s' % (input_mode, input_xml)
elif input_mode in ['ova']:
test.cancel("Unsupported input mode: %s" % input_mode)
else:
test.error("Unknown input mode %s" % input_mode)
input_format = params.get("input_format", "")
input_allo_mode = params.get("input_allo_mode")
if input_format:
input_option += " -if %s" % input_format
if not status_error:
logging.info("Check image before convert")
check_image(disk_img, "format", input_format)
if input_allo_mode:
check_image(disk_img, "allocation", input_allo_mode)
# Build output options
output_option = ""
if output_mode:
output_option = "-o %s -os %s" % (output_mode, output_storage)
if checkpoint == 'rhv':
output_option = output_option.replace('rhev', 'rhv')
output_format = params.get("output_format")
if output_format and output_format != input_format:
output_option += " -of %s" % output_format
output_allo_mode = params.get("output_allo_mode")
if output_allo_mode:
output_option += " -oa %s" % output_allo_mode
# Build vdsm related options
if output_mode in ['vdsm', 'rhev']:
if not os.path.isdir(mnt_point):
os.mkdir(mnt_point)
if not utils_misc.mount(nfs_storage, mnt_point, "nfs"):
test.error("Mount NFS Failed")
if output_mode == 'vdsm':
v2v_options += " --vdsm-image-uuid %s" % vdsm_image_uuid
v2v_options += " --vdsm-vol-uuid %s" % vdsm_vol_uuid
v2v_options += " --vdsm-vm-uuid %s" % vdsm_vm_uuid
v2v_options += " --vdsm-ovf-output %s" % vdsm_ovf_output
vdsm_domain_dir = os.path.join(mnt_point, fake_domain_uuid)
vdsm_image_dir = os.path.join(mnt_point, export_domain_uuid,
"images", vdsm_image_uuid)
vdsm_vm_dir = os.path.join(mnt_point, export_domain_uuid,
"master/vms", vdsm_vm_uuid)
# For vdsm_domain_dir, just create a dir to test BZ#1176591
os.makedirs(vdsm_domain_dir)
os.makedirs(vdsm_image_dir)
os.makedirs(vdsm_vm_dir)
# Output more messages except quiet mode
if checkpoint == 'quiet':
v2v_options += ' -q'
elif checkpoint not in [
'length_of_error', 'empty_nic_source_network',
'empty_nic_source_bridge', 'machine_readable'
]:
v2v_options += " -v -x"
# Prepare for libvirt unprivileged user session connection
if "qemu:///session" in v2v_options or no_root:
try:
pwd.getpwnam(v2v_user)
except KeyError:
# create new user
process.system("useradd %s" % v2v_user, ignore_status=True)
new_v2v_user = True
user_info = pwd.getpwnam(v2v_user)
logging.info("Convert to qemu:///session by user '%s'", v2v_user)
if input_mode == "disk":
# Copy image from souce and change the image owner and group
disk_path = os.path.join(data_dir.get_tmp_dir(),
os.path.basename(disk_img))
logging.info('Copy image file %s to %s', disk_img, disk_path)
shutil.copyfile(disk_img, disk_path)
input_option = input_option.replace(disk_img, disk_path)
os.chown(disk_path, user_info.pw_uid, user_info.pw_gid)
elif not no_root:
test.cancel("Only support convert local disk")
# Setup ssh-agent access to xen hypervisor
if hypervisor == 'xen':
user = params.get("xen_host_user", "root")
source_pwd = passwd = params.get("xen_host_passwd", "redhat")
logging.info("set up ssh-agent access ")
ssh_key.setup_ssh_key(remote_host,
user=user,
port=22,
password=passwd)
utils_misc.add_identities_into_ssh_agent()
# Check if xen guest exists
uri = utils_v2v.Uri(hypervisor).get_uri(remote_host)
if not virsh.domain_exists(vm_name, uri=uri):
logging.error('VM %s not exists', vm_name)
# If the input format is not define, we need to either define
# the original format in the source metadata(xml) or use '-of'
# to force the output format, see BZ#1141723 for detail.
if '-of' not in v2v_options and checkpoint != 'xen_no_output_format':
v2v_options += ' -of %s' % params.get("default_output_format",
"qcow2")
# Create password file for access to ESX hypervisor
if hypervisor == 'esx':
source_pwd = vpx_passwd = params.get("vpx_password")
vpx_passwd_file = os.path.join(data_dir.get_tmp_dir(),
"vpx_passwd")
logging.info("Building ESX no password interactive verification.")
pwd_f = open(vpx_passwd_file, 'w')
pwd_f.write(vpx_passwd)
pwd_f.close()
output_option += " --password-file %s" % vpx_passwd_file
# if don't specify any output option for virt-v2v, 'default' pool
# will be used.
if output_mode is None:
# Cleanup first to avoid failure if 'default' pool exists.
pvt.cleanup_pool(pool_name, pool_type, pool_target, emulated_img)
pvt.pre_pool(pool_name, pool_type, pool_target, emulated_img)
# Create libvirt dir pool
if output_mode == "libvirt":
create_pool()
# Work around till bug fixed
os.environ['LIBGUESTFS_BACKEND'] = 'direct'
if checkpoint in ['with_ic', 'without_ic']:
new_v2v_user = True
v2v_options += ' -on %s' % new_vm_name
create_pool(user_pool=True,
pool_name='src_pool',
pool_target='v2v_src_pool')
create_pool(user_pool=True)
logging.debug(virsh.pool_list(uri='qemu:///session'))
sh_install_vm = params.get('sh_install_vm')
if not sh_install_vm:
test.error('Source vm installing script missing')
with open(sh_install_vm) as fh:
cmd_install_vm = fh.read().strip()
process.run('su - %s -c "%s"' % (v2v_user, cmd_install_vm),
timeout=10,
shell=True)
params['cmd_clean_vm'] = "%s 'virsh undefine %s'" % (su_cmd,
vm_name)
if checkpoint == 'vmx':
mount_point = params.get('mount_point')
if not os.path.isdir(mount_point):
os.mkdir(mount_point)
nfs_vmx = params.get('nfs_vmx')
if not utils_misc.mount(nfs_vmx, mount_point, 'nfs', verbose=True):
test.error('Mount nfs for vmx failed')
vmx = params.get('vmx')
input_option = '-i vmx %s' % vmx
v2v_options += " -b %s -n %s" % (params.get("output_bridge"),
params.get("output_network"))
if checkpoint == 'vmx_ssh':
esx_user = params.get("esx_host_user", "root")
esx_pwd = params.get("esx_host_passwd", "123qweP")
vmx = params.get('vmx')
setup_esx_ssh_key(esx_ip, esx_user, esx_pwd)
try:
utils_misc.add_identities_into_ssh_agent()
except Exception:
process.run("ssh-agent -k")
raise exceptions.TestError("Fail to setup ssh-agent")
input_option = '-i vmx -it ssh %s' % vmx
v2v_options += " -b %s -n %s" % (params.get("output_bridge"),
params.get("output_network"))
if checkpoint == 'simulate_nfs':
simulate_images = params.get("simu_images_path")
simulate_vms = params.get("simu_vms_path")
simulate_dom_md = params.get("simu_dom_md_path")
os.makedirs(simulate_images)
os.makedirs(simulate_vms)
process.run('touch %s' % simulate_dom_md)
process.run('chmod -R 777 /tmp/rhv/')
# Running virt-v2v command
cmd = "%s %s %s %s" % (utils_v2v.V2V_EXEC, input_option, output_option,
v2v_options)
if v2v_user:
cmd_export_env = 'export LIBGUESTFS_BACKEND=direct'
cmd = "%s '%s;%s'" % (su_cmd, cmd_export_env, cmd)
if params.get('cmd_free') == 'yes':
cmd = params.get('check_command')
# only set error to 'ignore' to avoid exception for RHEL7-84978
if "guestfish" in cmd:
error_flag = "replace"
# Set timeout to kill v2v process before conversion succeed
if checkpoint == 'disk_not_exist':
v2v_timeout = 30
# Get tail content of /var/log/messages
if checkpoint == 'tail_log':
params['tail_log'] = os.path.join(data_dir.get_tmp_dir(),
'tail_log')
params['tail'] = aexpect.Tail(command='tail -f /var/log/messages',
output_func=utils_misc.log_line,
output_params=(params['tail_log'], ))
cmd_result = process.run(cmd,
timeout=v2v_timeout,
verbose=True,
ignore_status=True)
if new_vm_name:
vm_name = new_vm_name
params['main_vm'] = new_vm_name
check_result(cmd, cmd_result, status_error)
finally:
if hypervisor == "xen":
process.run("ssh-agent -k")
if hypervisor == "esx":
process.run("rm -rf %s" % vpx_passwd_file)
for vdsm_dir in [vdsm_domain_dir, vdsm_image_dir, vdsm_vm_dir]:
if os.path.exists(vdsm_dir):
shutil.rmtree(vdsm_dir)
if os.path.exists(mnt_point):
utils_misc.umount(nfs_storage, mnt_point, "nfs")
os.rmdir(mnt_point)
if output_mode == "local":
image_name = vm_name + "-sda"
img_file = os.path.join(output_storage, image_name)
xml_file = img_file + ".xml"
for local_file in [img_file, xml_file]:
if os.path.exists(local_file):
os.remove(local_file)
if output_mode == "libvirt":
if "qemu:///session" in v2v_options or no_root:
cmd = su_cmd + "'virsh undefine %s'" % vm_name
try:
process.system(cmd)
except Exception:
logging.error('Undefine "%s" failed', vm_name)
if no_root:
cleanup_pool(user_pool=True,
pool_name='src_pool',
pool_target='v2v_src_pool')
cleanup_pool(user_pool=True)
else:
virsh.remove_domain(vm_name)
cleanup_pool()
if output_mode is None:
pvt.cleanup_pool(pool_name, pool_type, pool_target, emulated_img)
vmcheck_flag = params.get("vmcheck_flag")
if vmcheck_flag:
vmcheck = utils_v2v.VMCheck(test, params, env)
vmcheck.cleanup()
if checkpoint in ['with_ic', 'without_ic']:
process.run(params['cmd_clean_vm'])
if new_v2v_user:
process.system("userdel -f %s" % v2v_user)
if backup_xml:
backup_xml.sync()
if checkpoint == 'vmx':
utils_misc.umount(params['nfs_vmx'], params['mount_point'], 'nfs')
os.rmdir(params['mount_point'])
if checkpoint == 'simulate_nfs':
process.run('rm -rf /tmp/rhv/')
def test_formatted_part(vm, params):
"""
1) Do some necessary check
2) Format additional disk with specific filesystem
3) Try to write a file to mounted device and get md5
4) Login to check writed file and its md5
"""
add_device = params.get("gf_additional_device", "/dev/vdb")
device_in_lgf = utils.run("echo %s | sed -e 's/vd/sd/g'" % add_device,
ignore_status=True).stdout.strip()
if utils_test.libguestfs.primary_disk_virtio(vm):
device_in_vm = add_device
else:
device_in_vm = "/dev/vda"
device_part = "%s1" % device_in_lgf
device_part_in_vm = "%s1" % device_in_vm
# Mount specific partition
params['special_mountpoints'] = [device_part]
vt = utils_test.libguestfs.VirtTools(vm, params)
# Create a new vm with additional disk
vt.update_vm_disk()
# Default vm_ref is oldvm, so switch it.
vm_ref = vt.newvm.name
# Format disk
format_result = vt.format_disk(filesystem="ext3", partition="mbr")
if format_result.exit_status:
raise error.TestFail("Format added disk failed.")
logging.info("Format added disk successfully.")
# List filesystems detail
list_fs_detail = vt.get_filesystems_info(vm_ref)
if list_fs_detail.exit_status:
raise error.TestFail("List filesystems detail failed.")
logging.info("List filesystems detail successfully.")
content = "This is file for formatted part test."
path = params.get("temp_file", "formatted_part")
mountpoint = params.get("vt_mountpoint", "/mnt")
writes, writeo = vt.write_file_with_guestmount(mountpoint, path,
content, vm_ref,
cleanup=False)
if writes is False:
utils_misc.umount("", mountpoint, "")
raise error.TestFail("Write file to mounted filesystem failed.")
logging.info("Create %s successfully.", writeo)
# Compute new file's md5
if os.path.isfile(writeo):
md5s, md5o = commands.getstatusoutput("md5sum %s" % writeo)
utils_misc.umount("", mountpoint, "")
if md5s:
raise error.TestFail("Compute %s's md5 failed." % writeo)
md5 = md5o.split()[0].strip()
logging.debug("%s's md5 in newvm is:%s", writeo, md5)
else:
utils_misc.umount("", mountpoint, "")
raise error.TestFail("Can not find %s." % writeo)
attached_vm = vt.newvm
try:
attached_vm.start()
session = attached_vm.wait_for_login()
except Exception, detail:
attached_vm.destroy()
raise error.TestFail(str(detail))
def run(test, params, env):
"""
Test steps:
1) Get the params from params.
2) check the environment
3) Strat the VM and check whether the VM been started successfully
4) Compare the Hugepage memory size to the Guest memory setted.
5) Check the hugepage memory usage.
6) Clean up
"""
test_type = params.get("test_type", 'normal')
tlbfs_enable = 'yes' == params.get("hugetlbfs_enable", 'no')
shp_num = int(params.get("static_hugepage_num", 1024))
thp_enable = 'yes' == params.get("trans_hugepage_enable", 'no')
mb_enable = 'yes' == params.get("mb_enable", 'yes')
delay = int(params.get("delay_time", 10))
# Skip cases early
vm_names = []
if test_type == "contrast":
vm_names = params.get("vms").split()[:2]
if len(vm_names) < 2:
raise error.TestNAError("This test requires two VMs")
# confirm no VM running
allvms = virsh.dom_list('--name').stdout.strip()
if allvms != '':
raise error.TestNAError("one or more VMs are alive")
err_range = float(params.get("mem_error_range", 1.25))
else:
vm_names.append(params.get("main_vm"))
if test_type == "stress":
target_path = params.get("target_path", "/tmp/test.out")
elif test_type == "unixbench":
unixbench_control_file = params.get("unixbench_controle_file",
"unixbench5.control")
# backup orignal setting
shp_orig_num = utils_memory.get_num_huge_pages()
thp_orig_status = utils_memory.get_transparent_hugepage()
page_size = utils_memory.get_huge_page_size()
# mount/umount hugetlbfs
tlbfs_status = utils_misc.is_mounted("hugetlbfs", "/dev/hugepages",
"hugetlbfs")
if tlbfs_enable is True:
if tlbfs_status is not True:
utils_misc.mount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
else:
if tlbfs_status is True:
utils_misc.umount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
# set static hugepage
utils_memory.set_num_huge_pages(shp_num)
# enable/disable transparent hugepage
if thp_enable:
utils_memory.set_transparent_hugepage('always')
else:
utils_memory.set_transparent_hugepage('never')
# set/del memoryBacking tag
for vm_name in vm_names:
if mb_enable:
vm_xml.VMXML.set_memoryBacking_tag(vm_name)
else:
vm_xml.VMXML.del_memoryBacking_tag(vm_name)
utils_libvirtd.libvirtd_restart()
non_started_free = utils_memory.get_num_huge_pages_free()
vms = []
sessions = []
try:
for vm_name in vm_names:
# try to start vm and login
try:
vm = env.get_vm(vm_name)
vm.start()
except VMError, e:
if mb_enable and not tlbfs_enable:
# if hugetlbfs not be mounted,
# VM start with memoryBacking tag will fail
logging.debug(e)
else:
error_msg = "Test failed in positive case. error: %s\n" % e
raise error.TestFail(error_msg)
if vm.is_alive() is not True:
break
vms.append(vm)
# try to login and run some program
try:
session = vm.wait_for_login()
except (LoginError, ShellError), e:
error_msg = "Test failed in positive case.\n error: %s\n" % e
raise error.TestFail(error_msg)
sessions.append(session)
if test_type == "stress":
# prepare file for increasing stress
stress_path = prepare_c_file()
remote.scp_to_remote(vm.get_address(), 22,
'root', params.get('password'),
stress_path, "/tmp/")
# Try to install gcc on guest first
utils_package.package_install(["gcc"], session, 360)
# increasing workload
session.cmd("gcc %s -o %s" % (stress_path, target_path))
session.cmd("%s &" % target_path)
if test_type == "unixbench":
params["main_vm"] = vm_name
params["test_control_file"] = unixbench_control_file
control_path = os.path.join(test.virtdir, "control",
unixbench_control_file)
# unixbench test need 'patch' and 'perl' commands installed
utils_package.package_install(["patch", "perl"], session, 360)
command = utils_test.run_autotest(vm, session, control_path,
None, None, params,
copy_only=True)
session.cmd("%s &" % command, ignore_all_errors=True)
# wait for autotest running on vm
time.sleep(delay)
def _is_unixbench_running():
cmd = "ps -ef | grep perl | grep Run"
return not session.cmd_status(cmd)
if not utils_misc.wait_for(_is_unixbench_running, timeout=240):
raise error.TestNAError("Failed to run unixbench in guest,"
" please make sure some necessary"
" packages are installed in guest,"
" such as gcc, tar, bzip2")
logging.debug("Unixbench test is running in VM")
def run(test, params, env):
"""
Qemu numa stress test:
1) Boot up a guest and find the node it used
2) Try to allocate memory in that node
3) Run memory heavy stress inside guest
4) Check the memory use status of qemu process
5) Repeat step 2 ~ 4 several times
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
host_numa_node = utils_misc.NumaInfo()
if len(host_numa_node.online_nodes) < 2:
test.cancel("Host only has one NUMA node, skipping test...")
timeout = float(params.get("login_timeout", 240))
test_count = int(params.get("test_count", 4))
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session = vm.wait_for_login(timeout=timeout)
qemu_pid = vm.get_pid()
if test_count < len(host_numa_node.online_nodes):
test_count = len(host_numa_node.online_nodes)
tmpfs_size = params.get_numeric("tmpfs_size")
for node in host_numa_node.nodes:
node_mem = int(host_numa_node.read_from_node_meminfo(node, "MemTotal"))
if tmpfs_size == 0:
tmpfs_size = node_mem
tmpfs_path = params.get("tmpfs_path", "tmpfs_numa_test")
tmpfs_path = utils_misc.get_path(data_dir.get_tmp_dir(), tmpfs_path)
tmpfs_write_speed = get_tmpfs_write_speed()
dd_timeout = tmpfs_size / tmpfs_write_speed * 1.5
mount_fs_size = "size=%dK" % tmpfs_size
memory_file = utils_misc.get_path(tmpfs_path, "test")
dd_cmd = "dd if=/dev/urandom of=%s bs=1k count=%s" % (memory_file,
tmpfs_size)
utils_memory.drop_caches()
if utils_memory.freememtotal() < tmpfs_size:
test.cancel("Host does not have enough free memory to run the test, "
"skipping test...")
if not os.path.isdir(tmpfs_path):
os.mkdir(tmpfs_path)
test_mem = float(params.get("mem")) * float(params.get("mem_ratio", 0.8))
stress_args = "--cpu 4 --io 4 --vm 2 --vm-bytes %sM" % int(test_mem / 2)
most_used_node, memory_used = max_mem_map_node(host_numa_node, qemu_pid)
for test_round in range(test_count):
if os.path.exists(memory_file):
os.remove(memory_file)
utils_memory.drop_caches()
if utils_memory.freememtotal() < tmpfs_size:
test.error("Don't have enough memory to execute this "
"test after %s round" % test_round)
error_context.context("Executing stress test round: %s" % test_round,
logging.info)
numa_node_malloc = most_used_node
numa_dd_cmd = "numactl -m %s %s" % (numa_node_malloc, dd_cmd)
error_context.context(
"Try to allocate memory in node %s" % numa_node_malloc,
logging.info)
try:
utils_misc.mount("none", tmpfs_path, "tmpfs", perm=mount_fs_size)
funcatexit.register(env, params.get("type"), utils_misc.umount,
"none", tmpfs_path, "tmpfs")
process.system(numa_dd_cmd, timeout=dd_timeout, shell=True)
except Exception as error_msg:
if "No space" in str(error_msg):
pass
else:
test.fail("Can not allocate memory in node %s."
" Error message:%s" %
(numa_node_malloc, str(error_msg)))
error_context.context("Run memory heavy stress in guest", logging.info)
stress_test = utils_test.VMStress(vm,
"stress",
params,
stress_args=stress_args)
stress_test.load_stress_tool()
error_context.context("Get the qemu process memory use status",
logging.info)
node_after, memory_after = max_mem_map_node(host_numa_node, qemu_pid)
if node_after == most_used_node and memory_after >= memory_used:
test.fail("Memory still stick in node %s" % numa_node_malloc)
else:
most_used_node = node_after
memory_used = memory_after
stress_test.unload_stress()
stress_test.clean()
utils_misc.umount("none", tmpfs_path, "tmpfs")
funcatexit.unregister(env, params.get("type"), utils_misc.umount,
"none", tmpfs_path, "tmpfs")
session.cmd("sync; echo 3 > /proc/sys/vm/drop_caches")
utils_memory.drop_caches()
session.close()
def run(test, params, env):
"""
Test various options of virt-v2v.
"""
if utils_v2v.V2V_EXEC is None:
raise ValueError('Missing command: virt-v2v')
for v in params.itervalues():
if "V2V_EXAMPLE" in v:
raise exceptions.TestSkipError("Please set real value for %s" % v)
vm_name = params.get("main_vm", "EXAMPLE")
new_vm_name = params.get("new_vm_name")
input_mode = params.get("input_mode")
v2v_options = params.get("v2v_options", "")
hypervisor = params.get("hypervisor", "kvm")
remote_host = params.get("remote_host", "EXAMPLE")
vpx_dc = params.get("vpx_dc", "EXAMPLE")
esx_ip = params.get("esx_ip", "EXAMPLE")
output_mode = params.get("output_mode")
output_storage = params.get("output_storage", "default")
disk_img = params.get("input_disk_image", "")
nfs_storage = params.get("nfs_storage")
mnt_point = params.get("mount_point")
export_domain_uuid = params.get("export_domain_uuid", "")
fake_domain_uuid = params.get("fake_domain_uuid")
vdsm_image_uuid = params.get("vdsm_image_uuid")
vdsm_vol_uuid = params.get("vdsm_vol_uuid")
vdsm_vm_uuid = params.get("vdsm_vm_uuid")
vdsm_ovf_output = params.get("vdsm_ovf_output")
v2v_user = params.get("unprivileged_user", "")
v2v_timeout = int(params.get("v2v_timeout", 1200))
status_error = "yes" == params.get("status_error", "no")
su_cmd = "su - %s -c " % v2v_user
output_uri = params.get("oc_uri", "")
pool_name = params.get("pool_name", "v2v_test")
pool_type = params.get("pool_type", "dir")
pool_target = params.get("pool_target_path", "v2v_pool")
emulated_img = params.get("emulated_image_path", "v2v-emulated-img")
pvt = utlv.PoolVolumeTest(test, params)
new_v2v_user = False
restore_image_owner = False
address_cache = env.get('address_cache')
params['vmcheck_flag'] = False
checkpoint = params.get('checkpoint', '')
def create_pool():
"""
Create libvirt pool as the output storage
"""
if output_uri == "qemu:///session":
target_path = os.path.join("/home", v2v_user, pool_target)
cmd = su_cmd + "'mkdir %s'" % target_path
process.system(cmd, verbose=True)
cmd = su_cmd + "'virsh pool-create-as %s dir" % pool_name
cmd += " --target %s'" % target_path
process.system(cmd, verbose=True)
else:
pvt.pre_pool(pool_name, pool_type, pool_target, emulated_img)
def cleanup_pool():
"""
Clean up libvirt pool
"""
if output_uri == "qemu:///session":
cmd = su_cmd + "'virsh pool-destroy %s'" % pool_name
process.system(cmd, verbose=True)
target_path = os.path.join("/home", v2v_user, pool_target)
cmd = su_cmd + "'rm -rf %s'" % target_path
process.system(cmd, verbose=True)
else:
pvt.cleanup_pool(pool_name, pool_type, pool_target, emulated_img)
def get_all_uuids(output):
"""
Get export domain uuid, image uuid and vol uuid from command output.
"""
tmp_target = re.findall(r"qemu-img\sconvert\s.+\s'(\S+)'\n", output)
if len(tmp_target) < 1:
raise exceptions.TestError("Fail to find tmp target file name when"
" converting vm disk image")
targets = tmp_target[0].split('/')
return (targets[3], targets[5], targets[6])
def get_ovf_content(output):
"""
Find and read ovf file.
"""
export_domain_uuid, _, vol_uuid = get_all_uuids(output)
export_vm_dir = os.path.join(mnt_point, export_domain_uuid,
'master/vms')
ovf_content = ""
if os.path.isdir(export_vm_dir):
ovf_id = "ovf:id='%s'" % vol_uuid
ret = process.system_output("grep -R \"%s\" %s" %
(ovf_id, export_vm_dir))
ovf_file = ret.split(":")[0]
if os.path.isfile(ovf_file):
ovf_f = open(ovf_file, "r")
ovf_content = ovf_f.read()
ovf_f.close()
else:
logging.error("Can't find ovf file to read")
return ovf_content
def get_img_path(output):
"""
Get the full path of the converted image.
"""
img_name = vm_name + "-sda"
if output_mode == "libvirt":
img_path = virsh.vol_path(img_name, output_storage).stdout.strip()
elif output_mode == "local":
img_path = os.path.join(output_storage, img_name)
elif output_mode in ["rhev", "vdsm"]:
export_domain_uuid, image_uuid, vol_uuid = get_all_uuids(output)
img_path = os.path.join(mnt_point, export_domain_uuid, 'images',
image_uuid, vol_uuid)
return img_path
def check_vmtype(ovf, expected_vmtype):
"""
Verify vmtype in ovf file.
"""
if output_mode != "rhev":
return
if expected_vmtype == "server":
vmtype_int = 1
elif expected_vmtype == "desktop":
vmtype_int = 0
else:
return
if "<VmType>%s</VmType>" % vmtype_int in ovf:
logging.info("Find VmType=%s in ovf file",
expected_vmtype)
else:
raise exceptions.TestFail("VmType check failed")
def check_image(img_path, check_point, expected_value):
"""
Verify image file allocation mode and format
"""
if not img_path or not os.path.isfile(img_path):
raise exceptions.TestError("Image path: '%s' is invalid" % img_path)
img_info = utils_misc.get_image_info(img_path)
logging.debug("Image info: %s", img_info)
if check_point == "allocation":
if expected_value == "sparse":
if img_info['vsize'] > img_info['dsize']:
logging.info("%s is a sparse image", img_path)
else:
raise exceptions.TestFail("%s is not a sparse image" % img_path)
elif expected_value == "preallocated":
if img_info['vsize'] <= img_info['dsize']:
logging.info("%s is a preallocated image", img_path)
else:
raise exceptions.TestFail("%s is not a preallocated image"
% img_path)
if check_point == "format":
if expected_value == img_info['format']:
logging.info("%s format is %s", img_path, expected_value)
else:
raise exceptions.TestFail("%s format is not %s"
% (img_path, expected_value))
def check_new_name(output, expected_name):
"""
Verify guest name changed to the new name.
"""
found = False
if output_mode == "libvirt":
found = virsh.domain_exists(expected_name)
if output_mode == "local":
found = os.path.isfile(os.path.join(output_storage,
expected_name + "-sda"))
if output_mode in ["rhev", "vdsm"]:
ovf = get_ovf_content(output)
found = "<Name>%s</Name>" % expected_name in ovf
else:
return
if found:
logging.info("Guest name renamed when converting it")
else:
raise exceptions.TestFail("Rename guest failed")
def check_nocopy(output):
"""
Verify no image created if convert command use --no-copy option
"""
img_path = get_img_path(output)
if not os.path.isfile(img_path):
logging.info("No image created with --no-copy option")
else:
raise exceptions.TestFail("Find %s" % img_path)
def check_connection(output, expected_uri):
"""
Check output connection uri used when converting guest
"""
init_msg = "Initializing the target -o libvirt -oc %s" % expected_uri
if init_msg in output:
logging.info("Find message: %s", init_msg)
else:
raise exceptions.TestFail("Not find message: %s" % init_msg)
def check_result(cmd, result, status_error):
"""
Check virt-v2v command result
"""
utlv.check_exit_status(result, status_error)
output = result.stdout + result.stderr
if status_error:
if checkpoint == 'length_of_error':
log_lines = output.split('\n')
v2v_start = False
for line in log_lines:
if line.startswith('virt-v2v:'):
v2v_start = True
if line.startswith('libvirt:'):
v2v_start = False
if v2v_start and line > 72:
raise exceptions.TestFail('Error log longer than 72 '
'charactors: %s', line)
else:
error_map = {
'conflict_options': ['option used more than once'],
'xen_no_output_format': ['The input metadata did not define'
' the disk format']
}
if not utils_v2v.check_log(output, error_map[checkpoint]):
raise exceptions.TestFail('Not found error message %s' %
error_map[checkpoint])
else:
if output_mode == "rhev" and checkpoint != 'quiet':
ovf = get_ovf_content(output)
logging.debug("ovf content: %s", ovf)
if '--vmtype' in cmd:
expected_vmtype = re.findall(r"--vmtype\s(\w+)", cmd)[0]
check_vmtype(ovf, expected_vmtype)
if '-oa' in cmd and '--no-copy' not in cmd:
expected_mode = re.findall(r"-oa\s(\w+)", cmd)[0]
img_path = get_img_path(output)
check_image(img_path, "allocation", expected_mode)
if '-of' in cmd and '--no-copy' not in cmd and checkpoint != 'quiet':
expected_format = re.findall(r"-of\s(\w+)", cmd)[0]
img_path = get_img_path(output)
check_image(img_path, "format", expected_format)
if '-on' in cmd:
expected_name = re.findall(r"-on\s(\w+)", cmd)[0]
check_new_name(output, expected_name)
if '--no-copy' in cmd:
check_nocopy(output)
if '-oc' in cmd:
expected_uri = re.findall(r"-oc\s(\S+)", cmd)[0]
check_connection(output, expected_uri)
if output_mode == "rhev":
if not utils_v2v.import_vm_to_ovirt(params, address_cache):
raise exceptions.TestFail("Import VM failed")
else:
params['vmcheck_flag'] = True
if output_mode == "libvirt":
if "qemu:///session" not in v2v_options:
virsh.start(vm_name, debug=True, ignore_status=False)
if checkpoint == 'quiet':
if len(output.strip()) != 0:
raise exceptions.TestFail('Output is not empty in quiet mode')
backup_xml = None
vdsm_domain_dir, vdsm_image_dir, vdsm_vm_dir = ("", "", "")
try:
# Build input options
input_option = ""
if input_mode is None:
pass
elif input_mode == "libvirt":
uri_obj = utils_v2v.Uri(hypervisor)
ic_uri = uri_obj.get_uri(remote_host, vpx_dc, esx_ip)
input_option = "-i %s -ic %s %s" % (input_mode, ic_uri, vm_name)
# Build network&bridge option to avoid network error
v2v_options += " -b %s -n %s" % (params.get("output_bridge"),
params.get("output_network"))
elif input_mode == "disk":
input_option += "-i %s %s" % (input_mode, disk_img)
elif input_mode in ['libvirtxml', 'ova']:
raise exceptions.TestNAError("Unsupported input mode: %s" % input_mode)
else:
raise exceptions.TestError("Unknown input mode %s" % input_mode)
input_format = params.get("input_format")
input_allo_mode = params.get("input_allo_mode")
if input_format:
input_option += " -if %s" % input_format
if not status_error:
logging.info("Check image before convert")
check_image(disk_img, "format", input_format)
if input_allo_mode:
check_image(disk_img, "allocation", input_allo_mode)
# Build output options
output_option = ""
if output_mode:
output_option = "-o %s -os %s" % (output_mode, output_storage)
output_format = params.get("output_format")
if output_format:
output_option += " -of %s" % output_format
output_allo_mode = params.get("output_allo_mode")
if output_allo_mode:
output_option += " -oa %s" % output_allo_mode
# Build vdsm related options
if output_mode in ['vdsm', 'rhev']:
if not os.path.isdir(mnt_point):
os.mkdir(mnt_point)
if not utils_misc.mount(nfs_storage, mnt_point, "nfs"):
raise exceptions.TestError("Mount NFS Failed")
if output_mode == 'vdsm':
v2v_options += " --vdsm-image-uuid %s" % vdsm_image_uuid
v2v_options += " --vdsm-vol-uuid %s" % vdsm_vol_uuid
v2v_options += " --vdsm-vm-uuid %s" % vdsm_vm_uuid
v2v_options += " --vdsm-ovf-output %s" % vdsm_ovf_output
vdsm_domain_dir = os.path.join(mnt_point, fake_domain_uuid)
vdsm_image_dir = os.path.join(mnt_point, export_domain_uuid,
"images", vdsm_image_uuid)
vdsm_vm_dir = os.path.join(mnt_point, export_domain_uuid,
"master/vms", vdsm_vm_uuid)
# For vdsm_domain_dir, just create a dir to test BZ#1176591
os.mkdir(vdsm_domain_dir)
os.mkdir(vdsm_image_dir)
os.mkdir(vdsm_vm_dir)
# Output more messages except quiet mode
if checkpoint == 'quiet':
v2v_options += ' -q'
elif checkpoint == 'length_of_error':
pass
else:
v2v_options += " -v -x"
# Prepare for libvirt unprivileged user session connection
if "qemu:///session" in v2v_options:
try:
pwd.getpwnam(v2v_user)
except KeyError:
# create new user
process.system("useradd %s" % v2v_user, ignore_status=True)
new_v2v_user = True
user_info = pwd.getpwnam(v2v_user)
logging.info("Convert to qemu:///session by user '%s'", v2v_user)
if input_mode == "disk":
# Change the image owner and group
ori_owner = os.stat(disk_img).st_uid
ori_group = os.stat(disk_img).st_uid
os.chown(disk_img, user_info.pw_uid, user_info.pw_gid)
restore_image_owner = True
else:
raise exceptions.TestNAError("Only support convert local disk")
# Setup ssh-agent access to xen hypervisor
if hypervisor == 'xen':
os.environ['LIBGUESTFS_BACKEND'] = 'direct'
user = params.get("xen_host_user", "root")
passwd = params.get("xen_host_passwd", "redhat")
logging.info("set up ssh-agent access ")
ssh_key.setup_ssh_key(remote_host, user=user,
port=22, password=passwd)
utils_misc.add_identities_into_ssh_agent()
# If the input format is not define, we need to either define
# the original format in the source metadata(xml) or use '-of'
# to force the output format, see BZ#1141723 for detail.
if '-of' not in v2v_options and checkpoint != 'xen_no_output_format':
v2v_options += ' -of %s' % params.get("default_output_format",
"qcow2")
# Create password file for access to ESX hypervisor
if hypervisor == 'esx':
os.environ['LIBGUESTFS_BACKEND'] = 'direct'
vpx_passwd = params.get("vpx_passwd")
vpx_passwd_file = os.path.join(test.tmpdir, "vpx_passwd")
logging.info("Building ESX no password interactive verification.")
pwd_f = open(vpx_passwd_file, 'w')
pwd_f.write(vpx_passwd)
pwd_f.close()
output_option += " --password-file %s" % vpx_passwd_file
# Create libvirt dir pool
if output_mode == "libvirt":
create_pool()
# Running virt-v2v command
cmd = "%s %s %s %s" % (utils_v2v.V2V_EXEC, input_option,
output_option, v2v_options)
if v2v_user:
cmd = su_cmd + "'%s'" % cmd
cmd_result = process.run(cmd, timeout=v2v_timeout, verbose=True,
ignore_status=True)
if new_vm_name:
vm_name = new_vm_name
params['main_vm'] = new_vm_name
check_result(cmd, cmd_result, status_error)
finally:
if hypervisor == "xen":
process.run("ssh-agent -k")
if hypervisor == "esx":
process.run("rm -rf %s" % vpx_passwd_file)
for vdsm_dir in [vdsm_domain_dir, vdsm_image_dir, vdsm_vm_dir]:
if os.path.exists(vdsm_dir):
shutil.rmtree(vdsm_dir)
if os.path.exists(mnt_point):
utils_misc.umount(nfs_storage, mnt_point, "nfs")
os.rmdir(mnt_point)
if output_mode == "local":
image_name = vm_name + "-sda"
img_file = os.path.join(output_storage, image_name)
xml_file = img_file + ".xml"
for local_file in [img_file, xml_file]:
if os.path.exists(local_file):
os.remove(local_file)
if output_mode == "libvirt":
if "qemu:///session" in v2v_options:
cmd = su_cmd + "'virsh undefine %s'" % vm_name
process.system(cmd)
else:
virsh.remove_domain(vm_name)
cleanup_pool()
vmcheck_flag = params.get("vmcheck_flag")
if vmcheck_flag:
vmcheck = utils_v2v.VMCheck(test, params, env)
vmcheck.cleanup()
if new_v2v_user:
process.system("userdel -f %s" % v2v_user)
if restore_image_owner:
os.chown(disk_img, ori_owner, ori_group)
if backup_xml:
backup_xml.sync()
def run(test, params, env):
"""
Test various options of virt-v2v.
"""
if utils_v2v.V2V_EXEC is None:
raise ValueError('Missing command: virt-v2v')
for v in params.itervalues():
if "V2V_EXAMPLE" in v:
test.cancel("Please set real value for %s" % v)
vm_name = params.get("main_vm", "EXAMPLE")
new_vm_name = params.get("new_vm_name")
input_mode = params.get("input_mode")
v2v_options = params.get("v2v_options", "")
hypervisor = params.get("hypervisor", "kvm")
remote_host = params.get("remote_host", "EXAMPLE")
vpx_dc = params.get("vpx_dc", "EXAMPLE")
esx_ip = params.get("esx_ip", "EXAMPLE")
output_mode = params.get("output_mode")
output_storage = params.get("output_storage", "default")
disk_img = params.get("input_disk_image", "")
nfs_storage = params.get("storage")
no_root = 'yes' == params.get('no_root', 'no')
mnt_point = params.get("mnt_point")
export_domain_uuid = params.get("export_domain_uuid", "")
fake_domain_uuid = params.get("fake_domain_uuid")
vdsm_image_uuid = params.get("vdsm_image_uuid")
vdsm_vol_uuid = params.get("vdsm_vol_uuid")
vdsm_vm_uuid = params.get("vdsm_vm_uuid")
vdsm_ovf_output = params.get("vdsm_ovf_output")
v2v_user = params.get("unprivileged_user", "")
v2v_timeout = int(params.get("v2v_timeout", 1200))
status_error = "yes" == params.get("status_error", "no")
su_cmd = "su - %s -c " % v2v_user
output_uri = params.get("oc_uri", "")
pool_name = params.get("pool_name", "v2v_test")
pool_type = params.get("pool_type", "dir")
pool_target = params.get("pool_target_path", "v2v_pool")
emulated_img = params.get("emulated_image_path", "v2v-emulated-img")
pvt = utlv.PoolVolumeTest(test, params)
new_v2v_user = False
address_cache = env.get('address_cache')
params['vmcheck_flag'] = False
checkpoint = params.get('checkpoint', '')
def create_pool(user_pool=False, pool_name=pool_name, pool_target=pool_target):
"""
Create libvirt pool as the output storage
"""
if output_uri == "qemu:///session" or user_pool:
target_path = os.path.join("/home", v2v_user, pool_target)
cmd = su_cmd + "'mkdir %s'" % target_path
process.system(cmd, verbose=True)
cmd = su_cmd + "'virsh pool-create-as %s dir" % pool_name
cmd += " --target %s'" % target_path
process.system(cmd, verbose=True)
else:
pvt.pre_pool(pool_name, pool_type, pool_target, emulated_img)
def cleanup_pool(user_pool=False, pool_name=pool_name, pool_target=pool_target):
"""
Clean up libvirt pool
"""
if output_uri == "qemu:///session" or user_pool:
cmd = su_cmd + "'virsh pool-destroy %s'" % pool_name
process.system(cmd, verbose=True)
target_path = os.path.join("/home", v2v_user, pool_target)
cmd = su_cmd + "'rm -rf %s'" % target_path
process.system(cmd, verbose=True)
else:
pvt.cleanup_pool(pool_name, pool_type, pool_target, emulated_img)
def get_all_uuids(output):
"""
Get export domain uuid, image uuid and vol uuid from command output.
"""
tmp_target = re.findall(r"qemu-img\s'convert'\s.+\s'(\S+)'\n", output)
if len(tmp_target) < 1:
test.error("Fail to find tmp target file name when converting vm"
" disk image")
targets = tmp_target[0].split('/')
return (targets[3], targets[5], targets[6])
def get_ovf_content(output):
"""
Find and read ovf file.
"""
export_domain_uuid, _, vol_uuid = get_all_uuids(output)
export_vm_dir = os.path.join(mnt_point, export_domain_uuid,
'master/vms')
ovf_content = ""
if os.path.isdir(export_vm_dir):
ovf_id = "ovf:id='%s'" % vol_uuid
ret = process.system_output("grep -R \"%s\" %s" %
(ovf_id, export_vm_dir))
ovf_file = ret.split(":")[0]
if os.path.isfile(ovf_file):
ovf_f = open(ovf_file, "r")
ovf_content = ovf_f.read()
ovf_f.close()
else:
logging.error("Can't find ovf file to read")
return ovf_content
def get_img_path(output):
"""
Get the full path of the converted image.
"""
img_name = vm_name + "-sda"
if output_mode == "libvirt":
img_path = virsh.vol_path(img_name, output_storage).stdout.strip()
elif output_mode == "local":
img_path = os.path.join(output_storage, img_name)
elif output_mode in ["rhev", "vdsm"]:
export_domain_uuid, image_uuid, vol_uuid = get_all_uuids(output)
img_path = os.path.join(mnt_point, export_domain_uuid, 'images',
image_uuid, vol_uuid)
return img_path
def check_vmtype(ovf, expected_vmtype):
"""
Verify vmtype in ovf file.
"""
if output_mode != "rhev":
return
if expected_vmtype == "server":
vmtype_int = 1
elif expected_vmtype == "desktop":
vmtype_int = 0
else:
return
if "<VmType>%s</VmType>" % vmtype_int in ovf:
logging.info("Find VmType=%s in ovf file",
expected_vmtype)
else:
test.fail("VmType check failed")
def check_image(img_path, check_point, expected_value):
"""
Verify image file allocation mode and format
"""
if not img_path or not os.path.isfile(img_path):
test.error("Image path: '%s' is invalid" % img_path)
img_info = utils_misc.get_image_info(img_path)
logging.debug("Image info: %s", img_info)
if check_point == "allocation":
if expected_value == "sparse":
if img_info['vsize'] > img_info['dsize']:
logging.info("%s is a sparse image", img_path)
else:
test.fail("%s is not a sparse image" % img_path)
elif expected_value == "preallocated":
if img_info['vsize'] <= img_info['dsize']:
logging.info("%s is a preallocated image", img_path)
else:
test.fail("%s is not a preallocated image" % img_path)
if check_point == "format":
if expected_value == img_info['format']:
logging.info("%s format is %s", img_path, expected_value)
else:
test.fail("%s format is not %s" % (img_path, expected_value))
def check_new_name(output, expected_name):
"""
Verify guest name changed to the new name.
"""
found = False
if output_mode == "libvirt":
found = virsh.domain_exists(expected_name)
if output_mode == "local":
found = os.path.isfile(os.path.join(output_storage,
expected_name + "-sda"))
if output_mode in ["rhev", "vdsm"]:
ovf = get_ovf_content(output)
found = "<Name>%s</Name>" % expected_name in ovf
else:
return
if found:
logging.info("Guest name renamed when converting it")
else:
test.fail("Rename guest failed")
def check_nocopy(output):
"""
Verify no image created if convert command use --no-copy option
"""
img_path = get_img_path(output)
if not os.path.isfile(img_path):
logging.info("No image created with --no-copy option")
else:
test.fail("Find %s" % img_path)
def check_connection(output, expected_uri):
"""
Check output connection uri used when converting guest
"""
init_msg = "Initializing the target -o libvirt -oc %s" % expected_uri
if init_msg in output:
logging.info("Find message: %s", init_msg)
else:
test.fail("Not find message: %s" % init_msg)
def check_ovf_snapshot_id(ovf_content):
"""
Check if snapshot id in ovf file consists of '0's
"""
search = re.search("ovf:vm_snapshot_id='(.*?)'", ovf_content)
if search:
snapshot_id = search.group(1)
logging.debug('vm_snapshot_id = %s', snapshot_id)
if snapshot_id.count('0') >= 32:
test.fail('vm_snapshot_id consists with "0"')
else:
test.fail('Fail to find snapshot_id')
def check_source(output):
"""
Check if --print-source option print the correct info
"""
# Parse source info
source = output.split('\n')[2:]
for i in range(len(source)):
if source[i].startswith('\t'):
source[i-1] += source[i]
source[i] = ''
source_strip = [x.strip() for x in source if x.strip()]
source_info = {}
for line in source_strip:
source_info[line.split(':')[0]] = line.split(':', 1)[1].strip()
logging.debug('Source info to check: %s', source_info)
checklist = ['nr vCPUs', 'hypervisor type', 'source name', 'memory',
'display', 'CPU features', 'disks', 'NICs']
for key in checklist:
if key not in source_info:
test.fail('%s info missing' % key)
# Check single values
fail = []
xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
check_map = {}
check_map['nr vCPUs'] = xml.vcpu
check_map['hypervisor type'] = xml.hypervisor_type
check_map['source name'] = xml.vm_name
check_map['memory'] = str(int(xml.max_mem) * 1024) + ' (bytes)'
check_map['display'] = xml.get_graphics_devices()[0].type_name
logging.info('KEY:\tSOURCE<-> XML')
for key in check_map:
logging.info('%-15s:%18s <-> %s', key, source_info[key], check_map[key])
if source_info[key] != str(check_map[key]):
fail.append(key)
# Check disk info
disk = xml.get_disk_all().values()[0]
bus, type = disk.find('target').get('bus'), disk.find('driver').get('type')
path = disk.find('source').get('file')
disks_info = "%s (%s) [%s]" % (path, type, bus)
source_disks = source_info['disks'].split()
source_disks_path = source_disks[0]
source_disks_type = source_disks[1].strip('()')
source_disks_bus = source_disks[2].strip('[]')
logging.info('disks:%s<->%s', source_info['disks'], disks_info)
if source_disks_path != path or source_disks_type != type or bus not in source_disks_bus:
fail.append('disks')
# Check nic info
nic = xml.get_iface_all().values()[0]
type = nic.get('type')
mac = nic.find('mac').get('address')
nic_source = nic.find('source')
name = nic_source.get(type)
nic_info = '%s "%s" mac: %s' % (type, name, mac)
logging.info('NICs:%s<->%s', source_info['NICs'], nic_info)
if nic_info.lower() not in source_info['NICs'].lower():
fail.append('NICs')
# Check cpu features
feature_list = xml.features.get_feature_list()
logging.info('CPU features:%s<->%s', source_info['CPU features'], feature_list)
if sorted(source_info['CPU features'].split(',')) != sorted(feature_list):
fail.append('CPU features')
if fail:
test.fail('Source info not correct for: %s' % fail)
def check_man_page(in_man, not_in_man):
"""
Check if content of man page or help info meets expectation
"""
man_page = process.run('man virt-v2v', verbose=False).stdout.strip()
if in_man:
logging.info('Checking man page of virt-v2v for "%s"', in_man)
if in_man not in man_page:
test.fail('"%s" not in man page' % in_man)
if not_in_man:
logging.info('Checking man page of virt-v2v for "%s"', not_in_man)
if not_in_man in man_page:
test.fail('"%s" not removed from man page' % not_in_man)
def check_result(cmd, result, status_error):
"""
Check virt-v2v command result
"""
utlv.check_exit_status(result, status_error)
output = result.stdout + result.stderr
if status_error:
if checkpoint == 'length_of_error':
log_lines = output.split('\n')
v2v_start = False
for line in log_lines:
if line.startswith('virt-v2v:'):
v2v_start = True
if line.startswith('libvirt:'):
v2v_start = False
if v2v_start and len(line) > 72:
test.fail('Error log longer than 72 charactors: %s' %
line)
if checkpoint == 'disk_not_exist':
vol_list = virsh.vol_list(pool_name)
logging.info(vol_list)
if vm_name in vol_list.stdout:
test.fail('Disk exists for vm %s' % vm_name)
else:
if output_mode == "rhev" and checkpoint != 'quiet':
ovf = get_ovf_content(output)
logging.debug("ovf content: %s", ovf)
check_ovf_snapshot_id(ovf)
if '--vmtype' in cmd:
expected_vmtype = re.findall(r"--vmtype\s(\w+)", cmd)[0]
check_vmtype(ovf, expected_vmtype)
if '-oa' in cmd and '--no-copy' not in cmd:
expected_mode = re.findall(r"-oa\s(\w+)", cmd)[0]
img_path = get_img_path(output)
def check_alloc():
try:
check_image(img_path, "allocation", expected_mode)
return True
except exceptions.TestFail:
pass
if not utils_misc.wait_for(check_alloc, timeout=600, step=10.0):
test.fail('Allocation check failed.')
if '-of' in cmd and '--no-copy' not in cmd and checkpoint != 'quiet':
expected_format = re.findall(r"-of\s(\w+)", cmd)[0]
img_path = get_img_path(output)
check_image(img_path, "format", expected_format)
if '-on' in cmd:
expected_name = re.findall(r"-on\s(\w+)", cmd)[0]
check_new_name(output, expected_name)
if '--no-copy' in cmd:
check_nocopy(output)
if '-oc' in cmd:
expected_uri = re.findall(r"-oc\s(\S+)", cmd)[0]
check_connection(output, expected_uri)
if output_mode == "rhev":
if not utils_v2v.import_vm_to_ovirt(params, address_cache):
test.fail("Import VM failed")
else:
params['vmcheck_flag'] = True
if output_mode == "libvirt":
if "qemu:///session" not in v2v_options and not no_root:
virsh.start(vm_name, debug=True, ignore_status=False)
if checkpoint == 'vmx':
vmchecker = VMChecker(test, params, env)
params['vmchecker'] = vmchecker
params['vmcheck_flag'] = True
ret = vmchecker.run()
if len(ret) == 0:
logging.info("All common checkpoints passed")
if checkpoint == 'quiet':
if len(output.strip()) != 0:
test.fail('Output is not empty in quiet mode')
if checkpoint == 'dependency':
if 'libguestfs-winsupport' not in output:
test.fail('libguestfs-winsupport not in dependency')
if 'VMF' not in output:
test.fail('OVMF/AAVMF not in dependency')
if 'qemu-kvm-rhev' in output:
test.fail('qemu-kvm-rhev is in dependency')
if 'libX11' in output:
test.fail('libX11 is in dependency')
win_img = params.get('win_image')
command = 'guestfish -a %s -i'
if process.run(command % win_img, ignore_status=True).exit_status == 0:
test.fail('Command "%s" success' % command % win_img)
if checkpoint == 'no_dcpath':
if '--dcpath' in output:
test.fail('"--dcpath" is not removed')
if checkpoint == 'debug_overlays':
search = re.search('Overlay saved as(.*)', output)
if not search:
test.fail('Not find log of saving overlays')
overlay_path = search.group(1).strip()
logging.debug('Overlay file location: %s' % overlay_path)
if os.path.isfile(overlay_path):
logging.info('Found overlay file: %s' % overlay_path)
else:
test.fail('Overlay file not saved')
if checkpoint.startswith('empty_nic_source'):
target_str = '%s "eth0" mac: %s' % (params[checkpoint][0], params[checkpoint][1])
logging.info('Expect log: %s', target_str)
if target_str not in result.stdout.lower():
test.fail('Expect log not found: %s' % target_str)
if checkpoint == 'print_source':
check_source(result.stdout)
if checkpoint == 'machine_readable':
if os.path.exists(params.get('example_file', '')):
expect_output = open(params['example_file']).read().strip()
logging.debug(expect_output)
if expect_output != result.stdout.strip():
test.fail('machine readable content not correct')
else:
test.error('No content to compare with')
if checkpoint == 'compress':
img_path = get_img_path(output)
logging.info('Image path: %s', img_path)
disk_check = process.run('qemu-img check %s' % img_path).stdout
logging.info(disk_check)
compress_info = disk_check.split(',')[-1].split('%')[0].strip()
compress_rate = float(compress_info)
logging.info('%s%% compressed', compress_rate)
if compress_rate < 0.1:
test.fail('Disk image NOT compressed')
if checkpoint == 'tail_log':
messages = params['tail'].get_output()
logging.info('Content of /var/log/messages during conversion:')
logging.info(messages)
msg_content = params['msg_content']
if msg_content in messages:
test.fail('Found "%s" in /var/log/messages' % msg_content)
log_check = utils_v2v.check_log(params, output)
if log_check:
test.fail(log_check)
check_man_page(params.get('in_man'), params.get('not_in_man'))
backup_xml = None
vdsm_domain_dir, vdsm_image_dir, vdsm_vm_dir = ("", "", "")
try:
if checkpoint.startswith('empty_nic_source'):
xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
iface = xml.get_devices('interface')[0]
disks = xml.get_devices('disk')
del iface.source
iface.type_name = checkpoint.split('_')[-1]
iface.source = {iface.type_name: ''}
params[checkpoint] = [iface.type_name, iface.mac_address]
logging.debug(iface.source)
devices = vm_xml.VMXMLDevices()
devices.extend(disks)
devices.append(iface)
xml.set_devices(devices)
logging.info(xml.xmltreefile)
params['input_xml'] = xml.xmltreefile.name
# Build input options
input_option = ""
if input_mode is None:
pass
elif input_mode == "libvirt":
uri_obj = utils_v2v.Uri(hypervisor)
ic_uri = uri_obj.get_uri(remote_host, vpx_dc, esx_ip)
if checkpoint == 'with_ic':
ic_uri = 'qemu:///session'
input_option = "-i %s -ic %s %s" % (input_mode, ic_uri, vm_name)
if checkpoint == 'without_ic':
input_option = '-i %s %s' % (input_mode, vm_name)
# Build network&bridge option to avoid network error
v2v_options += " -b %s -n %s" % (params.get("output_bridge"),
params.get("output_network"))
elif input_mode == "disk":
input_option += "-i %s %s" % (input_mode, disk_img)
elif input_mode == 'libvirtxml':
input_xml = params.get('input_xml')
input_option += '-i %s %s' % (input_mode, input_xml)
elif input_mode in ['ova']:
test.cancel("Unsupported input mode: %s" % input_mode)
else:
test.error("Unknown input mode %s" % input_mode)
input_format = params.get("input_format", "")
input_allo_mode = params.get("input_allo_mode")
if input_format:
input_option += " -if %s" % input_format
if not status_error:
logging.info("Check image before convert")
check_image(disk_img, "format", input_format)
if input_allo_mode:
check_image(disk_img, "allocation", input_allo_mode)
# Build output options
output_option = ""
if output_mode:
output_option = "-o %s -os %s" % (output_mode, output_storage)
if checkpoint == 'rhv':
output_option = output_option.replace('rhev', 'rhv')
output_format = params.get("output_format")
if output_format and output_format != input_format:
output_option += " -of %s" % output_format
output_allo_mode = params.get("output_allo_mode")
if output_allo_mode:
output_option += " -oa %s" % output_allo_mode
# Build vdsm related options
if output_mode in ['vdsm', 'rhev']:
if not os.path.isdir(mnt_point):
os.mkdir(mnt_point)
if not utils_misc.mount(nfs_storage, mnt_point, "nfs"):
test.error("Mount NFS Failed")
if output_mode == 'vdsm':
v2v_options += " --vdsm-image-uuid %s" % vdsm_image_uuid
v2v_options += " --vdsm-vol-uuid %s" % vdsm_vol_uuid
v2v_options += " --vdsm-vm-uuid %s" % vdsm_vm_uuid
v2v_options += " --vdsm-ovf-output %s" % vdsm_ovf_output
vdsm_domain_dir = os.path.join(mnt_point, fake_domain_uuid)
vdsm_image_dir = os.path.join(mnt_point, export_domain_uuid,
"images", vdsm_image_uuid)
vdsm_vm_dir = os.path.join(mnt_point, export_domain_uuid,
"master/vms", vdsm_vm_uuid)
# For vdsm_domain_dir, just create a dir to test BZ#1176591
os.makedirs(vdsm_domain_dir)
os.makedirs(vdsm_image_dir)
os.makedirs(vdsm_vm_dir)
# Output more messages except quiet mode
if checkpoint == 'quiet':
v2v_options += ' -q'
elif checkpoint not in ['length_of_error', 'empty_nic_source_network',
'empty_nic_source_bridge']:
v2v_options += " -v -x"
# Prepare for libvirt unprivileged user session connection
if "qemu:///session" in v2v_options or no_root:
try:
pwd.getpwnam(v2v_user)
except KeyError:
# create new user
process.system("useradd %s" % v2v_user, ignore_status=True)
new_v2v_user = True
user_info = pwd.getpwnam(v2v_user)
logging.info("Convert to qemu:///session by user '%s'", v2v_user)
if input_mode == "disk":
# Copy image from souce and change the image owner and group
disk_path = os.path.join(data_dir.get_tmp_dir(), os.path.basename(disk_img))
logging.info('Copy image file %s to %s', disk_img, disk_path)
shutil.copyfile(disk_img, disk_path)
input_option = string.replace(input_option, disk_img, disk_path)
os.chown(disk_path, user_info.pw_uid, user_info.pw_gid)
elif not no_root:
test.cancel("Only support convert local disk")
# Setup ssh-agent access to xen hypervisor
if hypervisor == 'xen':
user = params.get("xen_host_user", "root")
passwd = params.get("xen_host_passwd", "redhat")
logging.info("set up ssh-agent access ")
ssh_key.setup_ssh_key(remote_host, user=user,
port=22, password=passwd)
utils_misc.add_identities_into_ssh_agent()
# Check if xen guest exists
uri = utils_v2v.Uri(hypervisor).get_uri(remote_host)
if not virsh.domain_exists(vm_name, uri=uri):
logging.error('VM %s not exists', vm_name)
# If the input format is not define, we need to either define
# the original format in the source metadata(xml) or use '-of'
# to force the output format, see BZ#1141723 for detail.
if '-of' not in v2v_options and checkpoint != 'xen_no_output_format':
v2v_options += ' -of %s' % params.get("default_output_format",
"qcow2")
# Create password file for access to ESX hypervisor
if hypervisor == 'esx':
vpx_passwd = params.get("vpx_password")
vpx_passwd_file = os.path.join(test.tmpdir, "vpx_passwd")
logging.info("Building ESX no password interactive verification.")
pwd_f = open(vpx_passwd_file, 'w')
pwd_f.write(vpx_passwd)
pwd_f.close()
output_option += " --password-file %s" % vpx_passwd_file
# Create libvirt dir pool
if output_mode == "libvirt":
create_pool()
# Work around till bug fixed
os.environ['LIBGUESTFS_BACKEND'] = 'direct'
if checkpoint in ['with_ic', 'without_ic']:
new_v2v_user = True
v2v_options += ' -on %s' % new_vm_name
create_pool(user_pool=True, pool_name='src_pool', pool_target='v2v_src_pool')
create_pool(user_pool=True)
logging.debug(virsh.pool_list(uri='qemu:///session'))
sh_install_vm = params.get('sh_install_vm')
if not sh_install_vm:
test.error('Source vm installing script missing')
with open(sh_install_vm) as fh:
cmd_install_vm = fh.read().strip()
process.run('su - %s -c "%s"' % (v2v_user, cmd_install_vm),
timeout=10, shell=True)
if checkpoint == 'vmx':
mount_point = params.get('mount_point')
if not os.path.isdir(mount_point):
os.mkdir(mount_point)
nfs_vmx = params.get('nfs_vmx')
if not utils_misc.mount(nfs_vmx, mount_point, 'nfs', verbose=True):
test.error('Mount nfs for vmx failed')
vmx = params.get('vmx')
input_option = '-i vmx %s' % vmx
v2v_options += " -b %s -n %s" % (params.get("output_bridge"),
params.get("output_network"))
# Running virt-v2v command
cmd = "%s %s %s %s" % (utils_v2v.V2V_EXEC, input_option,
output_option, v2v_options)
if v2v_user:
cmd = su_cmd + "'%s'" % cmd
if params.get('cmd_free') == 'yes':
cmd = params.get('check_command')
# Set timeout to kill v2v process before conversion succeed
if checkpoint == 'disk_not_exist':
v2v_timeout = 30
# Get tail content of /var/log/messages
if checkpoint == 'tail_log':
params['tail_log'] = os.path.join(data_dir.get_tmp_dir(), 'tail_log')
params['tail'] = aexpect.Tail(
command='tail -f /var/log/messages',
output_func=utils_misc.log_line,
output_params=(params['tail_log'],)
)
cmd_result = process.run(cmd, timeout=v2v_timeout, verbose=True,
ignore_status=True)
if new_vm_name:
vm_name = new_vm_name
params['main_vm'] = new_vm_name
check_result(cmd, cmd_result, status_error)
finally:
if hypervisor == "xen":
process.run("ssh-agent -k")
if hypervisor == "esx":
process.run("rm -rf %s" % vpx_passwd_file)
for vdsm_dir in [vdsm_domain_dir, vdsm_image_dir, vdsm_vm_dir]:
if os.path.exists(vdsm_dir):
shutil.rmtree(vdsm_dir)
if os.path.exists(mnt_point):
utils_misc.umount(nfs_storage, mnt_point, "nfs")
os.rmdir(mnt_point)
if output_mode == "local":
image_name = vm_name + "-sda"
img_file = os.path.join(output_storage, image_name)
xml_file = img_file + ".xml"
for local_file in [img_file, xml_file]:
if os.path.exists(local_file):
os.remove(local_file)
if output_mode == "libvirt":
if "qemu:///session" in v2v_options or no_root:
cmd = su_cmd + "'virsh undefine %s'" % vm_name
try:
process.system(cmd)
except:
logging.error('Undefine "%s" failed', vm_name)
if no_root:
cleanup_pool(user_pool=True, pool_name='src_pool', pool_target='v2v_src_pool')
cleanup_pool(user_pool=True)
else:
virsh.remove_domain(vm_name)
cleanup_pool()
vmcheck_flag = params.get("vmcheck_flag")
if vmcheck_flag:
vmcheck = utils_v2v.VMCheck(test, params, env)
vmcheck.cleanup()
if new_v2v_user:
process.system("userdel -f %s" % v2v_user)
if backup_xml:
backup_xml.sync()
if checkpoint == 'vmx':
utils_misc.umount(params['nfs_vmx'], params['mount_point'], 'nfs')
os.rmdir(params['mount_point'])
def run(test, params, env):
"""
Convert specific esx guest
"""
V2V_UNSUPPORT_RHEV_APT_VER = "[virt-v2v-1.43.3-4.el9,)"
for v in list(params.values()):
if "V2V_EXAMPLE" in v:
test.cancel("Please set real value for %s" % v)
if utils_v2v.V2V_EXEC is None:
raise ValueError('Missing command: virt-v2v')
enable_legacy_cp = params.get("enable_legacy_crypto_policies",
'no') == 'yes'
version_requried = params.get("version_requried")
unprivileged_user = params_get(params, 'unprivileged_user')
vpx_hostname = params.get('vpx_hostname')
vpx_passwd = params.get("vpx_password")
esxi_host = esx_ip = params.get('esx_hostname')
vpx_dc = params.get('vpx_dc')
vm_name = params.get('main_vm')
output_mode = params.get('output_mode')
pool_name = params.get('pool_name', 'v2v_test')
pool_type = params.get('pool_type', 'dir')
pool_target = params.get('pool_target_path', 'v2v_pool')
pvt = libvirt.PoolVolumeTest(test, params)
v2v_timeout = int(params.get('v2v_timeout', 1200))
v2v_cmd_timeout = int(params.get('v2v_cmd_timeout', 18000))
v2v_opts = '-v -x' if params.get('v2v_debug', 'on') in ['on', 'force_on'
] else ''
if params.get("v2v_opts"):
# Add a blank by force
v2v_opts += ' ' + params.get("v2v_opts")
status_error = 'yes' == params.get('status_error', 'no')
address_cache = env.get('address_cache')
checkpoint = params.get('checkpoint', '').split(',')
skip_vm_check = params.get('skip_vm_check', 'no')
skip_reason = params.get('skip_reason')
error_list = []
remote_host = vpx_hostname
# For VDDK
input_transport = params.get("input_transport")
vddk_libdir = params.get('vddk_libdir')
# nfs mount source
vddk_libdir_src = params.get('vddk_libdir_src')
vddk_thumbprint = params.get('vddk_thumbprint')
src_uri_type = params.get('src_uri_type')
esxi_password = params.get('esxi_password')
json_disk_pattern = params.get('json_disk_pattern')
# For construct rhv-upload option in v2v cmd
output_method = params.get("output_method")
rhv_upload_opts = params.get("rhv_upload_opts")
storage_name = params.get('storage_name')
os_pool = os_storage = params.get('output_storage', 'default')
# for get ca.crt file from ovirt engine
rhv_passwd = params.get("rhv_upload_passwd")
rhv_passwd_file = params.get("rhv_upload_passwd_file")
ovirt_engine_passwd = params.get("ovirt_engine_password")
ovirt_hostname = params.get("ovirt_engine_url").split(
'/')[2] if params.get("ovirt_engine_url") else None
ovirt_ca_file_path = params.get("ovirt_ca_file_path")
local_ca_file_path = params.get("local_ca_file_path")
os_version = params.get('os_version')
os_type = params.get('os_type')
virtio_win_path = params.get('virtio_win_path')
# qemu-guest-agent path in virtio-win or rhv-guest-tools-iso
qa_path = params.get('qa_path')
# download url of qemu-guest-agent
qa_url = params.get('qa_url')
v2v_sasl = None
# default values for v2v_cmd
auto_clean = True
cmd_only = False
cmd_has_ip = 'yes' == params.get('cmd_has_ip', 'yes')
interaction_run = 'yes' == params.get('interaction_run', 'no')
def log_fail(msg):
"""
Log error and update error list
"""
logging.error(msg)
error_list.append(msg)
def check_vmtools(vmcheck, check):
"""
Check whether vmware tools packages have been removed,
or vmware-tools service has stopped
:param vmcheck: VMCheck object for vm checking
:param check: Checkpoint of different cases
:return: None
"""
if "service" not in check:
logging.info('Check if packages been removed')
pkgs = vmcheck.session.cmd('rpm -qa').strip()
removed_pkgs = params.get('removed_pkgs').strip().split(',')
if not removed_pkgs:
test.error('Missing param "removed_pkgs"')
for pkg in removed_pkgs:
if pkg in pkgs:
log_fail('Package "%s" not removed' % pkg)
else:
logging.info('Check if service stopped')
vmtools_service = params.get('service_name')
status = utils_misc.get_guest_service_status(
vmcheck.session, vmtools_service)
logging.info('Service %s status: %s', vmtools_service, status)
if status != 'inactive':
log_fail('Service "%s" is not stopped' % vmtools_service)
def check_modprobe(vmcheck):
"""
Check whether content of /etc/modprobe.conf meets expectation
"""
content = vmcheck.session.cmd('cat /etc/modprobe.conf').strip()
logging.debug(content)
cfg_content = params.get('cfg_content')
if not cfg_content:
test.error('Missing content for search')
logging.info('Search "%s" in /etc/modprobe.conf', cfg_content)
pattern = r'\s+'.join(cfg_content.split())
if not re.search(pattern, content):
log_fail('Not found "%s"' % cfg_content)
def check_device_map(vmcheck):
"""
Check if the content of device.map meets expectation.
"""
logging.info(vmcheck.session.cmd('fdisk -l').strip())
device_map = params.get('device_map_path')
content = vmcheck.session.cmd('cat %s' % device_map)
logging.debug('Content of device.map:\n%s', content)
logging.info('Found device: %d', content.count('/dev/'))
logging.info('Found virtio device: %d', content.count('/dev/vd'))
if content.count('/dev/') != content.count('/dev/vd'):
log_fail('Content of device.map not correct')
else:
logging.info('device.map has been remaped to "/dev/vd*"')
def check_resume_swap(vmcheck):
"""
Check the content of grub files meet expectation.
"""
if os_version == 'rhel7':
chkfiles = [
'/etc/default/grub', '/boot/grub2/grub.cfg', '/etc/grub2.cfg'
]
if os_version == 'rhel6':
chkfiles = ['/boot/grub/grub.conf', '/etc/grub.conf']
for file_i in chkfiles:
status, content = vmcheck.run_cmd('cat %s' % file_i)
if status != 0:
log_fail('%s does not exist' % file_i)
resume_dev_count = content.count('resume=/dev/')
if resume_dev_count == 0 or resume_dev_count != content.count(
'resume=/dev/vd'):
reason = 'Maybe the VM\'s swap pariton is lvm'
log_fail('Content of %s is not correct or %s' %
(file_i, reason))
content = vmcheck.session.cmd('cat /proc/cmdline')
logging.debug('Content of /proc/cmdline:\n%s', content)
if 'resume=/dev/vd' not in content:
log_fail('Content of /proc/cmdline is not correct')
def check_rhev_file_exist(vmcheck):
"""
Check if rhev files exist
"""
file_path = {
'rhev-apt.exe': r'C:\rhev-apt.exe',
'rhsrvany.exe':
r'"C:\Program Files\Guestfs\Firstboot\rhsrvany.exe"'
}
# rhev-apt.ext is removed on rhel9
if utils_v2v.multiple_versions_compare(V2V_UNSUPPORT_RHEV_APT_VER):
file_path.pop('rhev-apt.exe')
for key in file_path:
status = vmcheck.session.cmd_status('dir %s' % file_path[key])
if status == 0:
logging.info('%s exists' % key)
else:
log_fail('%s does not exist after convert to rhv' % key)
def check_file_architecture(vmcheck):
"""
Check the 3rd party module info
:param vmcheck: VMCheck object for vm checking
"""
content = vmcheck.session.cmd('uname -r').strip()
status = vmcheck.session.cmd_status(
'rpm -qf /lib/modules/%s/fileaccess/fileaccess_mod.ko ' % content)
if status == 0:
log_fail('3rd party module info is not correct')
else:
logging.info(
'file /lib/modules/%s/fileaccess/fileaccess_mod.ko is not owned by any package'
% content)
def check_windows_signature(vmcheck, full_name):
"""
Check signature of a file in windows VM
:param vmcheck: VMCheck object for vm checking
:param full_name: a file's full path name
"""
logging.info(
'powershell or signtool needs to be installed in guest first')
cmds = [('powershell "Get-AuthenticodeSignature %s | format-list"' %
full_name, r'SignerCertificate.*?Not After](.*?)\[Thumbprint',
'%m/%d/%Y %I:%M:%S %p'),
('signtool verify /v %s' % full_name,
r'Issued to: Red Hat.*?Expires:(.*?)SHA1 hash', '')]
for cmd, ptn, fmt in cmds:
_, output = vmcheck.run_cmd(cmd)
if re.search(ptn, output, re.S):
expire_time = re.search(ptn, output, re.S).group(1).strip()
if fmt:
expire_time = time.strptime(expire_time, fmt)
else:
expire_time = time.strptime(expire_time)
if time.time() > time.mktime(expire_time):
test.fail("Signature of '%s' has expired" % full_name)
return
# Get here means the guest doesn't have powershell or signtool
test.error("Powershell or Signtool must be installed in guest")
def check_windows_vmware_tools(vmcheck):
"""
Check vmware tools is uninstalled in VM
:param vmcheck: VMCheck object for vm checking
"""
def _get_vmware_info(cmd):
_, res = vmcheck.run_cmd(cmd)
if res and not re.search('vmtools', res, re.I):
return True
return False
cmds = ['tasklist', 'sc query vmtools']
for cmd in cmds:
res = utils_misc.wait_for(lambda: _get_vmware_info(cmd),
600,
step=30)
if not res:
test.fail("Failed to verification vmtools uninstallation")
def check_windows_service(vmcheck, service_name):
"""
Check service in VM
:param vmcheck: VMCheck object for vm checking
:param service_name: a service's name
"""
try:
res = utils_misc.wait_for(lambda: re.search(
'running', vmcheck.get_service_info(service_name), re.I),
600,
step=30)
except (ShellProcessTerminatedError, ShellStatusError):
# Windows guest may reboot after installing qemu-ga service
logging.debug('Windows guest is rebooting')
if vmcheck.session:
vmcheck.session.close()
vmcheck.session = None
# VM boots up is extremely slow when all testing in running on
# rhv server simultaneously, so set timeout to 1200.
vmcheck.create_session(timeout=1200)
res = utils_misc.wait_for(lambda: re.search(
'running', vmcheck.get_service_info(service_name), re.I),
600,
step=30)
if not res:
test.fail('Not found running %s service' % service_name)
def check_linux_ogac(vmcheck):
"""
Check qemu-guest-agent service in VM
:param vmcheck: VMCheck object for vm checking
"""
def get_pkgs(pkg_path):
"""
Get all qemu-guest-agent pkgs
"""
pkgs = []
for _, _, files in os.walk(pkg_path):
for file_name in files:
pkgs.append(file_name)
return pkgs
def get_pkg_version_vm():
"""
Get qemu-guest-agent version in VM
"""
vendor = vmcheck.get_vm_os_vendor()
if vendor in ['Ubuntu', 'Debian']:
cmd = 'dpkg -l qemu-guest-agent'
else:
cmd = 'rpm -q qemu-guest-agent'
_, output = vmcheck.run_cmd(cmd)
pkg_ver_ptn = [
r'qemu-guest-agent +[0-9]+:(.*?dfsg.*?) +',
r'qemu-guest-agent-(.*?)\.x86_64'
]
for ptn in pkg_ver_ptn:
if re.search(ptn, output):
return re.search(ptn, output).group(1)
return ''
if os.path.isfile(os.getenv('VIRTIO_WIN')):
mount_point = utils_v2v.v2v_mount(os.getenv('VIRTIO_WIN'),
'rhv_tools_setup_iso',
fstype='iso9660')
export_path = params['tmp_mount_point'] = mount_point
else:
export_path = os.getenv('VIRTIO_WIN')
qemu_guest_agent_dir = os.path.join(export_path, qa_path)
all_pkgs = get_pkgs(qemu_guest_agent_dir)
logging.debug('The installing qemu-guest-agent is: %s' % all_pkgs)
vm_pkg_ver = get_pkg_version_vm()
logging.debug('qemu-guest-agent version in vm: %s' % vm_pkg_ver)
# Check the service status of qemu-guest-agent in VM
status_ptn = r'Active: active \(running\)|qemu-ga \(pid +[0-9]+\) is running'
cmd = 'service qemu-ga status;systemctl status qemu-guest-agent;systemctl status qemu-ga*'
_, output = vmcheck.run_cmd(cmd)
if not re.search(status_ptn, output):
log_fail('qemu-guest-agent service exception')
def check_ubuntools(vmcheck):
"""
Check open-vm-tools, ubuntu-server in VM
:param vmcheck: VMCheck object for vm checking
"""
logging.info('Check if open-vm-tools service stopped')
status = utils_misc.get_guest_service_status(vmcheck.session,
'open-vm-tools')
logging.info('Service open-vm-tools status: %s', status)
if status != 'inactive':
log_fail('Service open-vm-tools is not stopped')
else:
logging.info('Check if the ubuntu-server exist')
content = vmcheck.session.cmd('dpkg -s ubuntu-server')
if 'install ok installed' in content:
logging.info('ubuntu-server has not been removed.')
else:
log_fail('ubuntu-server has been removed')
def global_pem_setup(f_pem):
"""
Setup global rhv server ca
:param f_pem: ca file path
"""
ca_anchors_dir = '/etc/pki/ca-trust/source/anchors'
shutil.copy(f_pem, ca_anchors_dir)
process.run('update-ca-trust extract', shell=True)
os.unlink(os.path.join(ca_anchors_dir, os.path.basename(f_pem)))
def global_pem_cleanup():
"""
Cleanup global rhv server ca
"""
process.run('update-ca-trust extract', shell=True)
def find_net(bridge_name):
"""
Find which network use specified bridge
:param bridge_name: bridge name you want to find
"""
net_list = virsh.net_state_dict(only_names=True)
net_name = ''
if len(net_list):
for net in net_list:
net_info = virsh.net_info(net).stdout.strip()
search = re.search(r'Bridge:\s+(\S+)', net_info)
if search:
if bridge_name == search.group(1):
net_name = net
else:
logging.info('Conversion server has no network')
return net_name
def destroy_net(net_name):
"""
destroy network in conversion server
"""
if virsh.net_state_dict()[net_name]['active']:
logging.info("Remove network %s in conversion server", net_name)
virsh.net_destroy(net_name)
if virsh.net_state_dict()[net_name]['autostart']:
virsh.net_autostart(net_name, "--disable")
output = virsh.net_list("--all").stdout.strip()
logging.info(output)
def start_net(net_name):
"""
start network in conversion server
"""
logging.info("Recover network %s in conversion server", net_name)
virsh.net_autostart(net_name)
if not virsh.net_state_dict()[net_name]['active']:
virsh.net_start(net_name)
output = virsh.net_list("--all").stdout.strip()
logging.info(output)
def check_static_ip_conf(vmcheck):
"""
Check static IP configuration in VM
:param vmcheck: VMCheck object for vm checking
"""
def _static_ip_check():
cmd = 'ipconfig /all'
_, output = vmcheck.run_cmd(cmd, debug=False)
v2v_cmd = params_get(params, 'v2v_command')
# --mac 00:50:56:ac:7a:4d:ip:192.168.1.2,192.168.1.1,22,192.168.1.100,10.73.2.108,10.66.127.10'
mac_ip_pattern = '--mac (([0-9a-zA-Z]{2}:){6})ip:([0-9,.]+)'
ip_config_list = re.search(mac_ip_pattern, v2v_cmd).group(3)
mac_addr = re.search(mac_ip_pattern,
v2v_cmd).group(1)[0:-1].upper().replace(
':', '-')
eth_adapter_ptn = r'Ethernet adapter Ethernet.*?NetBIOS over Tcpip'
try:
ipconfig = [
v for v in re.findall(eth_adapter_ptn, output, re.S)
if mac_addr in v
][0]
except IndexError:
return False
for i, value in enumerate(ip_config_list.split(',')):
if not value:
continue
# IP address
if i == 0:
ip_addr = r'IPv4 Address.*?: %s' % value
if not re.search(ip_addr, ipconfig, re.S):
logging.debug('Found IP addr failed')
return False
# Default gateway
if i == 1:
ip_gw = r'Default Gateway.*?: .*?%s' % value
if not re.search(ip_gw, ipconfig, re.S):
logging.debug('Found Gateway failed')
return False
# Subnet mask
if i == 2:
# convert subnet mask to cidr
bin_mask = '1' * int(value) + '0' * (32 - int(value))
cidr = '.'.join([
str(int(bin_mask[i * 8:i * 8 + 8], 2))
for i in range(4)
])
sub_mask = r'Subnet Mask.*?: %s' % cidr
if not re.search(sub_mask, ipconfig, re.S):
logging.debug('Found subnet mask failed')
return False
# DNS server list
if i >= 3:
dns_server = r'DNS Servers.*?:.*?%s' % value
if not re.search(dns_server, ipconfig, re.S):
logging.debug('Found DNS Server failed')
return False
return True
try:
vmcheck.create_session()
res = utils_misc.wait_for(_static_ip_check, 1800, step=300)
except (ShellTimeoutError, ShellProcessTerminatedError):
logging.debug(
'Lost connection to windows guest, the static IP may take effect'
)
if vmcheck.session:
vmcheck.session.close()
vmcheck.session = None
vmcheck.create_session()
res = utils_misc.wait_for(_static_ip_check, 300, step=30)
vmcheck.run_cmd('ipconfig /all') # debug msg
if not res:
test.fail('Checking static IP configuration failed')
def check_rhsrvany_checksums(vmcheck):
"""
Check if MD5 and SHA1 of rhsrvany.exe are correct
"""
def _get_expected_checksums(tool_exec, file):
val = process.run('%s %s' % (tool_exec, rhsrvany_path),
shell=True).stdout_text.split()[0]
if not val:
test.error('Get checksum failed')
logging.info('%s: Expect %s: %s', file, tool_exec, val)
return val
def _get_real_checksums(algorithm, file):
certutil_cmd = r'certutil -hashfile "%s"' % file
if algorithm == 'md5':
certutil_cmd += ' MD5'
res = vmcheck.session.cmd_output(certutil_cmd, safe=True)
logging.debug('%s output:\n%s', certutil_cmd, res)
val = res.strip().splitlines()[1].strip()
logging.info('%s: Real %s: %s', file, algorithm, val)
return val
logging.info('Check md5 and sha1 of rhsrvany.exe')
algorithms = {'md5': 'md5sum', 'sha1': 'sha1sum'}
rhsrvany_path = r'/usr/share/virt-tools/rhsrvany.exe'
rhsrvany_path_windows = r"C:\Program Files\Guestfs\Firstboot\rhsrvany.exe"
for key, val in algorithms.items():
expect_val = _get_expected_checksums(val, rhsrvany_path)
real_val = _get_real_checksums(key, rhsrvany_path_windows)
if expect_val == real_val:
logging.info('%s are correct', key)
else:
test.fail('%s of rhsrvany.exe is not correct' % key)
def check_result(result, status_error):
"""
Check virt-v2v command result
"""
def vm_check(status_error):
"""
Checking the VM
"""
if status_error:
return
if output_mode == 'json' and not check_json_output(params):
test.fail('check json output failed')
if output_mode == 'local' and not check_local_output(params):
test.fail('check local output failed')
if output_mode in ['null', 'json', 'local']:
return
# vmchecker must be put before skip_vm_check in order to clean up
# the VM.
vmchecker = VMChecker(test, params, env)
params['vmchecker'] = vmchecker
if skip_vm_check == 'yes':
logging.info('Skip checking vm after conversion: %s' %
skip_reason)
return
if output_mode == 'rhev':
if not utils_v2v.import_vm_to_ovirt(
params, address_cache, timeout=v2v_timeout):
test.fail('Import VM failed')
elif output_mode == 'libvirt':
virsh.start(vm_name, debug=True)
# Check guest following the checkpoint document after conversion
logging.info('Checking common checkpoints for v2v')
if 'ogac' in checkpoint:
# windows guests will reboot at any time after qemu-ga is
# installed. The process cannot be controlled. In order to
# don't break vmchecker.run() process, It's better to put
# check_windows_ogac before vmchecker.run(). Because in
# check_windows_ogac, it waits until rebooting completes.
vmchecker.checker.create_session()
if os_type == 'windows':
services = ['qemu-ga']
if not utils_v2v.multiple_versions_compare(
V2V_UNSUPPORT_RHEV_APT_VER):
services.append('rhev-apt')
if 'rhv-guest-tools' in os.getenv('VIRTIO_WIN'):
services.append('spice-ga')
for ser in services:
check_windows_service(vmchecker.checker, ser)
else:
check_linux_ogac(vmchecker.checker)
if 'mac_ip' in checkpoint:
check_static_ip_conf(vmchecker.checker)
ret = vmchecker.run()
if len(ret) == 0:
logging.info("All common checkpoints passed")
# Check specific checkpoints
if 'ogac' in checkpoint and 'signature' in checkpoint:
if not utils_v2v.multiple_versions_compare(
V2V_UNSUPPORT_RHEV_APT_VER):
check_windows_signature(vmchecker.checker,
r'c:\rhev-apt.exe')
if 'cdrom' in checkpoint and "device='cdrom'" not in vmchecker.vmxml:
test.fail('CDROM no longer exists')
if 'vmtools' in checkpoint:
check_vmtools(vmchecker.checker, checkpoint)
if 'modprobe' in checkpoint:
check_modprobe(vmchecker.checker)
if 'device_map' in checkpoint:
check_device_map(vmchecker.checker)
if 'resume_swap' in checkpoint:
check_resume_swap(vmchecker.checker)
if 'rhev_file' in checkpoint:
check_rhev_file_exist(vmchecker.checker)
if 'file_architecture' in checkpoint:
check_file_architecture(vmchecker.checker)
if 'ubuntu_tools' in checkpoint:
check_ubuntools(vmchecker.checker)
if 'vmware_tools' in checkpoint:
check_windows_vmware_tools(vmchecker.checker)
if 'without_default_net' in checkpoint:
if virsh.net_state_dict()[net_name]['active']:
log_fail("Bridge virbr0 already started during conversion")
if 'rhsrvany_checksum' in checkpoint:
check_rhsrvany_checksums(vmchecker.checker)
if 'block_dev' in checkpoint and not os.path.exists(blk_dev_link):
test.fail("checkpoint '%s' failed" % checkpoint)
# Merge 2 error lists
error_list.extend(vmchecker.errors)
# Virtio drivers will not be installed without virtio-win setup
if 'virtio_win_unset' in checkpoint:
missing_list = params.get('missing').split(',')
expect_errors = ['Not find driver: ' + x for x in missing_list]
logging.debug('Expect errors: %s' % expect_errors)
logging.debug('Actual errors: %s' % error_list)
if set(error_list) == set(expect_errors):
error_list[:] = []
else:
logging.error('Virtio drivers not meet expectation')
utils_v2v.check_exit_status(result, status_error)
output = result.stdout_text + result.stderr_text
# VM or local output checking
vm_check(status_error)
# Check log size decrease option
if 'log decrease' in checkpoint:
nbdkit_option = r'nbdkit\.backend\.datapath=0'
if not re.search(nbdkit_option, output):
test.fail("checkpoint '%s' failed" % checkpoint)
if 'fstrim_warning' in checkpoint:
# Actually, fstrim has no relationship with v2v, it may be related
# to kernel, this warning really doesn't matter and has no harm to
# the conversion.
V2V_FSTRIM_SUCESS_VER = "[virt-v2v-1.45.1-1.el9,)"
if utils_v2v.multiple_versions_compare(V2V_FSTRIM_SUCESS_VER):
params.update({'expect_msg': None})
# Log checking
log_check = utils_v2v.check_log(params, output)
if log_check:
log_fail(log_check)
if len(error_list):
test.fail('%d checkpoints failed: %s' %
(len(error_list), error_list))
try:
if version_requried and not utils_v2v.multiple_versions_compare(
version_requried):
test.cancel("Testing requires version: %s" % version_requried)
# See man virt-v2v-input-xen(1)
if enable_legacy_cp:
process.run('update-crypto-policies --set LEGACY',
verbose=True,
ignore_status=True,
shell=True)
v2v_params = {
'hostname': remote_host,
'hypervisor': 'esx',
'main_vm': vm_name,
'vpx_dc': vpx_dc,
'esx_ip': esx_ip,
'new_name': vm_name + utils_misc.generate_random_string(4),
'v2v_opts': v2v_opts,
'input_mode': 'libvirt',
'os_storage': os_storage,
'os_pool': os_pool,
'network': params.get('network'),
'bridge': params.get('bridge'),
'target': params.get('target'),
'password': vpx_passwd if src_uri_type != 'esx' else esxi_password,
'input_transport': input_transport,
'vcenter_host': vpx_hostname,
'vcenter_password': vpx_passwd,
'vddk_thumbprint': vddk_thumbprint,
'vddk_libdir': vddk_libdir,
'vddk_libdir_src': vddk_libdir_src,
'src_uri_type': src_uri_type,
'esxi_password': esxi_password,
'esxi_host': esxi_host,
'output_method': output_method,
'os_storage_name': storage_name,
'rhv_upload_opts': rhv_upload_opts,
'oo_json_disk_pattern': json_disk_pattern,
'cmd_has_ip': cmd_has_ip,
'params': params
}
os.environ['LIBGUESTFS_BACKEND'] = 'direct'
v2v_uri = utils_v2v.Uri('esx')
remote_uri = v2v_uri.get_uri(remote_host, vpx_dc, esx_ip)
# Create password file for access to ESX hypervisor
vpx_passwd_file = params.get("vpx_passwd_file")
with open(vpx_passwd_file, 'w') as pwd_f:
if src_uri_type == 'esx':
pwd_f.write(esxi_password)
else:
pwd_f.write(vpx_passwd)
v2v_params['v2v_opts'] += " -ip %s" % vpx_passwd_file
if params.get('output_format'):
v2v_params.update({'of_format': params['output_format']})
# Rename guest with special name while converting to rhev
if '#' in vm_name and output_mode == 'rhev':
v2v_params['new_name'] = v2v_params['new_name'].replace('#', '_')
# Create SASL user on the ovirt host
if output_mode == 'rhev':
# create different sasl_user name for different job
params.update({
'sasl_user':
params.get("sasl_user") + utils_misc.generate_random_string(3)
})
logging.info('sals user name is %s' % params.get("sasl_user"))
user_pwd = "[['%s', '%s']]" % (params.get("sasl_user"),
params.get("sasl_pwd"))
v2v_sasl = utils_sasl.SASL(sasl_user_pwd=user_pwd)
v2v_sasl.server_ip = params.get("remote_ip")
v2v_sasl.server_user = params.get('remote_user')
v2v_sasl.server_pwd = params.get('remote_pwd')
v2v_sasl.setup(remote=True)
logging.debug('A SASL session %s was created', v2v_sasl)
if output_method == 'rhv_upload':
# Create password file for '-o rhv_upload' to connect to ovirt
with open(rhv_passwd_file, 'w') as f:
f.write(rhv_passwd)
# Copy ca file from ovirt to local
remote.scp_from_remote(ovirt_hostname, 22, 'root',
ovirt_engine_passwd, ovirt_ca_file_path,
local_ca_file_path)
# Create libvirt dir pool
if output_mode == 'libvirt':
pvt.pre_pool(pool_name, pool_type, pool_target, '')
if 'root' in checkpoint and 'ask' in checkpoint:
v2v_params['v2v_opts'] += ' --root ask'
v2v_params['custom_inputs'] = params.get('choice', '2')
if 'root' in checkpoint and 'ask' not in checkpoint:
root_option = params.get('root_option')
v2v_params['v2v_opts'] += ' --root %s' % root_option
if 'with_proxy' in checkpoint:
http_proxy = params.get('esx_http_proxy')
https_proxy = params.get('esx_https_proxy')
logging.info('Set http_proxy=%s, https_proxy=%s', http_proxy,
https_proxy)
os.environ['http_proxy'] = http_proxy
os.environ['https_proxy'] = https_proxy
if 'ogac' in checkpoint:
os.environ['VIRTIO_WIN'] = virtio_win_path
if not os.path.exists(os.getenv('VIRTIO_WIN')):
test.fail('%s does not exist' % os.getenv('VIRTIO_WIN'))
if os.path.isdir(os.getenv('VIRTIO_WIN')) and os_type == 'linux':
export_path = os.getenv('VIRTIO_WIN')
qemu_guest_agent_dir = os.path.join(export_path, qa_path)
if not os.path.exists(qemu_guest_agent_dir) and os.access(
export_path, os.W_OK) and qa_url:
logging.debug(
'Not found qemu-guest-agent in virtio-win or rhv-guest-tools-iso,'
' Try to prepare it manually. This is not a permanent step, once'
' the official build includes it, this step should be removed.'
)
os.makedirs(qemu_guest_agent_dir)
rpm_name = os.path.basename(qa_url)
download.get_file(
qa_url, os.path.join(qemu_guest_agent_dir, rpm_name))
if 'virtio_iso_blk' in checkpoint:
if not os.path.exists(virtio_win_path):
test.fail('%s does not exist' % virtio_win_path)
# Find a free loop device
free_loop_dev = process.run("losetup --find",
shell=True).stdout_text.strip()
# Setup a loop device
cmd = 'losetup %s %s' % (free_loop_dev, virtio_win_path)
process.run(cmd, shell=True)
os.environ['VIRTIO_WIN'] = free_loop_dev
if 'block_dev' in checkpoint:
os_directory = params_get(params, 'os_directory')
block_count = params_get(params, 'block_count')
os_directory = tempfile.TemporaryDirectory(prefix='v2v_test_',
dir=os_directory)
diskimage = '%s/diskimage' % os_directory.name
# Update 'os_directory' for '-os' option
params['os_directory'] = os_directory.name
# Create a 1G image
cmd = 'dd if=/dev/zero of=%s bs=10M count=%s' % (diskimage,
block_count)
process.run(cmd, shell=True)
# Build filesystem
cmd = 'mkfs.ext4 %s' % diskimage
process.run(cmd, shell=True)
# Find a free loop device
free_loop_dev = process.run("losetup --find",
shell=True).stdout_text.strip()
# Setup the image as a block device
cmd = 'losetup %s %s' % (free_loop_dev, diskimage)
process.run(cmd, shell=True)
# Create a soft link to the loop device
blk_dev_link = '%s/mydisk1' % os_directory.name
cmd = 'ln -s %s %s' % (free_loop_dev, blk_dev_link)
process.run(cmd, shell=True)
if 'invalid_pem' in checkpoint:
# simply change the 2nd line to lowercase to get an invalid pem
with open(local_ca_file_path, 'r+') as fd:
for i in range(2):
pos = fd.tell()
res = fd.readline()
fd.seek(pos)
fd.write(res.lower())
fd.flush()
if 'without_default_net' in checkpoint:
net_name = find_net('virbr0')
if net_name:
destroy_net(net_name)
if 'bandwidth' in checkpoint:
dynamic_speeds = params_get(params, 'dynamic_speeds')
bandwidth_file = params_get(params, 'bandwidth_file')
with open(bandwidth_file, 'w') as fd:
fd.write(dynamic_speeds)
if checkpoint[0].startswith('virtio_win'):
cp = checkpoint[0]
src_dir = params.get('virtio_win_dir')
dest_dir = os.path.join(data_dir.get_tmp_dir(), 'virtio-win')
iso_path = os.path.join(dest_dir, 'virtio-win.iso')
if not os.path.exists(dest_dir):
shutil.copytree(src_dir, dest_dir)
virtio_win_env = params.get('virtio_win_env', 'VIRTIO_WIN')
process.run('rpm -e virtio-win')
if process.run('rpm -q virtio-win',
ignore_status=True).exit_status == 0:
test.error('not removed')
if cp.endswith('unset'):
logging.info('Unset env %s' % virtio_win_env)
os.unsetenv(virtio_win_env)
if cp.endswith('custom'):
logging.info('Set env %s=%s' % (virtio_win_env, dest_dir))
os.environ[virtio_win_env] = dest_dir
if cp.endswith('iso_mount'):
logging.info('Mount iso to /opt')
process.run('mount %s /opt' % iso_path)
os.environ[virtio_win_env] = '/opt'
if cp.endswith('iso_file'):
logging.info('Set env %s=%s' % (virtio_win_env, iso_path))
os.environ[virtio_win_env] = iso_path
if 'luks_dev_keys' in checkpoint:
luks_password = params_get(params, 'luks_password', '')
luks_keys = params_get(params, 'luks_keys', '')
keys_options = ' '.join(
list(
map(lambda i: '--key %s' % i if i else '',
luks_keys.split(';'))))
if 'invalid_pwd_file' not in checkpoint:
is_file_key = r'--key \S+:file:(\S+)'
for file_key in re.findall(is_file_key, keys_options):
with open(file_key, 'w') as fd:
fd.write(luks_password)
v2v_params['v2v_opts'] += ' ' + keys_options
if 'empty_cdrom' in checkpoint:
virsh_dargs = {
'uri': remote_uri,
'remote_ip': remote_host,
'remote_user': '******',
'remote_pwd': vpx_passwd,
'auto_close': True,
'debug': True
}
remote_virsh = virsh.VirshPersistent(**virsh_dargs)
v2v_result = remote_virsh.dumpxml(vm_name)
remote_virsh.close_session()
else:
if 'exist_uuid' in checkpoint:
auto_clean = False
if checkpoint[0] in [
'mismatched_uuid', 'no_uuid', 'invalid_source',
'system_rhv_pem'
]:
cmd_only = True
auto_clean = False
v2v_result = utils_v2v.v2v_cmd(v2v_params, auto_clean, cmd_only,
interaction_run)
if 'new_name' in v2v_params:
vm_name = params['main_vm'] = v2v_params['new_name']
if 'system_rhv_pem' in checkpoint:
if 'set' in checkpoint:
global_pem_setup(local_ca_file_path)
rhv_cafile = r'-oo rhv-cafile=\S+\s*'
new_cmd = utils_v2v.cmd_remove_option(v2v_result, rhv_cafile)
logging.debug('New v2v command:\n%s', new_cmd)
if 'mismatched_uuid' in checkpoint:
# append more uuid
new_cmd = v2v_result + ' -oo rhv-disk-uuid=%s' % str(uuid.uuid4())
if 'no_uuid' in checkpoint:
rhv_disk_uuid = r'-oo rhv-disk-uuid=\S+\s*'
new_cmd = utils_v2v.cmd_remove_option(v2v_result, rhv_disk_uuid)
logging.debug('New v2v command:\n%s', new_cmd)
if 'exist_uuid' in checkpoint:
# Use to cleanup the VM because it will not be run in check_result
vmchecker = VMChecker(test, params, env)
params['vmchecker'] = vmchecker
# Update name to avoid conflict
new_vm_name = v2v_params['new_name'] + '_exist_uuid'
new_cmd = v2v_result.command.replace('-on %s' % vm_name,
'-on %s' % new_vm_name)
new_cmd += ' --no-copy'
logging.debug('re-run v2v command:\n%s', new_cmd)
if 'invalid_source' in checkpoint:
if params.get('invalid_vpx_hostname'):
new_cmd = v2v_result.replace(
vpx_hostname, params.get('invalid_vpx_hostname'))
if params.get('invalid_esx_hostname'):
new_cmd = v2v_result.replace(
esxi_host, params.get('invalid_esx_hostname'))
if checkpoint[0] in [
'mismatched_uuid', 'no_uuid', 'invalid_source', 'exist_uuid',
'system_rhv_pem'
]:
v2v_result = utils_v2v.cmd_run(new_cmd,
params.get('v2v_dirty_resources'))
check_result(v2v_result, status_error)
finally:
if enable_legacy_cp:
process.run('update-crypto-policies --set DEFAULT',
verbose=True,
ignore_status=True,
shell=True)
if checkpoint[0].startswith('virtio_win'):
utils_package.package_install(['virtio-win'])
if 'virtio_win_iso_mount' in checkpoint:
process.run('umount /opt', ignore_status=True)
if 'ogac' in checkpoint and params.get('tmp_mount_point'):
if os.path.exists(params.get('tmp_mount_point')):
utils_misc.umount(os.getenv('VIRTIO_WIN'),
params['tmp_mount_point'], 'iso9660')
os.environ.pop('VIRTIO_WIN')
if 'block_dev' in checkpoint and hasattr(os_directory, 'name'):
process.run('losetup -d %s' % free_loop_dev, shell=True)
os_directory.cleanup()
if 'virtio_iso_blk' in checkpoint:
process.run('losetup -d %s' % free_loop_dev, shell=True)
os.environ.pop('VIRTIO_WIN')
if 'system_rhv_pem' in checkpoint and 'set' in checkpoint:
global_pem_cleanup()
if 'without_default_net' in checkpoint:
if net_name:
start_net(net_name)
if params.get('vmchecker'):
params['vmchecker'].cleanup()
if output_mode == 'rhev' and v2v_sasl:
v2v_sasl.cleanup()
logging.debug('SASL session %s is closing', v2v_sasl)
v2v_sasl.close_session()
if output_mode == 'libvirt':
pvt.cleanup_pool(pool_name, pool_type, pool_target, '')
if 'with_proxy' in checkpoint:
logging.info('Unset http_proxy&https_proxy')
os.environ.pop('http_proxy')
os.environ.pop('https_proxy')
if unprivileged_user:
process.system("userdel -fr %s" % unprivileged_user)
if params.get('os_directory') and os.path.isdir(
params['os_directory']):
shutil.rmtree(params['os_directory'], ignore_errors=True)
# Cleanup constant files
utils_v2v.cleanup_constant_files(params)
else:
if shp_num == 0 and started_anon <= 0:
raise error.TestFail("VM doesn't use transparent"
" hugepage, while static"
" hugepage is disabled")
finally:
# end up session
for session in sessions:
session.close()
for vm in vms:
if vm.is_alive():
vm.destroy()
for vm_name in vm_names:
if mb_enable:
vm_xml.VMXML.del_memoryBacking_tag(vm_name)
else:
vm_xml.VMXML.set_memoryBacking_tag(vm_name)
utils_libvirtd.libvirtd_restart()
if tlbfs_enable is True:
if tlbfs_status is not True:
utils_misc.umount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
else:
if tlbfs_status is True:
utils_misc.mount("hugetlbfs", "/dev/hugepages", "hugetlbfs")
utils_memory.set_num_huge_pages(shp_orig_num)
utils_memory.set_transparent_hugepage(thp_orig_status)
def test_formatted_part(test, vm, params):
"""
1) Do some necessary check
2) Format additional disk with specific filesystem
3) Try to write a file to mounted device and get md5
4) Login to check writed file and its md5
"""
add_device = params.get("gf_additional_device", "/dev/vdb")
device_in_lgf = process.run("echo %s | sed -e 's/vd/sd/g'" % add_device,
ignore_status=True, shell=True).stdout_text.strip()
if utils_test.libguestfs.primary_disk_virtio(vm):
device_in_vm = add_device
else:
device_in_vm = "/dev/vda"
device_part = "%s1" % device_in_lgf
device_part_in_vm = "%s1" % device_in_vm
# Mount specific partition
params['special_mountpoints'] = [device_part]
vt = utils_test.libguestfs.VirtTools(vm, params)
# Create a new vm with additional disk
vt.update_vm_disk()
# Default vm_ref is oldvm, so switch it.
vm_ref = vt.newvm.name
# Format disk
format_result = vt.format_disk(filesystem="ext3", partition="mbr")
if format_result.exit_status:
test.fail("Format added disk failed.")
logging.info("Format added disk successfully.")
# List filesystems detail
list_fs_detail = vt.get_filesystems_info(vm_ref)
if list_fs_detail.exit_status:
test.fail("List filesystems detail failed.")
logging.info("List filesystems detail successfully.")
content = "This is file for formatted part test."
path = params.get("temp_file", "formatted_part")
mountpoint = params.get("vt_mountpoint", "/mnt")
writes, writeo = vt.write_file_with_guestmount(mountpoint, path,
content, vm_ref,
cleanup=False)
if writes is False:
utils_misc.umount("", mountpoint, "")
test.fail("Write file to mounted filesystem failed.")
logging.info("Create %s successfully.", writeo)
# Compute new file's md5
if os.path.isfile(writeo):
md5s, md5o = process.getstatusoutput("md5sum %s" % writeo)
utils_misc.umount("", mountpoint, "")
if md5s:
test.fail("Compute %s's md5 failed." % writeo)
md5 = md5o.split()[0].strip()
logging.debug("%s's md5 in newvm is:%s", writeo, md5)
else:
utils_misc.umount("", mountpoint, "")
test.fail("Can not find %s." % writeo)
attached_vm = vt.newvm
try:
attached_vm.start()
session = attached_vm.wait_for_login()
except Exception as detail:
attached_vm.destroy()
test.fail(str(detail))
try:
file_path = os.path.join(mountpoint, path)
mounts = session.cmd_status("mount %s %s" % (device_part_in_vm,
mountpoint), timeout=10)
if mounts:
logging.error("Mount device in vm failed.")
md51 = session.cmd_output("md5sum %s" % file_path)
logging.debug(md51)
if not re.search(md5o, md51):
attached_vm.destroy()
attached_vm.wait_for_shutdown()
test.fail("Got a different md5.")
logging.info("Got matched md5.")
session.cmd_status("cat %s" % file_path, timeout=5)
attached_vm.destroy()
attached_vm.wait_for_shutdown()
except (virt_vm.VMError, remote.LoginError, aexpect.ShellError) as detail:
if attached_vm.is_alive():
attached_vm.destroy()
if not re.search(content, str(detail)):
test.fail(str(detail))
logging.info("Check file on guest successfully.")
def run(test, params, env):
"""
Test various options of virt-v2v.
"""
if utils_v2v.V2V_EXEC is None:
raise ValueError('Missing command: virt-v2v')
for v in params.itervalues():
if "V2V_EXAMPLE" in v:
raise exceptions.TestSkipError("Please set real value for %s" % v)
vm_name = params.get("main_vm", "EXAMPLE")
new_vm_name = params.get("new_vm_name")
input_mode = params.get("input_mode")
v2v_options = params.get("v2v_options", "")
hypervisor = params.get("hypervisor", "kvm")
remote_host = params.get("remote_host", "EXAMPLE")
vpx_dc = params.get("vpx_dc", "EXAMPLE")
esx_ip = params.get("esx_ip", "EXAMPLE")
output_mode = params.get("output_mode")
output_storage = params.get("output_storage", "default")
disk_img = params.get("input_disk_image", "")
nfs_storage = params.get("nfs_storage")
no_root = 'yes' == params.get('no_root', 'no')
mnt_point = params.get("mount_point")
export_domain_uuid = params.get("export_domain_uuid", "")
fake_domain_uuid = params.get("fake_domain_uuid")
vdsm_image_uuid = params.get("vdsm_image_uuid")
vdsm_vol_uuid = params.get("vdsm_vol_uuid")
vdsm_vm_uuid = params.get("vdsm_vm_uuid")
vdsm_ovf_output = params.get("vdsm_ovf_output")
v2v_user = params.get("unprivileged_user", "")
v2v_timeout = int(params.get("v2v_timeout", 1200))
status_error = "yes" == params.get("status_error", "no")
su_cmd = "su - %s -c " % v2v_user
output_uri = params.get("oc_uri", "")
pool_name = params.get("pool_name", "v2v_test")
pool_type = params.get("pool_type", "dir")
pool_target = params.get("pool_target_path", "v2v_pool")
emulated_img = params.get("emulated_image_path", "v2v-emulated-img")
pvt = utlv.PoolVolumeTest(test, params)
new_v2v_user = False
address_cache = env.get('address_cache')
params['vmcheck_flag'] = False
checkpoint = params.get('checkpoint', '')
def create_pool(user_pool=False,
pool_name=pool_name,
pool_target=pool_target):
"""
Create libvirt pool as the output storage
"""
if output_uri == "qemu:///session" or user_pool:
target_path = os.path.join("/home", v2v_user, pool_target)
cmd = su_cmd + "'mkdir %s'" % target_path
process.system(cmd, verbose=True)
cmd = su_cmd + "'virsh pool-create-as %s dir" % pool_name
cmd += " --target %s'" % target_path
process.system(cmd, verbose=True)
else:
pvt.pre_pool(pool_name, pool_type, pool_target, emulated_img)
def cleanup_pool(user_pool=False,
pool_name=pool_name,
pool_target=pool_target):
"""
Clean up libvirt pool
"""
if output_uri == "qemu:///session" or user_pool:
cmd = su_cmd + "'virsh pool-destroy %s'" % pool_name
process.system(cmd, verbose=True)
target_path = os.path.join("/home", v2v_user, pool_target)
cmd = su_cmd + "'rm -rf %s'" % target_path
process.system(cmd, verbose=True)
else:
pvt.cleanup_pool(pool_name, pool_type, pool_target, emulated_img)
def get_all_uuids(output):
"""
Get export domain uuid, image uuid and vol uuid from command output.
"""
tmp_target = re.findall(r"qemu-img\s'convert'\s.+\s'(\S+)'\n", output)
if len(tmp_target) < 1:
raise exceptions.TestError("Fail to find tmp target file name when"
" converting vm disk image")
targets = tmp_target[0].split('/')
return (targets[3], targets[5], targets[6])
def get_ovf_content(output):
"""
Find and read ovf file.
"""
export_domain_uuid, _, vol_uuid = get_all_uuids(output)
export_vm_dir = os.path.join(mnt_point, export_domain_uuid,
'master/vms')
ovf_content = ""
if os.path.isdir(export_vm_dir):
ovf_id = "ovf:id='%s'" % vol_uuid
ret = process.system_output("grep -R \"%s\" %s" %
(ovf_id, export_vm_dir))
ovf_file = ret.split(":")[0]
if os.path.isfile(ovf_file):
ovf_f = open(ovf_file, "r")
ovf_content = ovf_f.read()
ovf_f.close()
else:
logging.error("Can't find ovf file to read")
return ovf_content
def get_img_path(output):
"""
Get the full path of the converted image.
"""
img_name = vm_name + "-sda"
if output_mode == "libvirt":
img_path = virsh.vol_path(img_name, output_storage).stdout.strip()
elif output_mode == "local":
img_path = os.path.join(output_storage, img_name)
elif output_mode in ["rhev", "vdsm"]:
export_domain_uuid, image_uuid, vol_uuid = get_all_uuids(output)
img_path = os.path.join(mnt_point, export_domain_uuid, 'images',
image_uuid, vol_uuid)
return img_path
def check_vmtype(ovf, expected_vmtype):
"""
Verify vmtype in ovf file.
"""
if output_mode != "rhev":
return
if expected_vmtype == "server":
vmtype_int = 1
elif expected_vmtype == "desktop":
vmtype_int = 0
else:
return
if "<VmType>%s</VmType>" % vmtype_int in ovf:
logging.info("Find VmType=%s in ovf file", expected_vmtype)
else:
raise exceptions.TestFail("VmType check failed")
def check_image(img_path, check_point, expected_value):
"""
Verify image file allocation mode and format
"""
if not img_path or not os.path.isfile(img_path):
raise exceptions.TestError("Image path: '%s' is invalid" %
img_path)
img_info = utils_misc.get_image_info(img_path)
logging.debug("Image info: %s", img_info)
if check_point == "allocation":
if expected_value == "sparse":
if img_info['vsize'] > img_info['dsize']:
logging.info("%s is a sparse image", img_path)
else:
raise exceptions.TestFail("%s is not a sparse image" %
img_path)
elif expected_value == "preallocated":
if img_info['vsize'] <= img_info['dsize']:
logging.info("%s is a preallocated image", img_path)
else:
raise exceptions.TestFail(
"%s is not a preallocated image" % img_path)
if check_point == "format":
if expected_value == img_info['format']:
logging.info("%s format is %s", img_path, expected_value)
else:
raise exceptions.TestFail("%s format is not %s" %
(img_path, expected_value))
def check_new_name(output, expected_name):
"""
Verify guest name changed to the new name.
"""
found = False
if output_mode == "libvirt":
found = virsh.domain_exists(expected_name)
if output_mode == "local":
found = os.path.isfile(
os.path.join(output_storage, expected_name + "-sda"))
if output_mode in ["rhev", "vdsm"]:
ovf = get_ovf_content(output)
found = "<Name>%s</Name>" % expected_name in ovf
else:
return
if found:
logging.info("Guest name renamed when converting it")
else:
raise exceptions.TestFail("Rename guest failed")
def check_nocopy(output):
"""
Verify no image created if convert command use --no-copy option
"""
img_path = get_img_path(output)
if not os.path.isfile(img_path):
logging.info("No image created with --no-copy option")
else:
raise exceptions.TestFail("Find %s" % img_path)
def check_connection(output, expected_uri):
"""
Check output connection uri used when converting guest
"""
init_msg = "Initializing the target -o libvirt -oc %s" % expected_uri
if init_msg in output:
logging.info("Find message: %s", init_msg)
else:
raise exceptions.TestFail("Not find message: %s" % init_msg)
def check_ovf_snapshot_id(ovf_content):
"""
Check if snapshot id in ovf file consists of '0's
"""
search = re.search("ovf:vm_snapshot_id='(.*?)'", ovf_content)
if search:
snapshot_id = search.group(1)
logging.debug('vm_snapshot_id = %s', snapshot_id)
if snapshot_id.count('0') >= 32:
raise exceptions.TestFail('vm_snapshot_id consists with "0"')
else:
raise exceptions.TestFail('Fail to find snapshot_id')
def check_source(output):
"""
Check if --print-source option print the correct info
"""
# Parse source info
source = output.split('\n')[2:]
for i in range(len(source)):
if source[i].startswith('\t'):
source[i - 1] += source[i]
source[i] = ''
source_strip = [x.strip() for x in source if x.strip()]
source_info = {}
for line in source_strip:
source_info[line.split(':')[0]] = line.split(':', 1)[1].strip()
logging.debug('Source info to check: %s', source_info)
checklist = [
'nr vCPUs', 'hypervisor type', 'source name', 'memory', 'display',
'CPU features', 'disks', 'NICs'
]
for key in checklist:
if key not in source_info:
raise exceptions.TestFail('%s info missing' % key)
# Check single values
fail = []
xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
check_map = {}
check_map['nr vCPUs'] = xml.vcpu
check_map['hypervisor type'] = xml.hypervisor_type
check_map['source name'] = xml.vm_name
check_map['memory'] = str(int(xml.max_mem) * 1024) + ' (bytes)'
check_map['display'] = xml.get_graphics_devices()[0].type_name
logging.info('KEY:\tSOURCE<-> XML')
for key in check_map:
logging.info('%-15s:%18s <-> %s', key, source_info[key],
check_map[key])
if source_info[key] != str(check_map[key]):
fail.append(key)
# Check disk info
disk = xml.get_disk_all().values()[0]
bus, type = disk.find('target').get('bus'), disk.find('driver').get(
'type')
path = disk.find('source').get('file')
disks_info = "%s (%s) [%s]" % (path, type, bus)
logging.info('disks:%s<->%s', source_info['disks'], disks_info)
if source_info['disks'] != disks_info:
fail.append('disks')
# Check nic info
nic = xml.get_iface_all().values()[0]
type = nic.get('type')
mac = nic.find('mac').get('address')
nic_source = nic.find('source')
name = nic_source.get(type)
nic_info = '%s "%s" mac: %s' % (type, name, mac)
logging.info('NICs:%s<->%s', source_info['NICs'], nic_info)
if source_info['NICs'].lower() != nic_info.lower():
fail.append('NICs')
# Check cpu features
feature_list = xml.features.get_feature_list()
logging.info('CPU features:%s<->%s', source_info['CPU features'],
feature_list)
if sorted(source_info['CPU features'].split(',')) != sorted(
feature_list):
fail.append('CPU features')
if fail:
raise exceptions.TestFail('Source info not correct for: %s' % fail)
def check_result(cmd, result, status_error):
"""
Check virt-v2v command result
"""
utlv.check_exit_status(result, status_error)
output = result.stdout + result.stderr
if status_error:
if checkpoint == 'length_of_error':
log_lines = output.split('\n')
v2v_start = False
for line in log_lines:
if line.startswith('virt-v2v:'):
v2v_start = True
if line.startswith('libvirt:'):
v2v_start = False
if v2v_start and len(line) > 72:
raise exceptions.TestFail(
'Error log longer than 72 '
'charactors: %s', line)
if checkpoint == 'disk_not_exist':
vol_list = virsh.vol_list(pool_name)
logging.info(vol_list)
if vm_name in vol_list.stdout:
raise exceptions.TestFail('Disk exists for vm %s' %
vm_name)
else:
error_map = {
'conflict_options': ['option used more than once'],
'xen_no_output_format':
['The input metadata did not define'
' the disk format'],
'in_place':
['virt-v2v: error: --in-place cannot be used in RHEL 7']
}
if error_map.has_key(checkpoint) and not utils_v2v.check_log(
output, error_map[checkpoint]):
raise exceptions.TestFail('Not found error message %s' %
error_map[checkpoint])
else:
if output_mode == "rhev" and checkpoint != 'quiet':
ovf = get_ovf_content(output)
logging.debug("ovf content: %s", ovf)
check_ovf_snapshot_id(ovf)
if '--vmtype' in cmd:
expected_vmtype = re.findall(r"--vmtype\s(\w+)", cmd)[0]
check_vmtype(ovf, expected_vmtype)
if '-oa' in cmd and '--no-copy' not in cmd:
expected_mode = re.findall(r"-oa\s(\w+)", cmd)[0]
img_path = get_img_path(output)
def check_alloc():
try:
check_image(img_path, "allocation", expected_mode)
return True
except exceptions.TestFail:
pass
if not utils_misc.wait_for(check_alloc, timeout=600,
step=10.0):
raise exceptions.TestFail('Allocation check failed.')
if '-of' in cmd and '--no-copy' not in cmd and checkpoint != 'quiet':
expected_format = re.findall(r"-of\s(\w+)", cmd)[0]
img_path = get_img_path(output)
check_image(img_path, "format", expected_format)
if '-on' in cmd:
expected_name = re.findall(r"-on\s(\w+)", cmd)[0]
check_new_name(output, expected_name)
if '--no-copy' in cmd:
check_nocopy(output)
if '-oc' in cmd:
expected_uri = re.findall(r"-oc\s(\S+)", cmd)[0]
check_connection(output, expected_uri)
if output_mode == "rhev":
if not utils_v2v.import_vm_to_ovirt(params, address_cache):
raise exceptions.TestFail("Import VM failed")
else:
params['vmcheck_flag'] = True
if output_mode == "libvirt":
if "qemu:///session" not in v2v_options and not no_root:
virsh.start(vm_name, debug=True, ignore_status=False)
if checkpoint == 'quiet':
if len(output.strip()) != 0:
raise exceptions.TestFail(
'Output is not empty in quiet mode')
if checkpoint == 'dependency':
if 'libguestfs-winsupport' not in output:
raise exceptions.TestFail(
'libguestfs-winsupport not in dependency')
if 'qemu-kvm-rhev' in output:
raise exceptions.TestFail('qemu-kvm-rhev is in dependency')
win_img = params.get('win_image')
command = 'guestfish -a %s -i'
if process.run(command % win_img,
ignore_status=True).exit_status == 0:
raise exceptions.TestFail('Command "%s" success' %
command % win_img)
if checkpoint == 'no_dcpath':
if not utils_v2v.check_log(output, ['--dcpath'], expect=False):
raise exceptions.TestFail('"--dcpath" is not removed')
if checkpoint == 'debug_overlays':
search = re.search('Overlay saved as(.*)', output)
if not search:
raise exceptions.TestFail(
'Not find log of saving overlays')
overlay_path = search.group(1).strip()
logging.debug('Overlay file location: %s' % overlay_path)
if os.path.isfile(overlay_path):
logging.info('Found overlay file: %s' % overlay_path)
else:
raise exceptions.TestFail('Overlay file not saved')
if checkpoint.startswith('empty_nic_source'):
target_str = '%s "eth0" mac: %s' % (params[checkpoint][0],
params[checkpoint][1])
logging.info('Expect log: %s', target_str)
if target_str not in result.stdout.lower():
raise exceptions.TestFail('Expect log not found: %s' %
target_str)
if checkpoint == 'print_source':
check_source(result.stdout)
if checkpoint == 'machine_readable':
if os.path.exists(params.get('example_file', '')):
expect_output = open(params['example_file']).read().strip()
logging.debug(expect_output)
logging.debug(expect_output == result.stdout.strip())
else:
raise exceptions.TestError('No content to compare with')
if checkpoint == 'compress':
img_path = get_img_path(output)
logging.info('Image path: %s', img_path)
disk_check = process.run('qemu-img check %s' % img_path).stdout
logging.info(disk_check)
compress_info = disk_check.split(',')[-1].split('%')[0].strip()
compress_rate = float(compress_info)
logging.info('%s%% compressed', compress_rate)
if compress_rate < 0.1:
raise exceptions.TestFail('Disk image NOT compressed')
if checkpoint == 'tail_log':
messages = params['tail'].get_output()
logging.info('Content of /var/log/messages during conversion:')
logging.info(messages)
msg_content = params['msg_content']
if not utils_v2v.check_log(messages, [msg_content],
expect=False):
raise exceptions.TestFail(
'Found "%s" in /var/log/messages' % msg_content)
backup_xml = None
vdsm_domain_dir, vdsm_image_dir, vdsm_vm_dir = ("", "", "")
try:
if checkpoint.startswith('empty_nic_source'):
xml = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
iface = xml.get_devices('interface')[0]
disks = xml.get_devices('disk')
del iface.source
iface.type_name = checkpoint.split('_')[-1]
iface.source = {iface.type_name: ''}
params[checkpoint] = [iface.type_name, iface.mac_address]
logging.debug(iface.source)
devices = vm_xml.VMXMLDevices()
devices.extend(disks)
devices.append(iface)
xml.set_devices(devices)
logging.info(xml.xmltreefile)
params['input_xml'] = xml.xmltreefile.name
# Build input options
input_option = ""
if input_mode is None:
pass
elif input_mode == "libvirt":
uri_obj = utils_v2v.Uri(hypervisor)
ic_uri = uri_obj.get_uri(remote_host, vpx_dc, esx_ip)
if checkpoint == 'with_ic':
ic_uri = 'qemu:///session'
input_option = "-i %s -ic %s %s" % (input_mode, ic_uri, vm_name)
if checkpoint == 'without_ic':
input_option = '-i %s %s' % (input_mode, vm_name)
# Build network&bridge option to avoid network error
v2v_options += " -b %s -n %s" % (params.get("output_bridge"),
params.get("output_network"))
elif input_mode == "disk":
input_option += "-i %s %s" % (input_mode, disk_img)
elif input_mode == 'libvirtxml':
input_xml = params.get('input_xml')
input_option += '-i %s %s' % (input_mode, input_xml)
elif input_mode in ['ova']:
raise exceptions.TestSkipError("Unsupported input mode: %s" %
input_mode)
else:
raise exceptions.TestError("Unknown input mode %s" % input_mode)
input_format = params.get("input_format", "")
input_allo_mode = params.get("input_allo_mode")
if input_format:
input_option += " -if %s" % input_format
if not status_error:
logging.info("Check image before convert")
check_image(disk_img, "format", input_format)
if input_allo_mode:
check_image(disk_img, "allocation", input_allo_mode)
# Build output options
output_option = ""
if output_mode:
output_option = "-o %s -os %s" % (output_mode, output_storage)
output_format = params.get("output_format")
if output_format and output_format != input_format:
output_option += " -of %s" % output_format
output_allo_mode = params.get("output_allo_mode")
if output_allo_mode:
output_option += " -oa %s" % output_allo_mode
# Build vdsm related options
if output_mode in ['vdsm', 'rhev']:
if not os.path.isdir(mnt_point):
os.mkdir(mnt_point)
if not utils_misc.mount(nfs_storage, mnt_point, "nfs"):
raise exceptions.TestError("Mount NFS Failed")
if output_mode == 'vdsm':
v2v_options += " --vdsm-image-uuid %s" % vdsm_image_uuid
v2v_options += " --vdsm-vol-uuid %s" % vdsm_vol_uuid
v2v_options += " --vdsm-vm-uuid %s" % vdsm_vm_uuid
v2v_options += " --vdsm-ovf-output %s" % vdsm_ovf_output
vdsm_domain_dir = os.path.join(mnt_point, fake_domain_uuid)
vdsm_image_dir = os.path.join(mnt_point, export_domain_uuid,
"images", vdsm_image_uuid)
vdsm_vm_dir = os.path.join(mnt_point, export_domain_uuid,
"master/vms", vdsm_vm_uuid)
# For vdsm_domain_dir, just create a dir to test BZ#1176591
os.makedirs(vdsm_domain_dir)
os.makedirs(vdsm_image_dir)
os.makedirs(vdsm_vm_dir)
# Output more messages except quiet mode
if checkpoint == 'quiet':
v2v_options += ' -q'
elif checkpoint not in [
'length_of_error', 'empty_nic_source_network',
'empty_nic_source_bridge'
]:
v2v_options += " -v -x"
# Prepare for libvirt unprivileged user session connection
if "qemu:///session" in v2v_options or no_root:
try:
pwd.getpwnam(v2v_user)
except KeyError:
# create new user
process.system("useradd %s" % v2v_user, ignore_status=True)
new_v2v_user = True
user_info = pwd.getpwnam(v2v_user)
logging.info("Convert to qemu:///session by user '%s'", v2v_user)
if input_mode == "disk":
# Copy image from souce and change the image owner and group
disk_path = os.path.join(data_dir.get_tmp_dir(),
os.path.basename(disk_img))
logging.info('Copy image file %s to %s', disk_img, disk_path)
shutil.copyfile(disk_img, disk_path)
input_option = string.replace(input_option, disk_img,
disk_path)
os.chown(disk_path, user_info.pw_uid, user_info.pw_gid)
elif not no_root:
raise exceptions.TestSkipError(
"Only support convert local disk")
# Setup ssh-agent access to xen hypervisor
if hypervisor == 'xen':
user = params.get("xen_host_user", "root")
passwd = params.get("xen_host_passwd", "redhat")
logging.info("set up ssh-agent access ")
ssh_key.setup_ssh_key(remote_host,
user=user,
port=22,
password=passwd)
utils_misc.add_identities_into_ssh_agent()
# Check if xen guest exists
uri = utils_v2v.Uri(hypervisor).get_uri(remote_host)
if not virsh.domain_exists(vm_name, uri=uri):
logging.error('VM %s not exists', vm_name)
# If the input format is not define, we need to either define
# the original format in the source metadata(xml) or use '-of'
# to force the output format, see BZ#1141723 for detail.
if '-of' not in v2v_options and checkpoint != 'xen_no_output_format':
v2v_options += ' -of %s' % params.get("default_output_format",
"qcow2")
# Create password file for access to ESX hypervisor
if hypervisor == 'esx':
vpx_passwd = params.get("vpx_passwd")
vpx_passwd_file = os.path.join(test.tmpdir, "vpx_passwd")
logging.info("Building ESX no password interactive verification.")
pwd_f = open(vpx_passwd_file, 'w')
pwd_f.write(vpx_passwd)
pwd_f.close()
output_option += " --password-file %s" % vpx_passwd_file
# Create libvirt dir pool
if output_mode == "libvirt":
create_pool()
if hypervisor in ['esx', 'xen'
] or input_mode in ['disk', 'libvirtxml']:
os.environ['LIBGUESTFS_BACKEND'] = 'direct'
if checkpoint in ['with_ic', 'without_ic']:
new_v2v_user = True
v2v_options += ' -on %s' % new_vm_name
create_pool(user_pool=True,
pool_name='src_pool',
pool_target='v2v_src_pool')
create_pool(user_pool=True)
logging.debug(virsh.pool_list(uri='qemu:///session'))
sh_install_vm = params.get('sh_install_vm')
if not sh_install_vm:
raise exceptions.TestError(
'Source vm installing script missing')
process.run('su - %s -c %s' % (v2v_user, sh_install_vm))
# Running virt-v2v command
cmd = "%s %s %s %s" % (utils_v2v.V2V_EXEC, input_option, output_option,
v2v_options)
if v2v_user:
cmd = su_cmd + "'%s'" % cmd
if checkpoint in ['dependency', 'no_dcpath']:
cmd = params.get('check_command')
# Set timeout to kill v2v process before conversion succeed
if checkpoint == 'disk_not_exist':
v2v_timeout = 30
# Get tail content of /var/log/messages
if checkpoint == 'tail_log':
params['tail_log'] = os.path.join(data_dir.get_tmp_dir(),
'tail_log')
params['tail'] = aexpect.Tail(command='tail -f /var/log/messages',
output_func=utils_misc.log_line,
output_params=(params['tail_log'], ))
cmd_result = process.run(cmd,
timeout=v2v_timeout,
verbose=True,
ignore_status=True)
if new_vm_name:
vm_name = new_vm_name
params['main_vm'] = new_vm_name
check_result(cmd, cmd_result, status_error)
finally:
if hypervisor == "xen":
process.run("ssh-agent -k")
if hypervisor == "esx":
process.run("rm -rf %s" % vpx_passwd_file)
for vdsm_dir in [vdsm_domain_dir, vdsm_image_dir, vdsm_vm_dir]:
if os.path.exists(vdsm_dir):
shutil.rmtree(vdsm_dir)
if os.path.exists(mnt_point):
utils_misc.umount(nfs_storage, mnt_point, "nfs")
os.rmdir(mnt_point)
if output_mode == "local":
image_name = vm_name + "-sda"
img_file = os.path.join(output_storage, image_name)
xml_file = img_file + ".xml"
for local_file in [img_file, xml_file]:
if os.path.exists(local_file):
os.remove(local_file)
if output_mode == "libvirt":
if "qemu:///session" in v2v_options or no_root:
cmd = su_cmd + "'virsh undefine %s'" % vm_name
try:
process.system(cmd)
except:
logging.error('Undefine "%s" failed', vm_name)
if no_root:
cleanup_pool(user_pool=True,
pool_name='src_pool',
pool_target='v2v_src_pool')
cleanup_pool(user_pool=True)
else:
virsh.remove_domain(vm_name)
cleanup_pool()
vmcheck_flag = params.get("vmcheck_flag")
if vmcheck_flag:
vmcheck = utils_v2v.VMCheck(test, params, env)
vmcheck.cleanup()
if new_v2v_user:
process.system("userdel -f %s" % v2v_user)
if backup_xml:
backup_xml.sync()
def umount(self):
"""
Umount the given mount point.
"""
return utils_misc.umount(self.mount_src, self.mount_dir, "nfs")
def run(test, params, env):
"""
Test command: virsh event and virsh qemu-monitor-event
1. Run virsh event/qemu-monitor-event in a new ShellSession
2. Trigger various events
3. Catch the return of virsh event and qemu-monitor-event, and check it.
"""
vms = []
if params.get("multi_vms") == "yes":
vms = env.get_all_vms()
else:
vm_name = params.get("main_vm")
vms.append(env.get_vm(vm_name))
event_name = params.get("event_name")
event_all_option = "yes" == params.get("event_all_option", "no")
event_list_option = "yes" == params.get("event_list_option", "no")
event_loop = "yes" == params.get("event_loop", "no")
event_timeout = params.get("event_timeout")
event_option = params.get("event_option", "")
status_error = "yes" == params.get("status_error", "no")
qemu_monitor_test = "yes" == params.get("qemu_monitor_test", "no")
signal_name = params.get("signal", None)
panic_model = params.get("panic_model")
addr_type = params.get("addr_type")
addr_iobase = params.get("addr_iobase")
disk_format = params.get("disk_format", "")
disk_prealloc = "yes" == params.get("disk_prealloc", "yes")
event_cmd = "event"
dump_path = '/var/lib/libvirt/qemu/dump'
part_format = params.get("part_format")
strict_order = "yes" == params.get("strict_order", "no")
if qemu_monitor_test:
event_cmd = "qemu-monitor-event"
events_list = params.get("events_list")
if events_list:
events_list = events_list.split(",")
else:
events_list = []
virsh_dargs = {'debug': True, 'ignore_status': True}
virsh_session = aexpect.ShellSession(virsh.VIRSH_EXEC)
for dom in vms:
if dom.is_alive():
dom.destroy()
vmxml_backup = []
for dom in vms:
vmxml = vm_xml.VMXML.new_from_inactive_dumpxml(dom.name)
vmxml_backup.append(vmxml.copy())
tmpdir = data_dir.get_tmp_dir()
mount_point = tmpdir
small_part = os.path.join(tmpdir, params.get("part_name", "io-error_part"))
def create_iface_xml():
"""
Create interface xml file
"""
iface = Interface("bridge")
iface.source = eval("{'bridge':'virbr0'}")
iface.model = "virtio"
logging.debug("Create new interface xml: %s", iface)
return iface
def add_disk(vm_name, init_source, target_device, extra_param, format=''):
"""
Add disk/cdrom for test vm
:param vm_name: guest name
:param init_source: source file
:param target_device: target of disk device
:param extra_param: additional arguments to command
:param format: init_source format(qcow2 or raw)
"""
if not os.path.exists(init_source):
disk_size = params.get("disk_size", "1G")
if format == "qcow2":
create_option = "" if not disk_prealloc else "-o preallocation=full"
process.run('qemu-img create -f qcow2 %s %s %s' % (init_source, disk_size, create_option),
shell=True, verbose=True)
elif format == "raw":
process.run('qemu-img create -f raw %s %s' % (init_source, disk_size),
shell=True, verbose=True)
else:
open(init_source, 'a').close()
if virsh.is_alive(vm_name) and 'cdrom' in extra_param:
virsh.destroy(vm_name)
if 'cdrom' in extra_param:
init_source = "''"
virsh.attach_disk(vm_name, init_source, target_device,
extra_param, **virsh_dargs)
vmxml_disk = vm_xml.VMXML.new_from_dumpxml(vm_name)
logging.debug("Current vmxml after adding disk is %s\n" % vmxml_disk)
def wait_for_shutoff(vm):
"""
Wait for the vm to reach state shutoff
:param vm: VM instance
"""
def is_shutoff():
state = vm.state()
logging.debug("Current state: %s", state)
return "shut off" in state
utils_misc.wait_for(is_shutoff, timeout=90, first=1,
step=1, text="Waiting for vm state to be shut off")
def prepare_vmxml_mem(vmxml):
"""
Prepare memory and numa settings in vmxml before hotplug dimm
param vmxml: guest current xml
"""
# Prepare memory settings
vmxml.max_mem_rt = int(params.get("max_mem"))
vmxml.max_mem_rt_slots = int(params.get("maxmem_slots"))
mem_unit = params.get("mem_unit")
vmxml.max_mem_rt_unit = mem_unit
current_mem = int(params.get("current_mem"))
vmxml.current_mem = current_mem
vmxml.current_mem_unit = mem_unit
vmxml.memory = int(params.get("memory"))
# Prepare numa settings in <cpu>
host_numa_node = utils_misc.NumaInfo()
host_numa_node_list = host_numa_node.online_nodes
numa_nodes = len(host_numa_node_list)
if numa_nodes == 0:
test.cancel("No host numa node available")
numa_dict = {}
numa_dict_list = []
cpu_idx = 0
for index in range(numa_nodes):
numa_dict['id'] = str(index)
numa_dict['memory'] = str(current_mem // numa_nodes)
numa_dict['unit'] = mem_unit
numa_dict['cpus'] = "%s-%s" % (str(cpu_idx), str(cpu_idx + 1))
cpu_idx += 2
numa_dict_list.append(numa_dict)
numa_dict = {}
vmxml.vcpu = numa_nodes * 2
vmxml_cpu = vm_xml.VMCPUXML()
vmxml_cpu.xml = "<cpu><numa/></cpu>"
vmxml_cpu.numa_cell = vmxml_cpu.dicts_to_cells(numa_dict_list)
logging.debug(vmxml_cpu.numa_cell)
vmxml.cpu = vmxml_cpu
vmxml.sync()
def trigger_events(dom, events_list=[]):
"""
Trigger various events in events_list
:param dom: the vm objects corresponding to the domain
:return: the expected output that virsh event command prints out
"""
expected_events_list = []
save_path = os.path.join(tmpdir, "%s_event.save" % dom.name)
print(dom.name)
xmlfile = dom.backup_xml()
new_disk = os.path.join(tmpdir, "%s_new_disk.img" % dom.name)
dest_path = os.path.join(data_dir.get_data_dir(), "copy")
try:
for event in events_list:
logging.debug("Current event is: %s", event)
if event in ['start', 'restore', 'create', 'edit', 'define',
'undefine', 'crash', 'device-removal-failed',
'watchdog', 'io-error']:
if dom.is_alive():
dom.destroy()
if event in ['create', 'define']:
dom.undefine()
else:
if not dom.is_alive():
dom.start()
dom.wait_for_login().close()
if event == "resume":
dom.pause()
if event == "undefine":
virsh.undefine(dom.name, **virsh_dargs)
expected_events_list.append("'lifecycle' for %s:"
" Undefined Removed")
elif event == "create":
virsh.create(xmlfile, **virsh_dargs)
expected_events_list.append("'lifecycle' for %s:"
" Resumed Unpaused")
expected_events_list.append("'lifecycle' for %s:"
" Started Booted")
elif event == "destroy":
virsh.destroy(dom.name, **virsh_dargs)
expected_events_list.append("'lifecycle' for %s:"
" Stopped Destroyed")
elif event == "define":
virsh.define(xmlfile, **virsh_dargs)
expected_events_list.append("'lifecycle' for %s:"
" Defined Added")
elif event == "start":
virsh.start(dom.name, **virsh_dargs)
expected_events_list.append("'lifecycle' for %s:"
" Resumed Unpaused")
expected_events_list.append("'lifecycle' for %s:"
" Started Booted")
dom.wait_for_login().close()
elif event == "suspend":
virsh.suspend(dom.name, **virsh_dargs)
expected_events_list.append("'lifecycle' for %s:"
" Suspended Paused")
if not libvirt_version.version_compare(5, 3, 0):
expected_events_list.append("'lifecycle' for %s:"
" Suspended Paused")
elif event == "resume":
virsh.resume(dom.name, **virsh_dargs)
expected_events_list.append("'lifecycle' for %s:"
" Resumed Unpaused")
elif event == "save":
virsh.save(dom.name, save_path, **virsh_dargs)
expected_events_list.append("'lifecycle' for %s:"
" Suspended Paused")
expected_events_list.append("'lifecycle' for %s:"
" Stopped Saved")
elif event == "restore":
if not os.path.exists(save_path):
logging.error("%s not exist", save_path)
else:
virsh.restore(save_path, **virsh_dargs)
expected_events_list.append("'lifecycle' for %s:"
" Started Restored")
expected_events_list.append("'lifecycle' for %s:"
" Resumed Snapshot")
elif event == "edit":
#Check whether 'description' element exists.
domxml = virsh.dumpxml(dom.name).stdout.strip()
find_desc = parseString(domxml).getElementsByTagName("description")
if find_desc == []:
#If not exists, add one for it.
logging.info("Adding <description> to guest")
virsh.desc(dom.name, "--config", "Added desc for testvm", **virsh_dargs)
#The edit operation is to delete 'description' element.
edit_cmd = [r":g/<description.*<\/description>/d"]
utlv.exec_virsh_edit(dom.name, edit_cmd)
expected_events_list.append("'lifecycle' for %s:"
" Defined Updated")
elif event == "shutdown":
if signal_name is None:
virsh.shutdown(dom.name, **virsh_dargs)
# Wait a few seconds for shutdown finish
time.sleep(3)
if utils_misc.compare_qemu_version(2, 9, 0):
#Shutdown reason distinguished from qemu_2.9.0-9
expected_events_list.append("'lifecycle' for %s:"
" Shutdown Finished after guest request")
else:
os.kill(dom.get_pid(), getattr(signal, signal_name))
if utils_misc.compare_qemu_version(2, 9, 0):
expected_events_list.append("'lifecycle' for %s:"
" Shutdown Finished after host request")
if not utils_misc.compare_qemu_version(2, 9, 0):
expected_events_list.append("'lifecycle' for %s:"
" Shutdown Finished")
wait_for_shutoff(dom)
expected_events_list.append("'lifecycle' for %s:"
" Stopped Shutdown")
elif event == "crash":
if not vmxml.xmltreefile.find('devices').findall('panic'):
# Set panic device
panic_dev = Panic()
panic_dev.model = panic_model
panic_dev.addr_type = addr_type
panic_dev.addr_iobase = addr_iobase
vmxml.add_device(panic_dev)
vmxml.on_crash = "coredump-restart"
vmxml.sync()
logging.info("Guest xml now is: %s", vmxml)
dom.start()
session = dom.wait_for_login()
# Stop kdump in the guest
session.cmd("systemctl stop kdump", ignore_all_errors=True)
# Enable sysRq
session.cmd("echo 1 > /proc/sys/kernel/sysrq")
try:
# Crash the guest
session.cmd("echo c > /proc/sysrq-trigger", timeout=90)
except (ShellTimeoutError, ShellProcessTerminatedError) as details:
logging.info(details)
session.close()
expected_events_list.append("'lifecycle' for %s:"
" Crashed Panicked")
expected_events_list.append("'lifecycle' for %s:"
" Resumed Unpaused")
elif event == "reset":
virsh.reset(dom.name, **virsh_dargs)
expected_events_list.append("'reboot' for %s")
elif event == "vcpupin":
virsh.vcpupin(dom.name, '0', '0', **virsh_dargs)
expected_events_list.append("'tunable' for %s:"
"\n\tcputune.vcpupin0: 0")
elif event == "emulatorpin":
virsh.emulatorpin(dom.name, '0', **virsh_dargs)
expected_events_list.append("'tunable' for %s:"
"\n\tcputune.emulatorpin: 0")
elif event == "setmem":
mem_size = int(params.get("mem_size", 512000))
virsh.setmem(dom.name, mem_size, **virsh_dargs)
expected_events_list.append("'balloon-change' for %s:")
elif event == "device-added-removed":
add_disk(dom.name, new_disk, 'vdb', '')
expected_events_list.append("'device-added' for %s:"
" virtio-disk1")
virsh.detach_disk(dom.name, 'vdb', **virsh_dargs)
expected_events_list.append("'device-removed' for %s:"
" virtio-disk1")
iface_xml_obj = create_iface_xml()
iface_xml_obj.xmltreefile.write()
virsh.detach_device(dom.name, iface_xml_obj.xml, **virsh_dargs)
expected_events_list.append("'device-removed' for %s:"
" net0")
time.sleep(2)
virsh.attach_device(dom.name, iface_xml_obj.xml, **virsh_dargs)
expected_events_list.append("'device-added' for %s:"
" net0")
elif event == "block-threshold":
add_disk(dom.name, new_disk, 'vdb', '', format=disk_format)
logging.debug(process.run('qemu-img info %s -U' % new_disk))
virsh.domblkthreshold(vm_name, 'vdb', '100M')
session = dom.wait_for_login()
session.cmd("mkfs.ext4 /dev/vdb && mount /dev/vdb /mnt && ls /mnt && "
"dd if=/dev/urandom of=/mnt/bigfile bs=1M count=300 && sync")
time.sleep(5)
session.close()
expected_events_list.append("'block-threshold' for %s:"
" dev: vdb(%s) 104857600 29368320")
virsh.detach_disk(dom.name, 'vdb', **virsh_dargs)
elif event == "change-media":
target_device = "hdc"
device_target_bus = params.get("device_target_bus", "ide")
disk_blk = vm_xml.VMXML.get_disk_blk(dom.name)
logging.info("disk_blk %s", disk_blk)
if target_device not in disk_blk:
logging.info("Adding cdrom to guest")
if dom.is_alive():
dom.destroy()
add_disk(dom.name, new_disk, target_device,
("--type cdrom --sourcetype file --driver qemu " +
"--config --targetbus %s" % device_target_bus))
dom.start()
all_options = new_disk + " --insert"
virsh.change_media(dom.name, target_device,
all_options, **virsh_dargs)
expected_events_list.append("'tray-change' for %s disk" + " .*%s.*:" % device_target_bus +
" opened")
expected_events_list.append("'tray-change' for %s disk" + " .*%s.*:" % device_target_bus +
" closed")
all_options = new_disk + " --eject"
virsh.change_media(dom.name, target_device,
all_options, **virsh_dargs)
expected_events_list.append("'tray-change' for %s disk" + " .*%s.*:" % device_target_bus +
" opened")
elif event == "hwclock":
session = dom.wait_for_login()
try:
session.cmd("hwclock --systohc", timeout=60)
except (ShellTimeoutError, ShellProcessTerminatedError) as details:
logging.info(details)
session.close()
expected_events_list.append("'rtc-change' for %s:")
elif event == "metadata_set":
metadata_uri = params.get("metadata_uri")
metadata_key = params.get("metadata_key")
metadata_value = params.get("metadata_value")
virsh.metadata(dom.name,
metadata_uri,
options="",
key=metadata_key,
new_metadata=metadata_value,
**virsh_dargs)
expected_events_list.append("'metadata-change' for %s: "
"element http://app.org/")
elif event == "metadata_edit":
metadata_uri = "http://herp.derp/"
metadata_key = "herp"
metadata_value = "<derp xmlns:foobar='http://foo.bar/'>foo<bar></bar></derp>"
virsh_cmd = r"virsh metadata %s --uri %s --key %s %s"
virsh_cmd = virsh_cmd % (dom.name, metadata_uri,
metadata_key, "--edit")
session = aexpect.ShellSession("sudo -s")
logging.info("Running command: %s", virsh_cmd)
try:
session.sendline(virsh_cmd)
session.sendline(r":insert")
session.sendline(metadata_value)
session.sendline(".")
session.send('ZZ')
remote.handle_prompts(session, None, None, r"[\#\$]\s*$",
debug=True, timeout=60)
except Exception as e:
test.error("Error occured: %s" % e)
session.close()
# Check metadata after edit
virsh.metadata(dom.name,
metadata_uri,
options="",
key=metadata_key,
**virsh_dargs)
expected_events_list.append("'metadata-change' for %s: "
"element http://app.org/")
elif event == "metadata_remove":
virsh.metadata(dom.name,
metadata_uri,
options="--remove",
key=metadata_key,
**virsh_dargs)
expected_events_list.append("'metadata-change' for %s: "
"element http://app.org/")
elif event == "blockcommit":
disk_path = dom.get_blk_devices()['vda']['source']
virsh.snapshot_create_as(dom.name, "s1 --disk-only --no-metadata", **virsh_dargs)
snapshot_path = dom.get_blk_devices()['vda']['source']
virsh.blockcommit(dom.name, "vda", "--active --pivot", **virsh_dargs)
expected_events_list.append("'block-job' for %s: "
"Active Block Commit for " + "%s" % snapshot_path + " ready")
expected_events_list.append("'block-job-2' for %s: "
"Active Block Commit for vda ready")
expected_events_list.append("'block-job' for %s: "
"Active Block Commit for " + "%s" % disk_path + " completed")
expected_events_list.append("'block-job-2' for %s: "
"Active Block Commit for vda completed")
os.unlink(snapshot_path)
elif event == "blockcopy":
disk_path = dom.get_blk_devices()['vda']['source']
dom.undefine()
virsh.blockcopy(dom.name, "vda", dest_path, "--pivot", **virsh_dargs)
expected_events_list.append("'block-job' for %s: "
"Block Copy for " + "%s" % disk_path + " ready")
expected_events_list.append("'block-job-2' for %s: "
"Block Copy for vda ready")
expected_events_list.append("'block-job' for %s: "
"Block Copy for " + "%s" % dest_path + " completed")
expected_events_list.append("'block-job-2' for %s: "
"Block Copy for vda completed")
elif event == "detach-dimm":
prepare_vmxml_mem(vmxml)
tg_size = params.get("dimm_size")
tg_sizeunit = params.get("dimm_unit")
dimm_xml = utils_hotplug.create_mem_xml(tg_size, None, None, tg_sizeunit)
virsh.attach_device(dom.name, dimm_xml.xml,
flagstr="--config", **virsh_dargs)
vmxml_dimm = vm_xml.VMXML.new_from_dumpxml(dom.name)
logging.debug("Current vmxml with plugged dimm dev is %s\n" % vmxml_dimm)
virsh.start(dom.name, **virsh_dargs)
dom.wait_for_login().close()
result = virsh.detach_device(dom.name, dimm_xml.xml, debug=True, ignore_status=True)
expected_fails = params.get("expected_fails")
utlv.check_result(result, expected_fails)
vmxml_live = vm_xml.VMXML.new_from_dumpxml(dom.name)
logging.debug("Current vmxml after hot-unplug dimm is %s\n" % vmxml_live)
expected_events_list.append("'device-removal-failed' for %s: dimm0")
elif event == "watchdog":
vmxml.remove_all_device_by_type('watchdog')
watchdog_dev = Watchdog()
watchdog_dev.model_type = params.get("watchdog_model")
action = params.get("action")
watchdog_dev.action = action
vmxml.add_device(watchdog_dev)
vmxml.sync()
logging.debug("Current vmxml with watchdog dev is %s\n" % vmxml)
virsh.start(dom.name, **virsh_dargs)
session = dom.wait_for_login()
try:
session.cmd("echo 0 > /dev/watchdog")
except (ShellTimeoutError, ShellProcessTerminatedError) as details:
test.fail("Failed to trigger watchdog: %s" % details)
session.close()
# watchdog acts slowly, waiting for it.
time.sleep(30)
expected_events_list.append("'watchdog' for %s: " + "%s" % action)
if action == 'pause':
expected_events_list.append("'lifecycle' for %s: Suspended Watchdog")
virsh.resume(dom.name, **virsh_dargs)
else:
# action == 'reset'
expected_events_list.append("'reboot' for %s")
elif event == "io-error":
part_size = params.get("part_size")
resume_event = params.get("resume_event")
suspend_event = params.get("suspend_event")
process.run("truncate -s %s %s" % (part_size, small_part), shell=True)
utlv.mkfs(small_part, part_format)
utils_misc.mount(small_part, mount_point, None)
add_disk(dom.name, new_disk, 'vdb', '--subdriver qcow2 --config', 'qcow2')
dom.start()
session = dom.wait_for_login()
session.cmd("mkfs.ext4 /dev/vdb && mount /dev/vdb /mnt && ls /mnt && "
"dd if=/dev/zero of=/mnt/test.img bs=1M count=50", ignore_all_errors=True)
time.sleep(5)
session.close()
expected_events_list.append("'io-error' for %s: " + "%s" % new_disk + r" \(virtio-disk1\) pause")
expected_events_list.append("'io-error-reason' for %s: " + "%s" % new_disk + r" \(virtio-disk1\) pause due to enospc")
expected_events_list.append(suspend_event)
process.run("df -hT")
virsh.resume(dom.name, **virsh_dargs)
time.sleep(5)
expected_events_list.append(resume_event)
expected_events_list.append("'io-error' for %s: " + "%s" % new_disk + r" \(virtio-disk1\) pause")
expected_events_list.append("'io-error-reason' for %s: " + "%s" % new_disk + r" \(virtio-disk1\) pause due to enospc")
expected_events_list.append(suspend_event)
ret = virsh.domstate(dom.name, "--reason", **virsh_dargs)
if ret.stdout.strip() != "paused (I/O error)":
test.fail("Domain state should still be paused due to I/O error!")
else:
test.error("Unsupported event: %s" % event)
# Event may not received immediately
time.sleep(3)
finally:
if os.path.exists(save_path):
os.unlink(save_path)
if os.path.exists(new_disk):
os.unlink(new_disk)
if os.path.exists(dest_path):
os.unlink(dest_path)
return [(dom.name, event) for event in expected_events_list]
def check_output(output, expected_events_list):
"""
Check received domain event in output.
:param output: The virsh shell output, such as:
Welcome to virsh, the virtualization interactive terminal.
Type: 'help' for help with commands
'quit' to quit
virsh # event 'lifecycle' for domain avocado-vt-vm1: Started Booted
events received: 1
virsh #
:param expected_events_list: A list of expected events
"""
logging.debug("Actual events: %s", output)
event_idx = 0
for dom_name, event in expected_events_list:
if event in expected_events_list[0] and not strict_order:
event_idx = 0
if re.search("block-threshold", event):
event_str = "block-threshold"
else:
event_str = "event " + event % ("domain '%s'" % dom_name)
logging.info("Expected event: %s", event_str)
match = re.search(event_str, output[event_idx:])
if match:
event_idx = event_idx + match.start(0) + len(match.group(0))
continue
else:
test.fail("Not find expected event:%s. Is your "
"guest too slow to get started in %ss?" %
(event_str, event_timeout))
# Extra event check for io-error resume
if events_list == ['io-error']:
event_str = "event 'lifecycle' for domain " + vm_name + ": Resumed Unpaused"
if re.search(event_str, output[event_idx:]):
test.fail("Extra 'resume' occurred after I/O error!")
try:
# Set vcpu placement to static to avoid emulatorpin fail
vmxml.placement = 'static'
# Using a large memeoy(>1048576) to avoid setmem fail
vmxml.max_mem = 2097152
vmxml.current_mem = 2097152
vmxml.sync()
if event_all_option and not qemu_monitor_test:
event_option += " --all"
if event_list_option:
event_option += " --list"
if event_loop:
event_option += " --loop"
if not status_error and not event_list_option:
event_cmd += " %s" % event_option
if event_name and not qemu_monitor_test:
event_cmd += " --event %s" % event_name
if event_timeout:
event_cmd += " --timeout %s" % event_timeout
# Run the command in a new virsh session, then waiting for
# various events
logging.info("Sending '%s' to virsh shell", event_cmd)
virsh_session.sendline(event_cmd)
elif qemu_monitor_test:
result = virsh.qemu_monitor_event(event=event_name,
event_timeout=event_timeout,
options=event_option,
**virsh_dargs)
utlv.check_exit_status(result, status_error)
else:
result = virsh.event(event=event_name,
event_timeout=event_timeout,
options=event_option, **virsh_dargs)
utlv.check_exit_status(result, status_error)
if not status_error:
if not event_list_option:
expected_events_list = []
virsh_dargs['ignore_status'] = False
for dom in vms:
expected_events_list.extend(trigger_events(dom, events_list))
if event_timeout:
# Make sure net-event will timeout on time
time.sleep(int(event_timeout))
elif event_loop:
virsh_session.send_ctrl("^C")
time.sleep(5)
ret_output = virsh_session.get_stripped_output()
if qemu_monitor_test:
# Not check for qemu-monitor-event output
expected_events_list = []
check_output(ret_output, expected_events_list)
finally:
for dom in vms:
if dom.is_alive():
dom.destroy()
virsh_session.close()
for xml in vmxml_backup:
xml.sync()
if os.path.exists(dump_path):
shutil.rmtree(dump_path)
os.mkdir(dump_path)
if utils_misc.is_mounted("/dev/loop0", mount_point, part_format):
utils_misc.umount("/dev/loop0", mount_point, part_format)
if os.path.exists(small_part):
os.unlink(small_part)
def run(test, params, env):
"""
Convert specific esx guest
"""
for v in list(params.values()):
if "V2V_EXAMPLE" in v:
test.cancel("Please set real value for %s" % v)
if utils_v2v.V2V_EXEC is None:
raise ValueError('Missing command: virt-v2v')
version_requried = params.get("version_requried")
vpx_hostname = params.get('vpx_hostname')
vpx_passwd = params.get("vpx_password")
esxi_host = esx_ip = params.get('esx_hostname')
vpx_dc = params.get('vpx_dc')
vm_name = params.get('main_vm')
output_mode = params.get('output_mode')
pool_name = params.get('pool_name', 'v2v_test')
pool_type = params.get('pool_type', 'dir')
pool_target = params.get('pool_target_path', 'v2v_pool')
pvt = libvirt.PoolVolumeTest(test, params)
v2v_timeout = int(params.get('v2v_timeout', 1200))
v2v_cmd_timeout = int(params.get('v2v_cmd_timeout', 18000))
v2v_opts = '-v -x' if params.get('v2v_debug', 'on') == 'on' else ''
if params.get("v2v_opts"):
# Add a blank by force
v2v_opts += ' ' + params.get("v2v_opts")
status_error = 'yes' == params.get('status_error', 'no')
address_cache = env.get('address_cache')
checkpoint = params.get('checkpoint', '')
skip_vm_check = params.get('skip_vm_check', 'no')
skip_reason = params.get('skip_reason')
error_list = []
remote_host = vpx_hostname
# For VDDK
input_transport = params.get("input_transport")
vddk_libdir = params.get('vddk_libdir')
# nfs mount source
vddk_libdir_src = params.get('vddk_libdir_src')
vddk_thumbprint = params.get('vddk_thumbprint')
src_uri_type = params.get('src_uri_type')
esxi_password = params.get('esxi_password')
json_disk_pattern = params.get('json_disk_pattern')
# For construct rhv-upload option in v2v cmd
output_method = params.get("output_method")
rhv_upload_opts = params.get("rhv_upload_opts")
storage_name = params.get('storage_name')
os_pool = os_storage = params.get('output_storage', 'default')
# for get ca.crt file from ovirt engine
rhv_passwd = params.get("rhv_upload_passwd")
rhv_passwd_file = params.get("rhv_upload_passwd_file")
ovirt_engine_passwd = params.get("ovirt_engine_password")
ovirt_hostname = params.get("ovirt_engine_url").split(
'/')[2] if params.get("ovirt_engine_url") else None
ovirt_ca_file_path = params.get("ovirt_ca_file_path")
local_ca_file_path = params.get("local_ca_file_path")
os_version = params.get('os_version')
os_type = params.get('os_type')
virtio_win_path = params.get('virtio_win_path')
# qemu-guest-agent path in virtio-win or rhv-guest-tools-iso
qa_path = params.get('qa_path')
# download url of qemu-guest-agent
qa_url = params.get('qa_url')
v2v_sasl = None
# default values for v2v_cmd
auto_clean = True
cmd_only = False
def log_fail(msg):
"""
Log error and update error list
"""
logging.error(msg)
error_list.append(msg)
def check_device_exist(check, virsh_session_id):
"""
Check if device exist after convertion
"""
xml = virsh.dumpxml(vm_name, session_id=virsh_session_id).stdout
if check == 'cdrom':
if "device='cdrom'" not in xml:
log_fail('CDROM no longer exists')
def check_vmtools(vmcheck, check):
"""
Check whether vmware tools packages have been removed,
or vmware-tools service has stopped
:param vmcheck: VMCheck object for vm checking
:param check: Checkpoint of different cases
:return: None
"""
if check == 'vmtools':
logging.info('Check if packages been removed')
pkgs = vmcheck.session.cmd('rpm -qa').strip()
removed_pkgs = params.get('removed_pkgs').strip().split(',')
if not removed_pkgs:
test.error('Missing param "removed_pkgs"')
for pkg in removed_pkgs:
if pkg in pkgs:
log_fail('Package "%s" not removed' % pkg)
elif check == 'vmtools_service':
logging.info('Check if service stopped')
vmtools_service = params.get('service_name')
status = utils_misc.get_guest_service_status(
vmcheck.session, vmtools_service)
logging.info('Service %s status: %s', vmtools_service, status)
if status != 'inactive':
log_fail('Service "%s" is not stopped' % vmtools_service)
def check_modprobe(vmcheck):
"""
Check whether content of /etc/modprobe.conf meets expectation
"""
content = vmcheck.session.cmd('cat /etc/modprobe.conf').strip()
logging.debug(content)
cfg_content = params.get('cfg_content')
if not cfg_content:
test.error('Missing content for search')
logging.info('Search "%s" in /etc/modprobe.conf', cfg_content)
pattern = r'\s+'.join(cfg_content.split())
if not re.search(pattern, content):
log_fail('Not found "%s"' % cfg_content)
def check_device_map(vmcheck):
"""
Check if the content of device.map meets expectation.
"""
logging.info(vmcheck.session.cmd('fdisk -l').strip())
device_map = params.get('device_map_path')
content = vmcheck.session.cmd('cat %s' % device_map)
logging.debug('Content of device.map:\n%s', content)
logging.info('Found device: %d', content.count('/dev/'))
logging.info('Found virtio device: %d', content.count('/dev/vd'))
if content.count('/dev/') != content.count('/dev/vd'):
log_fail('Content of device.map not correct')
else:
logging.info('device.map has been remaped to "/dev/vd*"')
def check_resume_swap(vmcheck):
"""
Check the content of grub files meet expectation.
"""
if os_version == 'rhel7':
chkfiles = [
'/etc/default/grub', '/boot/grub2/grub.cfg', '/etc/grub2.cfg'
]
if os_version == 'rhel6':
chkfiles = ['/boot/grub/grub.conf', '/etc/grub.conf']
for file_i in chkfiles:
status, content = vmcheck.run_cmd('cat %s' % file_i)
if status != 0:
log_fail('%s does not exist' % file_i)
resume_dev_count = content.count('resume=/dev/')
if resume_dev_count == 0 or resume_dev_count != content.count(
'resume=/dev/vd'):
reason = 'Maybe the VM\'s swap pariton is lvm'
log_fail('Content of %s is not correct or %s' %
(file_i, reason))
content = vmcheck.session.cmd('cat /proc/cmdline')
logging.debug('Content of /proc/cmdline:\n%s', content)
if 'resume=/dev/vd' not in content:
log_fail('Content of /proc/cmdline is not correct')
def check_rhev_file_exist(vmcheck):
"""
Check if rhev files exist
"""
file_path = {
'rhev-apt.exe': r'C:\rhev-apt.exe',
'rhsrvany.exe':
r'"C:\Program Files\Guestfs\Firstboot\rhsrvany.exe"'
}
for key in file_path:
status = vmcheck.session.cmd_status('dir %s' % file_path[key])
if status == 0:
logging.info('%s exists' % key)
else:
log_fail('%s does not exist after convert to rhv' % key)
def check_file_architecture(vmcheck):
"""
Check the 3rd party module info
:param vmcheck: VMCheck object for vm checking
"""
content = vmcheck.session.cmd('uname -r').strip()
status = vmcheck.session.cmd_status(
'rpm -qf /lib/modules/%s/fileaccess/fileaccess_mod.ko ' % content)
if status == 0:
log_fail('3rd party module info is not correct')
else:
logging.info(
'file /lib/modules/%s/fileaccess/fileaccess_mod.ko is not owned by any package'
% content)
def check_windows_ogac(vmcheck):
"""
Check qemu-guest-agent service in VM
:param vmcheck: VMCheck object for vm checking
"""
try:
res = utils_misc.wait_for(lambda: re.search(
'running', vmcheck.get_service_info('qemu-ga'), re.I),
300,
step=30)
except ShellProcessTerminatedError:
# Windows guest may reboot after installing qemu-ga service
logging.debug('Windows guest is rebooting')
if vmcheck.session:
vmcheck.session.close()
vmcheck.session = None
vmcheck.create_session()
res = utils_misc.wait_for(lambda: re.search(
'running', vmcheck.get_service_info('qemu-ga'), re.I),
300,
step=30)
if not res:
test.fail('Not found running qemu-ga service')
def check_linux_ogac(vmcheck):
"""
Check qemu-guest-agent service in VM
:param vmcheck: VMCheck object for vm checking
"""
def get_pkgs(pkg_path):
"""
Get all qemu-guest-agent pkgs
"""
pkgs = []
for _, _, files in os.walk(pkg_path):
for file_name in files:
pkgs.append(file_name)
return pkgs
def get_pkg_version_vm():
"""
Get qemu-guest-agent version in VM
"""
vender = vmcheck.get_vm_os_vendor()
if vender in ['Ubuntu', 'Debian']:
cmd = 'dpkg -l qemu-guest-agent'
else:
cmd = 'rpm -q qemu-guest-agent'
_, output = vmcheck.run_cmd(cmd)
pkg_ver_ptn = [
r'qemu-guest-agent +[0-9]+:(.*?dfsg.*?) +',
r'qemu-guest-agent-(.*?)\.x86_64'
]
for ptn in pkg_ver_ptn:
if re.search(ptn, output):
return re.search(ptn, output).group(1)
return ''
if os.path.isfile(os.getenv('VIRTIO_WIN')):
mount_point = utils_v2v.v2v_mount(os.getenv('VIRTIO_WIN'),
'rhv_tools_setup_iso',
fstype='iso9660')
export_path = params['tmp_mount_point'] = mount_point
else:
export_path = os.getenv('VIRTIO_WIN')
qemu_guest_agent_dir = os.path.join(export_path, qa_path)
all_pkgs = get_pkgs(qemu_guest_agent_dir)
logging.debug('The installing qemu-guest-agent is: %s' % all_pkgs)
vm_pkg_ver = get_pkg_version_vm()
logging.debug('qemu-guest-agent verion in vm: %s' % vm_pkg_ver)
# If qemu-guest-agent version in VM is higher than the pkg in qemu-guest-agent-iso,
# v2v will not update the qemu-guest-agent version and report a warning.
#
# e.g.
# virt-v2v: warning: failed to install QEMU Guest Agent: command: package
# qemu-guest-agent-10:2.12.0-3.el7.x86_64 (which is newer than
# qemu-guest-agent-10:2.12.0-2.el7.x86_64) is already installed
if not any([vm_pkg_ver in pkg for pkg in all_pkgs]):
logging.debug(
'Wrong qemu-guest-agent version, maybe it is higher than package version in ISO'
)
logging.info(
'Unexpected qemu-guest-agent version, set v2v log checking')
expect_msg_ptn = r'virt-v2v: warning: failed to install QEMU Guest Agent.*?is newer than.*? is already installed'
params.update({'msg_content': expect_msg_ptn, 'expect_msg': 'yes'})
# Check the service status of qemu-guest-agent in VM
status_ptn = r'Active: active \(running\)|qemu-ga \(pid +[0-9]+\) is running'
cmd = 'service qemu-ga status;systemctl status qemu-guest-agent'
_, output = vmcheck.run_cmd(cmd)
if not re.search(status_ptn, output):
log_fail('qemu-guest-agent service exception')
def check_ubuntools(vmcheck):
"""
Check open-vm-tools, ubuntu-server in VM
:param vmcheck: VMCheck object for vm checking
"""
logging.info('Check if open-vm-tools service stopped')
status = utils_misc.get_guest_service_status(vmcheck.session,
'open-vm-tools')
logging.info('Service open-vm-tools status: %s', status)
if status != 'inactive':
log_fail('Service open-vm-tools is not stopped')
else:
logging.info('Check if the ubuntu-server exist')
content = vmcheck.session.cmd('dpkg -s ubuntu-server')
if 'install ok installed' in content:
logging.info('ubuntu-server has not been removed.')
else:
log_fail('ubuntu-server has been removed')
def global_pem_setup(f_pem):
"""
Setup global rhv server ca
:param f_pem: ca file path
"""
ca_anchors_dir = '/etc/pki/ca-trust/source/anchors'
shutil.copy(f_pem, ca_anchors_dir)
process.run('update-ca-trust extract', shell=True)
os.unlink(os.path.join(ca_anchors_dir, os.path.basename(f_pem)))
def global_pem_cleanup():
"""
Cleanup global rhv server ca
"""
process.run('update-ca-trust extract', shell=True)
def cmd_remove_option(cmd, opt_pattern):
"""
Remove an option from cmd
:param cmd: the cmd
:param opt_pattern: a pattern stands for the option
"""
for item in re.findall(opt_pattern, cmd):
cmd = cmd.replace(item, '').strip()
return cmd
def find_net(bridge_name):
"""
Find which network use specified bridge
:param bridge_name: bridge name you want to find
"""
net_list = virsh.net_state_dict(only_names=True)
net_name = ''
if len(net_list):
for net in net_list:
net_info = virsh.net_info(net).stdout.strip()
search = re.search(r'Bridge:\s+(\S+)', net_info)
if search:
if bridge_name == search.group(1):
net_name = net
else:
logging.info('Conversion server has no network')
return net_name
def destroy_net(net_name):
"""
destroy network in conversion server
"""
if virsh.net_state_dict()[net_name]['active']:
logging.info("Remove network %s in conversion server", net_name)
virsh.net_destroy(net_name)
if virsh.net_state_dict()[net_name]['autostart']:
virsh.net_autostart(net_name, "--disable")
output = virsh.net_list("--all").stdout.strip()
logging.info(output)
def start_net(net_name):
"""
start network in conversion server
"""
logging.info("Recover network %s in conversion server", net_name)
virsh.net_autostart(net_name)
if not virsh.net_state_dict()[net_name]['active']:
virsh.net_start(net_name)
output = virsh.net_list("--all").stdout.strip()
logging.info(output)
def check_result(result, status_error):
"""
Check virt-v2v command result
"""
def vm_check(status_error):
"""
Checking the VM
"""
if status_error:
return
if output_mode == 'json' and not check_json_output(params):
test.fail('check json output failed')
if output_mode == 'local' and not check_local_output(params):
test.fail('check local output failed')
if output_mode in ['null', 'json', 'local']:
return
# vmchecker must be put before skip_vm_check in order to clean up
# the VM.
vmchecker = VMChecker(test, params, env)
params['vmchecker'] = vmchecker
if skip_vm_check == 'yes':
logging.info('Skip checking vm after conversion: %s' %
skip_reason)
return
if output_mode == 'rhev':
if not utils_v2v.import_vm_to_ovirt(
params, address_cache, timeout=v2v_timeout):
test.fail('Import VM failed')
elif output_mode == 'libvirt':
virsh.start(vm_name, debug=True)
# Check guest following the checkpoint document after convertion
logging.info('Checking common checkpoints for v2v')
if checkpoint == 'ogac':
# windows guests will reboot at any time after qemu-ga is
# installed. The process cannot be controled. In order to
# don't break vmchecker.run() process, It's better to put
# check_windows_ogac before vmchecker.run(). Because in
# check_windows_ogac, it waits until rebooting completes.
vmchecker.checker.create_session()
if os_type == 'windows':
check_windows_ogac(vmchecker.checker)
else:
check_linux_ogac(vmchecker.checker)
ret = vmchecker.run()
if len(ret) == 0:
logging.info("All common checkpoints passed")
# Check specific checkpoints
if checkpoint == 'cdrom':
virsh_session = utils_sasl.VirshSessionSASL(params)
virsh_session_id = virsh_session.get_id()
check_device_exist('cdrom', virsh_session_id)
virsh_session.close()
if checkpoint.startswith('vmtools'):
check_vmtools(vmchecker.checker, checkpoint)
if checkpoint == 'modprobe':
check_modprobe(vmchecker.checker)
if checkpoint == 'device_map':
check_device_map(vmchecker.checker)
if checkpoint == 'resume_swap':
check_resume_swap(vmchecker.checker)
if checkpoint == 'rhev_file':
check_rhev_file_exist(vmchecker.checker)
if checkpoint == 'file_architecture':
check_file_architecture(vmchecker.checker)
if checkpoint == 'ubuntu_tools':
check_ubuntools(vmchecker.checker)
if checkpoint == 'without_default_net':
if virsh.net_state_dict()[net_name]['active']:
log_fail("Bridge virbr0 already started during conversion")
# Merge 2 error lists
error_list.extend(vmchecker.errors)
libvirt.check_exit_status(result, status_error)
output = result.stdout_text + result.stderr_text
# VM or local output checking
vm_check(status_error)
# Log checking
log_check = utils_v2v.check_log(params, output)
if log_check:
log_fail(log_check)
if len(error_list):
test.fail('%d checkpoints failed: %s' %
(len(error_list), error_list))
try:
if version_requried and not utils_v2v.compare_version(
version_requried):
test.cancel("Testing requries version: %s" % version_requried)
v2v_params = {
'hostname': remote_host,
'hypervisor': 'esx',
'main_vm': vm_name,
'vpx_dc': vpx_dc,
'esx_ip': esx_ip,
'new_name': vm_name + utils_misc.generate_random_string(4),
'v2v_opts': v2v_opts,
'input_mode': 'libvirt',
'os_storage': os_storage,
'os_pool': os_pool,
'network': params.get('network'),
'bridge': params.get('bridge'),
'target': params.get('target'),
'password': vpx_passwd if src_uri_type != 'esx' else esxi_password,
'input_transport': input_transport,
'vcenter_host': vpx_hostname,
'vcenter_password': vpx_passwd,
'vddk_thumbprint': vddk_thumbprint,
'vddk_libdir': vddk_libdir,
'vddk_libdir_src': vddk_libdir_src,
'src_uri_type': src_uri_type,
'esxi_password': esxi_password,
'esxi_host': esxi_host,
'output_method': output_method,
'os_storage_name': storage_name,
'rhv_upload_opts': rhv_upload_opts,
'oo_json_disk_pattern': json_disk_pattern,
'params': params
}
os.environ['LIBGUESTFS_BACKEND'] = 'direct'
v2v_uri = utils_v2v.Uri('esx')
remote_uri = v2v_uri.get_uri(remote_host, vpx_dc, esx_ip)
# Create password file for access to ESX hypervisor
vpx_passwd_file = params.get("vpx_passwd_file")
with open(vpx_passwd_file, 'w') as pwd_f:
if src_uri_type == 'esx':
pwd_f.write(esxi_password)
else:
pwd_f.write(vpx_passwd)
v2v_params['v2v_opts'] += " -ip %s" % vpx_passwd_file
if params.get('output_format'):
v2v_params.update({'of_format': params['output_format']})
# Rename guest with special name while converting to rhev
if '#' in vm_name and output_mode == 'rhev':
v2v_params['new_name'] = v2v_params['new_name'].replace('#', '_')
# Create SASL user on the ovirt host
if output_mode == 'rhev':
# create different sasl_user name for different job
params.update({
'sasl_user':
params.get("sasl_user") + utils_misc.generate_random_string(3)
})
logging.info('sals user name is %s' % params.get("sasl_user"))
user_pwd = "[['%s', '%s']]" % (params.get("sasl_user"),
params.get("sasl_pwd"))
v2v_sasl = utils_sasl.SASL(sasl_user_pwd=user_pwd)
v2v_sasl.server_ip = params.get("remote_ip")
v2v_sasl.server_user = params.get('remote_user')
v2v_sasl.server_pwd = params.get('remote_pwd')
v2v_sasl.setup(remote=True)
logging.debug('A SASL session %s was created', v2v_sasl)
if output_method == 'rhv_upload':
# Create password file for '-o rhv_upload' to connect to ovirt
with open(rhv_passwd_file, 'w') as f:
f.write(rhv_passwd)
# Copy ca file from ovirt to local
remote.scp_from_remote(ovirt_hostname, 22, 'root',
ovirt_engine_passwd, ovirt_ca_file_path,
local_ca_file_path)
# Create libvirt dir pool
if output_mode == 'libvirt':
pvt.pre_pool(pool_name, pool_type, pool_target, '')
if checkpoint == 'root_ask':
v2v_params['v2v_opts'] += ' --root ask'
v2v_params['custom_inputs'] = params.get('choice', '2')
if checkpoint.startswith('root_') and checkpoint != 'root_ask':
root_option = params.get('root_option')
v2v_params['v2v_opts'] += ' --root %s' % root_option
if checkpoint == 'with_proxy':
http_proxy = params.get('esx_http_proxy')
https_proxy = params.get('esx_https_proxy')
logging.info('Set http_proxy=%s, https_proxy=%s', http_proxy,
https_proxy)
os.environ['http_proxy'] = http_proxy
os.environ['https_proxy'] = https_proxy
if checkpoint == 'ogac':
os.environ['VIRTIO_WIN'] = virtio_win_path
if not os.path.exists(os.getenv('VIRTIO_WIN')):
test.fail('%s does not exist' % os.getenv('VIRTIO_WIN'))
if os.path.isdir(os.getenv('VIRTIO_WIN')) and os_type == 'linux':
export_path = os.getenv('VIRTIO_WIN')
qemu_guest_agent_dir = os.path.join(export_path, qa_path)
if not os.path.exists(qemu_guest_agent_dir) and os.access(
export_path, os.W_OK) and qa_url:
logging.debug(
'Not found qemu-guest-agent in virtio-win or rhv-guest-tools-iso,'
' Try to prepare it manually. This is not a permanant step, once'
' the official build includes it, this step should be removed.'
)
os.makedirs(qemu_guest_agent_dir)
rpm_name = os.path.basename(qa_url)
download.get_file(
qa_url, os.path.join(qemu_guest_agent_dir, rpm_name))
if checkpoint == 'virtio_iso_blk':
if not os.path.exists(virtio_win_path):
test.fail('%s does not exist' % virtio_win_path)
# Find a free loop device
free_loop_dev = process.run("losetup --find",
shell=True).stdout_text.strip()
# Setup a loop device
cmd = 'losetup %s %s' % (free_loop_dev, virtio_win_path)
process.run(cmd, shell=True)
os.environ['VIRTIO_WIN'] = free_loop_dev
if checkpoint == 'invalid_pem':
# simply change the 2nd line to lowercase to get an invalid pem
with open(local_ca_file_path, 'r+') as fd:
for i in range(2):
pos = fd.tell()
res = fd.readline()
fd.seek(pos)
fd.write(res.lower())
fd.flush()
if checkpoint == 'without_default_net':
net_name = find_net('virbr0')
if net_name:
destroy_net(net_name)
if checkpoint == 'empty_cdrom':
virsh_dargs = {
'uri': remote_uri,
'remote_ip': remote_host,
'remote_user': '******',
'remote_pwd': vpx_passwd,
'auto_close': True,
'debug': True
}
remote_virsh = virsh.VirshPersistent(**virsh_dargs)
v2v_result = remote_virsh.dumpxml(vm_name)
remote_virsh.close_session()
else:
if checkpoint == 'exist_uuid':
auto_clean = False
if checkpoint in [
'mismatched_uuid', 'no_uuid', 'system_rhv_pem_set',
'system_rhv_pem_unset'
]:
cmd_only = True
auto_clean = False
v2v_result = utils_v2v.v2v_cmd(v2v_params, auto_clean, cmd_only)
if 'new_name' in v2v_params:
vm_name = params['main_vm'] = v2v_params['new_name']
if checkpoint.startswith('system_rhv_pem'):
if checkpoint == 'system_rhv_pem_set':
global_pem_setup(local_ca_file_path)
rhv_cafile = r'-oo rhv-cafile=\S+\s*'
new_cmd = cmd_remove_option(v2v_result, rhv_cafile)
logging.debug('New v2v command:\n%s', new_cmd)
if checkpoint == 'mismatched_uuid':
# append more uuid
new_cmd = v2v_result + ' -oo rhv-disk-uuid=%s' % str(uuid.uuid4())
if checkpoint == 'no_uuid':
rhv_disk_uuid = r'-oo rhv-disk-uuid=\S+\s*'
new_cmd = cmd_remove_option(v2v_result, rhv_disk_uuid)
logging.debug('New v2v command:\n%s', new_cmd)
if checkpoint == 'exist_uuid':
new_vm_name = v2v_params['new_name'] + '_exist_uuid'
new_cmd = v2v_result.command.replace('-on %s' % vm_name,
'-on %s' % new_vm_name)
logging.debug('re-run v2v command:\n%s', new_cmd)
if checkpoint in [
'mismatched_uuid', 'no_uuid', 'exist_uuid',
'system_rhv_pem_set', 'system_rhv_pem_unset'
]:
v2v_result = utils_v2v.cmd_run(new_cmd,
params.get('v2v_dirty_resources'))
check_result(v2v_result, status_error)
finally:
if checkpoint == 'ogac' and params.get('tmp_mount_point'):
if os.path.exists(params.get('tmp_mount_point')):
utils_misc.umount(os.getenv('VIRTIO_WIN'),
params['tmp_mount_point'], 'iso9660')
os.environ.pop('VIRTIO_WIN')
if checkpoint == 'virtio_iso_blk':
process.run('losetup -d %s' % free_loop_dev, shell=True)
os.environ.pop('VIRTIO_WIN')
if checkpoint == 'system_rhv_pem_set':
global_pem_cleanup()
if checkpoint == 'without_default_net':
if net_name:
start_net(net_name)
if params.get('vmchecker'):
params['vmchecker'].cleanup()
if output_mode == 'rhev' and v2v_sasl:
v2v_sasl.cleanup()
logging.debug('SASL session %s is closing', v2v_sasl)
v2v_sasl.close_session()
if output_mode == 'libvirt':
pvt.cleanup_pool(pool_name, pool_type, pool_target, '')
if checkpoint == 'with_proxy':
logging.info('Unset http_proxy&https_proxy')
os.environ.pop('http_proxy')
os.environ.pop('https_proxy')
# Cleanup constant files
utils_v2v.cleanup_constant_files(params)
def test_formatted_part(vm, params):
"""
1) Do some necessary check
2) Format additional disk with specific filesystem
3) Try to write a file to mounted device and get md5
4) Login to check writed file and its md5
"""
add_device = params.get("gf_additional_device", "/dev/vdb")
device_in_lgf = utils.run("echo %s | sed -e 's/vd/sd/g'" % add_device,
ignore_status=True).stdout.strip()
if utils_test.libguestfs.primary_disk_virtio(vm):
device_in_vm = add_device
else:
device_in_vm = "/dev/vda"
device_part = "%s1" % device_in_lgf
device_part_in_vm = "%s1" % device_in_vm
# Mount specific partition
params['special_mountpoints'] = [device_part]
vt = utils_test.libguestfs.VirtTools(vm, params)
# Create a new vm with additional disk
vt.update_vm_disk()
# Default vm_ref is oldvm, so switch it.
vm_ref = vt.newvm.name
# Format disk
format_result = vt.format_disk(filesystem="ext3", partition="mbr")
if format_result.exit_status:
raise error.TestFail("Format added disk failed.")
logging.info("Format added disk successfully.")
# List filesystems detail
list_fs_detail = vt.get_filesystems_info(vm_ref)
if list_fs_detail.exit_status:
raise error.TestFail("List filesystems detail failed.")
logging.info("List filesystems detail successfully.")
content = "This is file for formatted part test."
path = params.get("temp_file", "formatted_part")
mountpoint = params.get("vt_mountpoint", "/mnt")
writes, writeo = vt.write_file_with_guestmount(mountpoint, path,
content, vm_ref,
cleanup=False)
if writes is False:
utils_misc.umount("", mountpoint, "")
raise error.TestFail("Write file to mounted filesystem failed.")
logging.info("Create %s successfully.", writeo)
# Compute new file's md5
if os.path.isfile(writeo):
md5s, md5o = commands.getstatusoutput("md5sum %s" % writeo)
utils_misc.umount("", mountpoint, "")
if md5s:
raise error.TestFail("Compute %s's md5 failed." % writeo)
md5 = md5o.split()[0].strip()
logging.debug("%s's md5 in newvm is:%s", writeo, md5)
else:
utils_misc.umount("", mountpoint, "")
raise error.TestFail("Can not find %s." % writeo)
attached_vm = vt.newvm
try:
attached_vm.start()
session = attached_vm.wait_for_login()
except Exception, detail:
attached_vm.destroy()
raise error.TestFail(str(detail))