def verify_data_files(self):
non_existed_files = {}
disks_info = {}
# The last file should not exist on back2
for i, data_img in enumerate(self.src_img_tags):
non_existed_files[data_img] = self.files_info[data_img].pop()
disks_info[data_img] = self.disks_info[data_img]
# Check md5sum for the first two files
super(BlockdevIncBackupPullModeTest, self).verify_data_files()
# Check the files should not exist on back2
session = self.clone_vm.wait_for_login()
try:
for tag, info in six.iteritems(disks_info):
utils_disk.mount(info[0], info[1], session=session)
file_path = "%s/%s" % (info[1], non_existed_files[tag])
cat_cmd = "cat %s" % file_path
logging.info('Check %s should not exist' % file_path)
s, o = session.cmd_status_output(cat_cmd)
if s == 0:
self.test.fail('File (%s) exists' % non_existed_files[tag])
elif 'No such file' not in o.strip():
self.test.fail('Unknown error: %s' % o)
finally:
if session:
session.close()
def verify_data_files(self):
session = self.clone_vm.wait_for_login()
try:
for tag, info in self.disks_info.items():
logging.debug("mount target disk in VM!")
utils_disk.mount(info[0], info[1], session=session)
backup_utils.verify_file_md5(self.clone_vm, info[1], "data")
finally:
session.close()
def mount_data_disks(self):
if self.params["os_type"] == "windows":
return
session = self.clone_vm.wait_for_login()
try:
for info in self.disks_info:
disk_path = info[0]
mount_point = info[1]
utils_disk.mount(disk_path, mount_point, session=session)
finally:
session.close()
def glusterfs_mount(g_uri, mount_point, session=None):
"""
Mount gluster volume to mountpoint.
:param g_uri: stripped gluster uri from create_gluster_uri(.., True)
:type g_uri: str
:param mount_point: mount point, str
:session: mount within the session if given
"""
glusterfs_umount(g_uri, mount_point, session)
utils_disk.mount(g_uri, mount_point, "glusterfs", None, False, session)
def verify_data_files(self):
"""
Verify temp file's md5sum in all data disks
"""
session = self.clone_vm.wait_for_login()
try:
for tag, info in self.disks_info.items():
if tag != 'image1':
logging.debug("mount target disk in VM!")
utils_disk.mount(info[0], info[1], session=session)
for data_file in self.files_info[tag]:
backup_utils.verify_file_md5(
self.clone_vm, info[1], data_file)
finally:
session.close()
def mount_data_disks(self):
session = self.clone_vm.wait_for_login()
backup_utils.refresh_mounts(self.disks_info, self.params, session)
for info in self.disks_info.values():
disk_path = info[0]
mount_point = info[1]
if utils_disk.mount(disk_path, mount_point, session=session):
self.test.fail("Read-only disk is mounted with rw")
session.close()
def run(test, params, env):
"""
Test virtio-fs with mounting by read-only options.
Steps:
1. Create a shared directory for testing on the host.
2. Run the virtiofsd daemon on the host.
3. Boot a guest on the host.
4. Log into guest then mount the virtiofs with option "-o ro".
5. Generate a file on the mount point in guest.
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
fs_target = params.get('fs_target')
fs_dest = params.get('fs_dest')
vm = env.get_vm(params.get("main_vm"))
vm.verify_alive()
session = vm.wait_for_login()
error_context.context("Create a destination directory inside guest.",
logging.info)
utils_misc.make_dirs(fs_dest, session)
error_context.context(
"Mount the virtiofs target with read-only to "
"the destination directory inside guest.", logging.info)
utils_disk.mount(fs_target, fs_dest, 'virtiofs', 'ro', session=session)
try:
error_context.context(
"Create file under the destination "
"directory inside guest.", logging.info)
output = session.cmd_output(params.get('cmd_create_file'))
logging.info(output)
if params.get('check_str') not in output:
test.fail('Failed to mount the virtiofs target with read-only.')
finally:
utils_disk.umount(fs_target, fs_dest, 'virtiofs', session=session)
utils_misc.safe_rmdir(fs_dest, session=session)
def run(test, params, env):
"""
Test virtio-fs by sharing the data between host and guest.
Steps:
1. Create shared directories on the host.
2. Set capability on the host.
3. Run virtiofsd daemons on capability shell env.
4. Boot a guest on the host with virtiofs options.
5. Log into guest then mount the virtiofs targets.
6. Generate files or run stress on the mount points inside guest.
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def get_viofs_exe(session):
"""
Get viofs.exe from virtio win iso,such as E:\viofs\2k19\amd64
"""
media_type = params["virtio_win_media_type"]
try:
get_drive_letter = getattr(virtio_win,
"drive_letter_%s" % media_type)
get_product_dirname = getattr(virtio_win,
"product_dirname_%s" % media_type)
get_arch_dirname = getattr(virtio_win,
"arch_dirname_%s" % media_type)
except AttributeError:
test.error("Not supported virtio win media type '%s'", media_type)
viowin_ltr = get_drive_letter(session)
if not viowin_ltr:
test.error("Could not find virtio-win drive in guest")
guest_name = get_product_dirname(session)
if not guest_name:
test.error("Could not get product dirname of the vm")
guest_arch = get_arch_dirname(session)
if not guest_arch:
test.error("Could not get architecture dirname of the vm")
exe_middle_path = ("{name}\\{arch}" if media_type == "iso" else
"{arch}\\{name}").format(name=guest_name,
arch=guest_arch)
exe_file_name = "virtiofs.exe"
exe_find_cmd = 'dir /b /s %s\\%s | findstr "\\%s\\\\"'
exe_find_cmd %= (viowin_ltr, exe_file_name, exe_middle_path)
exe_path = session.cmd(exe_find_cmd).strip()
logging.info("Found exe file '%s'", exe_path)
return exe_path
# data io config
cmd_dd = params.get('cmd_dd')
cmd_md5 = params.get('cmd_md5')
io_timeout = params.get_numeric('io_timeout')
# remove capability config
cmd_create_fs_source = params.get('cmd_create_fs_source')
cmd_run_virtiofsd = params.get('cmd_run_virtiofsd')
capability = params.get('capability')
cmd_capsh_print = params.get('cmd_capsh_print')
cmd_capsh_drop = params.get('cmd_capsh_drop')
# set trusted config
cmd_yum_attr = params.get('cmd_yum_attr')
cmd_set_trusted = params.get('cmd_set_trusted')
cmd_get_trusted = params.get('cmd_get_trusted')
cmd_create_file = params.get('cmd_create_file')
cmd_set_capability = params.get('cmd_set_capability')
cmd_get_capability = params.get('cmd_get_capability')
cmd_echo_file = params.get('cmd_echo_file')
# set fs daemon path
target = params.get('fs_target')
fs_source = params.get('fs_source_dir')
base_dir = params.get('fs_source_base_dir', data_dir.get_data_dir())
if not os.path.isabs(fs_source):
fs_source = os.path.join(base_dir, fs_source)
if os.path.exists(fs_source):
shutil.rmtree(fs_source, ignore_errors=True)
logging.info("Create filesystem source %s.", fs_source)
os.makedirs(fs_source)
sock_path = os.path.join(
data_dir.get_tmp_dir(), '-'.join(
('avocado-vt-vm1', 'viofs', 'virtiofsd.sock')))
# set capability
cmd_capsh_drop = (cmd_capsh_drop % capability)
error_context.context("Remove capability on host.", logging.info)
session = aexpect.ShellSession(
cmd_capsh_drop,
auto_close=False,
output_func=utils_misc.log_line,
output_params=('virtiofs_fs-virtiofs.log', ),
prompt=r"^\[.*\][\#\$]\s*$")
output = session.cmd_output(cmd_capsh_print)
logging.info("Check current capability is %s.", output)
if capability in output:
test.error("It's failed to check the trusted info from the host.")
# run daemon
session.sendline(cmd_create_fs_source)
cmd_run_virtiofsd = cmd_run_virtiofsd % sock_path
cmd_run_virtiofsd += ' -o source=%s' % fs_source
cmd_run_virtiofsd += params.get('fs_binary_extra_options')
logging.info('Running daemon command %s.', cmd_run_virtiofsd)
session.sendline(cmd_run_virtiofsd)
# insert devices
vm = env.get_vm(params.get("main_vm"))
vm.devices, _ = vm.make_create_command()
machine_type = params.get("machine_type", "")
qbus_type = "PCI"
if machine_type.startswith("q35") or machine_type.startswith("arm64"):
qbus_type = "PCIE"
devices = []
vfsd = qdevices.QCustomDevice('chardev null,id=serial_vfsd',
aobject='fs',
child_bus=qdevices.QUnixSocketBus(
sock_path, 'fs'))
devices.append(vfsd)
char_params = Params()
char_params["backend"] = "socket"
char_params["id"] = 'virtiofs_fs'
sock_bus = {'busid': sock_path}
char = qdevices.CharDevice(char_params, parent_bus=sock_bus)
char.set_aid('virtiofs_fs')
devices.append(char)
qdriver = "vhost-user-fs"
if "-mmio:" in machine_type:
qdriver += "-device"
qbus_type = "virtio-bus"
elif machine_type.startswith("s390"):
qdriver += "-ccw"
qbus_type = "virtio-bus"
else:
qdriver += "-pci"
bus = {"type": qbus_type}
dev_params = {
"id": "vufs_virtiofs_fs",
"chardev": char.get_qid(),
"tag": target
}
fs_driver_props = json.loads(params.get("fs_driver_props", "{}"))
dev_params.update(fs_driver_props)
vufs = qdevices.QDevice(qdriver, params=dev_params, parent_bus=bus)
vufs.set_aid('virtiofs_fs')
devices.append(vufs)
vm.devices.insert(devices)
# Since if 'redirs' has a value, the vm.create() method will reset the devices.
# So set 'redirs' to empty for a workaround.
vm.params['redirs'] = ''
vm.create()
vm.verify_alive()
is_windows = params.get("os_type") == "windows"
session = vm.wait_for_login()
if is_windows:
cmd_timeout = params.get_numeric("cmd_timeout", 120)
driver_name = params["driver_name"]
install_path = params["install_path"]
check_installed_cmd = params["check_installed_cmd"] % install_path
# Check whether windows driver is running,and enable driver verifier
session = utils_test.qemu.windrv_check_running_verifier(
session, vm, test, driver_name)
# Install winfsp tool
error_context.context("Install winfsp for windows guest.",
logging.info)
is_installed = session.cmd_status(check_installed_cmd) == 0
if is_installed:
logging.info("Winfsp tool is already installed.")
else:
install_cmd = utils_misc.set_winutils_letter(
session, params["install_cmd"])
session.cmd(install_cmd, cmd_timeout)
if not utils_misc.wait_for(
lambda: not session.cmd_status(check_installed_cmd), 60):
test.error("Winfsp tool is not installed.")
fs_params = params.object_params('fs')
fs_target = fs_params.get("fs_target")
fs_dest = fs_params.get("fs_dest")
host_data = os.path.join(fs_source, 'fs_test')
if not is_windows:
error_context.context(
"Create a destination directory %s "
"inside guest." % fs_dest, logging.info)
utils_misc.make_dirs(fs_dest, session)
error_context.context(
"Mount virtiofs target %s to %s inside"
" guest." % (fs_target, fs_dest), logging.info)
if not utils_disk.mount(
fs_target, fs_dest, 'virtiofs', session=session):
test.fail('Mount virtiofs target failed.')
else:
error_context.context("Start virtiofs service in guest.", logging.info)
viofs_sc_create_cmd = params["viofs_sc_create_cmd"]
viofs_sc_start_cmd = params["viofs_sc_start_cmd"]
viofs_sc_query_cmd = params["viofs_sc_query_cmd"]
logging.info("Check if virtiofs service is registered.")
status, output = session.cmd_status_output(viofs_sc_query_cmd)
if "not exist as an installed service" in output:
logging.info("Register virtiofs service in windows guest.")
exe_path = get_viofs_exe(session)
viofs_sc_create_cmd = viofs_sc_create_cmd % exe_path
sc_create_s, sc_create_o = session.cmd_status_output(
viofs_sc_create_cmd)
if sc_create_s != 0:
test.fail("Failed to register virtiofs service, output is %s" %
sc_create_o)
logging.info("Check if virtiofs service is started.")
status, output = session.cmd_status_output(viofs_sc_query_cmd)
if "RUNNING" not in output:
logging.info("Start virtiofs service.")
sc_start_s, sc_start_o = session.cmd_status_output(
viofs_sc_start_cmd)
if sc_start_s != 0:
test.fail("Failed to start virtiofs service, output is %s" %
sc_start_o)
else:
logging.info("Virtiofs service is running.")
# get fs dest for vm
virtio_fs_disk_label = fs_target
error_context.context(
"Get Volume letter of virtio fs target, the disk"
"lable is %s." % virtio_fs_disk_label, logging.info)
vol_con = "VolumeName='%s'" % virtio_fs_disk_label
vol_func = utils_misc.get_win_disk_vol(session, condition=vol_con)
volume_letter = utils_misc.wait_for(lambda: vol_func, 120)
if volume_letter is None:
test.fail("Could not get virtio-fs mounted volume letter.")
fs_dest = "%s:" % volume_letter
guest_file = os.path.join(fs_dest, 'fs_test')
logging.info("The guest file in shared dir is %s.", guest_file)
try:
# No extended attributes (file steams) in virtio-fs for windows
if not is_windows:
if cmd_set_trusted:
error_context.context(
"Trusted attribute test without "
"%s for linux guest" % capability, logging.info)
host_attributes = params["host_attributes"]
guest_trusted = params["guest_trusted"]
file_capability = params["file_capability"]
logging.info("Set a trusted on guest.")
session.cmd(cmd_yum_attr)
session.cmd(cmd_set_trusted)
output = session.cmd_output(cmd_get_trusted)
logging.info(
"Failed to check the trusted attribute from "
"guest, the output is %s.", output)
if guest_trusted not in output:
test.fail(
"It's failed to check the trusted info from the guest."
)
process.run(cmd_yum_attr)
output = str(
process.run('getfattr %s' % fs_source).stdout.strip())
logging.info("The host file trusted is %s.", output)
if host_attributes not in output:
test.fail("Failed to check the trusted attribute from "
"host, the output is %s." % output)
session.cmd(cmd_create_file)
error_context.context(
"Privileged capabilities test without "
"%s for linux guest" % capability, logging.info)
session.cmd(cmd_set_capability)
output = session.cmd_output(cmd_get_capability)
logging.info("The guest file capability is %s.", output)
if file_capability not in output:
test.fail("Failed to check the trusted attribute from "
"guest, the output is %s." % output)
logging.info(
"Modify file content and check the file capability.")
session.cmd(cmd_echo_file)
output = session.cmd_output(cmd_get_capability)
logging.info("The guest change file capability is %s.", output)
if file_capability in output:
test.fail(
"Still can get capability after file content is changed."
)
if cmd_dd:
error_context.context(
"Creating file under %s inside guest." % fs_dest, logging.info)
session.cmd(cmd_dd % guest_file, io_timeout)
if not is_windows:
cmd_md5_vm = cmd_md5 % guest_file
else:
guest_file_win = guest_file.replace("/", "\\")
cmd_md5_vm = cmd_md5 % (volume_letter, guest_file_win)
md5_guest = session.cmd_output(cmd_md5_vm,
io_timeout).strip().split()[0]
logging.info(md5_guest)
md5_host = process.run("md5sum %s" % host_data,
io_timeout).stdout_text.strip().split()[0]
if md5_guest != md5_host:
test.fail('The md5 value of host is not same to guest.')
finally:
if not is_windows:
utils_disk.umount(fs_target, fs_dest, 'virtiofs', session=session)
utils_misc.safe_rmdir(fs_dest, session=session)
session.close()
vm.destroy()
utils_misc.safe_rmdir(fs_source)
def run(test, params, env):
"""
Test virtio-fs by sharing the data between host and guest.
Steps:
1. Create shared directories on the host.
2. Run virtiofsd daemons on the host.
3. Boot a guest on the host with virtiofs options.
4. Log into guest then mount the virtiofs targets.
5. Generate files or run stress on the mount points inside guest.
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def get_viofs_exe(session):
"""
Get viofs.exe from virtio win iso,such as E:\viofs\2k19\amd64
"""
logging.info("Get virtiofs exe full path.")
media_type = params["virtio_win_media_type"]
try:
get_drive_letter = getattr(virtio_win, "drive_letter_%s" % media_type)
get_product_dirname = getattr(virtio_win,
"product_dirname_%s" % media_type)
get_arch_dirname = getattr(virtio_win, "arch_dirname_%s" % media_type)
except AttributeError:
test.error("Not supported virtio win media type '%s'", media_type)
viowin_ltr = get_drive_letter(session)
if not viowin_ltr:
test.error("Could not find virtio-win drive in guest")
guest_name = get_product_dirname(session)
if not guest_name:
test.error("Could not get product dirname of the vm")
guest_arch = get_arch_dirname(session)
if not guest_arch:
test.error("Could not get architecture dirname of the vm")
exe_middle_path = ("{name}\\{arch}" if media_type == "iso"
else "{arch}\\{name}").format(name=guest_name,
arch=guest_arch)
exe_file_name = "virtiofs.exe"
exe_find_cmd = 'dir /b /s %s\\%s | findstr "\\%s\\\\"'
exe_find_cmd %= (viowin_ltr, exe_file_name, exe_middle_path)
exe_path = session.cmd(exe_find_cmd).strip()
logging.info("Found exe file '%s'", exe_path)
return exe_path
# data io config
cmd_dd = params.get('cmd_dd')
cmd_md5 = params.get('cmd_md5')
cmd_new_folder = params.get('cmd_new_folder')
cmd_copy_file = params.get('cmd_copy_file')
cmd_del_folder = params.get('cmd_del_folder')
# pjdfs test config
cmd_pjdfstest = params.get('cmd_pjdfstest')
cmd_unpack = params.get('cmd_unpack')
cmd_yum_deps = params.get('cmd_yum_deps')
cmd_autoreconf = params.get('cmd_autoreconf')
cmd_configure = params.get('cmd_configure')
cmd_make = params.get('cmd_make')
pjdfstest_pkg = params.get('pjdfstest_pkg')
username = params.get('username')
password = params.get('password')
port = params.get('file_transfer_port')
# fio config
fio_options = params.get('fio_options')
io_timeout = params.get_numeric('io_timeout')
# xfstest config
cmd_xfstest = params.get('cmd_xfstest')
fs_dest_fs2 = params.get('fs_dest_fs2')
cmd_download_xfstest = params.get('cmd_download_xfstest')
cmd_yum_install = params.get('cmd_yum_install')
cmd_make_xfs = params.get('cmd_make_xfs')
cmd_setenv = params.get('cmd_setenv')
cmd_setenv_nfs = params.get('cmd_setenv_nfs', '')
cmd_useradd = params.get('cmd_useradd')
fs_dest_fs1 = params.get('fs_dest_fs1')
cmd_get_tmpfs = params.get('cmd_get_tmpfs')
cmd_set_tmpfs = params.get('cmd_set_tmpfs')
size_mem1 = params.get('size_mem1')
# xfstest-nfs config
setup_local_nfs = params.get('setup_local_nfs')
if cmd_xfstest:
# /dev/shm is the default memory-backend-file, the default value is the
# half of the host memory. Increase it to guest memory size to avoid crash
ori_tmpfs_size = process.run(cmd_get_tmpfs, shell=True).stdout_text.replace("\n", "")
logging.debug("original tmpfs size is %s", ori_tmpfs_size)
params["post_command"] = cmd_set_tmpfs % ori_tmpfs_size
params["pre_command"] = cmd_set_tmpfs % size_mem1
if setup_local_nfs:
for fs in params.objects("filesystems"):
nfs_params = params.object_params(fs)
params["export_dir"] = nfs_params.get("export_dir")
params["nfs_mount_src"] = nfs_params.get("nfs_mount_src")
params["nfs_mount_dir"] = nfs_params.get("fs_source_dir")
nfs_local = nfs.Nfs(params)
nfs_local.setup()
params["start_vm"] = "yes"
env_process.preprocess(test, params, env)
os_type = params.get("os_type")
vm = env.get_vm(params.get("main_vm"))
vm.verify_alive()
session = vm.wait_for_login()
host_addr = vm.get_address()
if os_type == "windows":
cmd_timeout = params.get_numeric("cmd_timeout", 120)
driver_name = params["driver_name"]
install_path = params["install_path"]
check_installed_cmd = params["check_installed_cmd"] % install_path
# Check whether windows driver is running,and enable driver verifier
session = utils_test.qemu.windrv_check_running_verifier(session,
vm, test,
driver_name)
# install winfsp tool
error_context.context("Install winfsp for windows guest.",
logging.info)
installed = session.cmd_status(check_installed_cmd) == 0
if installed:
logging.info("Winfsp tool is already installed.")
else:
install_cmd = utils_misc.set_winutils_letter(session,
params["install_cmd"])
session.cmd(install_cmd, cmd_timeout)
if not utils_misc.wait_for(lambda: not session.cmd_status(
check_installed_cmd), 60):
test.error("Winfsp tool is not installed.")
for fs in params.objects("filesystems"):
fs_params = params.object_params(fs)
fs_target = fs_params.get("fs_target")
fs_dest = fs_params.get("fs_dest")
fs_source = fs_params.get("fs_source_dir")
base_dir = fs_params.get("fs_source_base_dir",
data_dir.get_data_dir())
if not os.path.isabs(fs_source):
fs_source = os.path.join(base_dir, fs_source)
host_data = os.path.join(fs_source, 'fs_test')
if os_type == "linux":
error_context.context("Create a destination directory %s "
"inside guest." % fs_dest, logging.info)
utils_misc.make_dirs(fs_dest, session)
if not cmd_xfstest:
error_context.context("Mount virtiofs target %s to %s inside"
" guest." % (fs_target, fs_dest),
logging.info)
if not utils_disk.mount(fs_target, fs_dest, 'virtiofs', session=session):
test.fail('Mount virtiofs target failed.')
else:
error_context.context("Start virtiofs service in guest.", logging.info)
viofs_sc_create_cmd = params["viofs_sc_create_cmd"]
viofs_sc_start_cmd = params["viofs_sc_start_cmd"]
viofs_sc_query_cmd = params["viofs_sc_query_cmd"]
logging.info("Check if virtiofs service is registered.")
status, output = session.cmd_status_output(viofs_sc_query_cmd)
if "not exist as an installed service" in output:
logging.info("Register virtiofs service in windows guest.")
exe_path = get_viofs_exe(session)
viofs_sc_create_cmd = viofs_sc_create_cmd % exe_path
sc_create_s, sc_create_o = session.cmd_status_output(viofs_sc_create_cmd)
if sc_create_s != 0:
test.fail("Failed to register virtiofs service, output is %s" % sc_create_o)
logging.info("Check if virtiofs service is started.")
status, output = session.cmd_status_output(viofs_sc_query_cmd)
if "RUNNING" not in output:
logging.info("Start virtiofs service.")
sc_start_s, sc_start_o = session.cmd_status_output(viofs_sc_start_cmd)
if sc_start_s != 0:
test.fail("Failed to start virtiofs service, output is %s" % sc_start_o)
else:
logging.info("Virtiofs service is running.")
viofs_log_file_cmd = params.get("viofs_log_file_cmd")
if viofs_log_file_cmd:
error_context.context("Check if LOG file is created.", logging.info)
log_dir_s = session.cmd_status(viofs_log_file_cmd)
if log_dir_s != 0:
test.fail("Virtiofs log is not created.")
# get fs dest for vm
virtio_fs_disk_label = fs_target
error_context.context("Get Volume letter of virtio fs target, the disk"
"lable is %s." % virtio_fs_disk_label,
logging.info)
vol_con = "VolumeName='%s'" % virtio_fs_disk_label
vol_func = utils_misc.get_win_disk_vol(session, condition=vol_con)
volume_letter = utils_misc.wait_for(lambda: vol_func, cmd_timeout)
if volume_letter is None:
test.fail("Could not get virtio-fs mounted volume letter.")
fs_dest = "%s:" % volume_letter
guest_file = os.path.join(fs_dest, 'fs_test')
logging.info("The guest file in shared dir is %s", guest_file)
try:
if cmd_dd:
error_context.context("Creating file under %s inside "
"guest." % fs_dest, logging.info)
session.cmd(cmd_dd % guest_file, io_timeout)
if os_type == "linux":
cmd_md5_vm = cmd_md5 % guest_file
else:
guest_file_win = guest_file.replace("/", "\\")
cmd_md5_vm = cmd_md5 % (volume_letter, guest_file_win)
md5_guest = session.cmd_output(cmd_md5_vm, io_timeout).strip().split()[0]
logging.info(md5_guest)
md5_host = process.run("md5sum %s" % host_data,
io_timeout).stdout_text.strip().split()[0]
if md5_guest != md5_host:
test.fail('The md5 value of host is not same to guest.')
if cmd_new_folder and cmd_copy_file and cmd_del_folder:
error_context.context("Folder test under %s inside "
"guest." % fs_dest, logging.info)
session.cmd(cmd_new_folder % fs_dest)
test_file = guest_file if os_type == "linux" \
else "%s:\\%s" % (volume_letter, 'fs_test')
session.cmd(cmd_copy_file % (test_file, fs_dest))
session.cmd(cmd_del_folder % fs_dest)
if fio_options:
error_context.context("Run fio on %s." % fs_dest, logging.info)
fio = generate_instance(params, vm, 'fio')
try:
fio.run(fio_options % guest_file, io_timeout)
finally:
fio.clean()
vm.verify_dmesg()
if cmd_pjdfstest:
error_context.context("Run pjdfstest on %s." % fs_dest, logging.info)
host_path = os.path.join(data_dir.get_deps_dir('pjdfstest'), pjdfstest_pkg)
scp_to_remote(host_addr, port, username, password, host_path, fs_dest)
session.cmd(cmd_unpack.format(fs_dest), 180)
session.cmd(cmd_yum_deps, 180)
session.cmd(cmd_autoreconf % fs_dest, 180)
session.cmd(cmd_configure.format(fs_dest), 180)
session.cmd(cmd_make % fs_dest, io_timeout)
session.cmd(cmd_pjdfstest % fs_dest, io_timeout)
if cmd_xfstest:
error_context.context("Run xfstest on guest.", logging.info)
utils_misc.make_dirs(fs_dest_fs2, session)
if session.cmd_status(cmd_download_xfstest, 360):
test.error("Failed to download xfstests-dev")
session.cmd(cmd_yum_install, 180)
session.cmd(cmd_make_xfs, 360)
session.cmd(cmd_setenv, 180)
session.cmd(cmd_setenv_nfs, 180)
session.cmd(cmd_useradd, 180)
try:
output = session.cmd_output(cmd_xfstest, io_timeout)
logging.info("%s", output)
if 'Failed' in output:
test.fail('The xfstest failed.')
else:
break
except (aexpect.ShellStatusError, aexpect.ShellTimeoutError):
test.fail('The xfstest failed.')
finally:
if os_type == "linux":
utils_disk.umount(fs_target, fs_dest, 'virtiofs', session=session)
utils_misc.safe_rmdir(fs_dest, session=session)
if setup_local_nfs:
session.close()
vm.destroy()
for fs in params.objects("filesystems"):
nfs_params = params.object_params(fs)
params["export_dir"] = nfs_params.get("export_dir")
params["nfs_mount_dir"] = nfs_params.get("fs_source_dir")
params["rm_export_dir"] = nfs_params.get("export_dir")
params["rm_mount_dir"] = nfs_params.get("fs_source_dir")
nfs_local = nfs.Nfs(params)
nfs_local.cleanup()
utils_misc.safe_rmdir(params["export_dir"])
# during all virtio fs is mounted, reboot vm
if params.get('reboot_guest', 'no') == 'yes':
def get_vfsd_num():
"""
Get virtiofsd daemon number during vm boot up.
:return: virtiofsd daemon count.
"""
cmd_ps_virtiofsd = params.get('cmd_ps_virtiofsd')
vfsd_num = 0
for device in vm.devices:
if isinstance(device, qdevices.QVirtioFSDev):
sock_path = device.get_param('sock_path')
cmd_ps_virtiofsd = cmd_ps_virtiofsd % sock_path
vfsd_ps = process.system_output(cmd_ps_virtiofsd, shell=True)
vfsd_num += len(vfsd_ps.strip().splitlines())
return vfsd_num
error_context.context("Check virtiofs daemon before reboot vm.",
logging.info)
vfsd_num_bf = get_vfsd_num()
error_context.context("Reboot guest and check virtiofs daemon.",
logging.info)
vm.reboot()
if not vm.is_alive():
test.fail("After rebooting vm quit unexpectedly.")
vfsd_num_af = get_vfsd_num()
if vfsd_num_bf != vfsd_num_af:
test.fail("Virtiofs daemon is different before and after reboot.\n"
"Before reboot: %s\n"
"After reboot: %s\n", (vfsd_num_bf, vfsd_num_af))
def run(test, params, env):
"""
KVM block resize test:
1) Start guest with data disk or system disk.
2) Do format disk in guest if needed.
3) Record md5 of test file on the data disk.
Enlarge the data disk image from qemu monitor.
4) Extend data disk partition/file-system in guest.
5) Verify the data disk size match expected size.
6) Reboot the guest.
7) Do iozone test, compare the md5 of test file.
8) Shrink data disk partition/file-system in guest.
9) Shrink data disk image from qemu monitor.
10) Verify the data disk size match expected size.
11) Reboot the guest.
12) Do iozone test, compare the md5 of test file.
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
def verify_disk_size(session, os_type, disk):
"""
Verify the current block size match with the expected size.
"""
global current_size
current_size = utils_disk.get_disk_size(session, os_type, disk)
accept_ratio = float(params.get("accept_ratio", 0))
if (current_size <= block_size
and current_size >= block_size * (1 - accept_ratio)):
logging.info(
"Block Resizing Finished !!! \n"
"Current size %s is same as the expected %s", current_size,
block_size)
return True
def create_md5_file(filename):
"""
Create the file to verify md5 value.
"""
logging.debug("create md5 file %s" % filename)
if os_type == 'windows':
vm.copy_files_to(params["tmp_md5_file"], filename)
else:
session.cmd(params["dd_cmd"] % filename)
def get_md5_of_file(filename):
"""
Get the md5 value of filename.
"""
ex_args = (mpoint, filename) if os_type == 'windows' else filename
return session.cmd(md5_cmd % ex_args).split()[0]
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
timeout = float(params.get("login_timeout", 240))
driver_name = params.get("driver_name")
os_type = params["os_type"]
fstype = params.get("fstype")
labeltype = params.get("labeltype", "msdos")
img_size = params.get("image_size_stg", "10G")
mpoint = params.get("disk_letter", "C")
disk = params.get("disk_index", 0)
md5_cmd = params.get("md5_cmd", "md5sum %s")
md5_file = params.get("md5_file", "md5.dat")
data_image = params.get("images").split()[-1]
data_image_params = params.object_params(data_image)
data_image_filename = storage.get_image_filename(data_image_params,
data_dir.get_data_dir())
data_image_dev = vm.get_block({'file': data_image_filename})
img = QemuImg(data_image_params, data_dir.get_data_dir(), data_image)
block_virtual_size = json.loads(img.info(force_share=True,
output="json"))["virtual-size"]
session = vm.wait_for_login(timeout=timeout)
if os_type == 'windows' and driver_name:
session = utils_test.qemu.windrv_check_running_verifier(
session, vm, test, driver_name, timeout)
if params.get("format_disk") == "yes":
if os_type == 'linux':
disk = sorted(utils_disk.get_linux_disks(session).keys())[0]
else:
disk = utils_disk.get_windows_disks_index(session, img_size)[0]
utils_disk.update_windows_disk_attributes(session, disk)
error_context.context("Formatting disk", logging.info)
mpoint = utils_disk.configure_empty_disk(session,
disk,
img_size,
os_type,
fstype=fstype,
labeltype=labeltype)[0]
partition = mpoint.replace('mnt', 'dev') if 'mnt' in mpoint else None
for ratio in params.objects("disk_change_ratio"):
block_size = int(int(block_virtual_size) * float(ratio))
# Record md5
if params.get('md5_test') == 'yes':
junction = ":\\" if os_type == 'windows' else "/"
md5_filename = mpoint + junction + md5_file
create_md5_file(md5_filename)
md5 = get_md5_of_file(md5_filename)
logging.debug("Got md5 %s ratio:%s on %s" % (md5, ratio, disk))
# We need shrink the disk in guest first, than in monitor
if float(ratio) < 1.0:
error_context.context(
"Shrink disk size to %s in guest" % block_size, logging.info)
if os_type == 'windows':
shr_size = utils_numeric.normalize_data_size(
str(
utils_disk.get_disk_size(session, os_type, disk) -
block_size), 'M').split(".")[0]
drive.shrink_volume(session, mpoint, shr_size)
else:
utils_disk.resize_filesystem_linux(session, partition,
str(block_size))
utils_disk.resize_partition_linux(session, partition,
str(block_size))
error_context.context("Change disk size to %s in monitor" % block_size,
logging.info)
if vm.check_capability(Flags.BLOCKDEV):
args = (None, block_size, data_image_dev)
else:
args = (data_image_dev, block_size)
vm.monitor.block_resize(*args)
if params.get("guest_prepare_cmd", ""):
session.cmd(params.get("guest_prepare_cmd"))
# Update GPT due to size changed
if os_type == "linux" and labeltype == "gpt":
session.cmd("sgdisk -e /dev/%s" % disk)
if params.get("need_reboot") == "yes":
session = vm.reboot(session=session)
if params.get("need_rescan") == "yes":
drive.rescan_disks(session)
# We need extend disk in monitor first than extend it in guest
if float(ratio) > 1.0:
error_context.context("Extend disk to %s in guest" % block_size,
logging.info)
if os_type == 'windows':
drive.extend_volume(session, mpoint)
else:
utils_disk.resize_partition_linux(session, partition,
str(block_size))
utils_disk.resize_filesystem_linux(session, partition,
utils_disk.SIZE_AVAILABLE)
global current_size
current_size = 0
if not wait.wait_for(lambda: verify_disk_size(session, os_type, disk),
20, 0, 1, "Block Resizing"):
test.fail("Block size get from guest is not same as expected.\n"
"Reported: %s\nExpect: %s\n" %
(current_size, block_size))
session = vm.reboot(session=session)
if os_type == 'linux':
if not utils_disk.is_mount(
partition, dst=mpoint, fstype=fstype, session=session):
utils_disk.mount(partition,
mpoint,
fstype=fstype,
session=session)
if params.get('iozone_test') == 'yes':
iozone_timeout = float(params.get("iozone_timeout", 1800))
iozone_cmd_options = params.get("iozone_option") % mpoint
io_test = generate_instance(params, vm, 'iozone')
try:
io_test.run(iozone_cmd_options, iozone_timeout)
finally:
io_test.clean()
# Verify md5
if params.get('md5_test') == 'yes':
new_md5 = get_md5_of_file(md5_filename)
test.assertTrue(new_md5 == md5, "Unmatched md5: %s" % new_md5)
session.close()
def run(test, params, env):
"""
Test to virtio-fs with the multiple VMs and virtiofs daemons.
Steps:
1. Create shared directories on the host.
2. Run virtiofs daemons on the host.
3. Boot guests on the host with virtiofs options.
4. Log into guest then mount the virtiofs targets.
5. Generate files on the mount points inside guests.
6. Compare the md5 among guests if multiple virtiofs
daemons share the source.
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def get_viofs_exe(session):
"""
Get viofs.exe from virtio win iso,such as E:\viofs\2k19\amd64
"""
media_type = params["virtio_win_media_type"]
try:
get_drive_letter = getattr(virtio_win,
"drive_letter_%s" % media_type)
get_product_dirname = getattr(virtio_win,
"product_dirname_%s" % media_type)
get_arch_dirname = getattr(virtio_win,
"arch_dirname_%s" % media_type)
except AttributeError:
test.error("Not supported virtio win media type '%s'", media_type)
viowin_ltr = get_drive_letter(session)
if not viowin_ltr:
test.error("Could not find virtio-win drive in guest")
guest_name = get_product_dirname(session)
if not guest_name:
test.error("Could not get product dirname of the vm")
guest_arch = get_arch_dirname(session)
if not guest_arch:
test.error("Could not get architecture dirname of the vm")
exe_middle_path = ("{name}\\{arch}" if media_type == "iso" else
"{arch}\\{name}").format(name=guest_name,
arch=guest_arch)
exe_file_name = "virtiofs.exe"
exe_find_cmd = 'dir /b /s %s\\%s | findstr "\\%s\\\\"'
exe_find_cmd %= (viowin_ltr, exe_file_name, exe_middle_path)
exe_path = session.cmd(exe_find_cmd).strip()
logging.info("Found exe file '%s'", exe_path)
return exe_path
cmd_dd = params.get('cmd_dd')
cmd_md5 = params.get('cmd_md5')
io_timeout = params.get_numeric('io_timeout')
shared_fs_source_dir = params.get('shared_fs_source_dir')
os_type = params.get('os_type')
# cfg for windows vm
cmd_timeout = params.get_numeric("cmd_timeout", 120)
driver_name = params.get("driver_name")
wfsp_install_cmd = params.get("wfsp_install_cmd")
check_installed_cmd = params.get("check_installed_cmd")
sessions = []
vms = env.get_all_vms()
for vm in vms:
vm.verify_alive()
sessions.append(vm.wait_for_login())
mapping = {}
for vm, session in zip(params.objects('vms'), sessions):
vm_params = params.object_params(vm)
mapping[vm] = {'session': session, 'filesystems': []}
# check driver verifier in windows vm
# install winfsp tool and start virtiofs exe in windows vm
if os_type == "windows":
# Check whether windows driver is running,and enable driver verifier
session = utils_test.qemu.windrv_check_running_verifier(
session, vm, test, driver_name)
error_context.context("%s: Install winfsp for windows guest." % vm,
logging.info)
installed = session.cmd_status(check_installed_cmd) == 0
if installed:
logging.info("%s: Winfsp tool is already installed.", vm)
else:
install_cmd = utils_misc.set_winutils_letter(
session, wfsp_install_cmd)
session.cmd(install_cmd, cmd_timeout)
if not utils_misc.wait_for(
lambda: not session.cmd_status(check_installed_cmd),
60):
test.error("%s: Winfsp tool is not installed." % vm)
error_context.context("%s: Start virtiofs service in guest." % vm,
logging.info)
exe_path = get_viofs_exe(session)
start_vfs_cmd = params["start_vfs_cmd"] % exe_path
session.sendline(start_vfs_cmd)
error_context.context(
"%s: Check if virtiofs service is started." % vm, logging.info)
check_virtiofs_cmd = params["check_virtiofs_cmd"]
if not utils_misc.wait_for(
lambda: re.search("virtiofs",
session.cmd_output(check_virtiofs_cmd),
re.IGNORECASE), 30):
test.fail("%s: Virtiofs service is failed to start." % vm)
# get fs dest for vm
for fs in vm_params.objects('filesystems'):
fs_params = vm_params.object_params(fs)
fs_target = fs_params.get("fs_target")
fs_dest = fs_params.get("fs_dest")
if os_type == "linux":
error_context.context(
"%s: Create a destination directory %s inside guest." %
(vm, fs_dest), logging.info)
utils_misc.make_dirs(fs_dest, session)
error_context.context(
"%s: Mount the virtiofs target %s to %s inside guest." %
(vm, fs_target, fs_dest), logging.info)
utils_disk.mount(fs_target,
fs_dest,
'virtiofs',
session=session)
else:
virtio_fs_disk_label = fs_target
error_context.context(
"%s: Get Volume letter of virtio fs"
" target, the disk lable is %s." %
(vm, virtio_fs_disk_label), logging.info)
vol_con = "VolumeName='%s'" % virtio_fs_disk_label
vol_func = utils_misc.get_win_disk_vol(session,
condition=vol_con)
volume_letter = utils_misc.wait_for(lambda: vol_func,
cmd_timeout)
fs_dest = "%s:" % volume_letter
guest_file = os.path.join(fs_dest, 'fs_test')
logging.info("%s: The guest file in shared dir is %s", vm,
guest_file)
mapping[vm]['filesystems'].append({
'fs_target': fs_target,
'fs_dest': fs_dest,
'guest_file': guest_file
})
if cmd_dd:
logging.info("%s: Creating file under %s inside guest.", vm,
fs_dest)
session.cmd(cmd_dd % guest_file, io_timeout)
if shared_fs_source_dir:
continue
if os_type == "linux":
error_context.context(
"%s: Umount the viriofs target %s." % (vm, fs_target),
logging.info)
utils_disk.umount(fs_target,
fs_dest,
'virtiofs',
session=session)
if shared_fs_source_dir:
error_context.context("Compare the md5 among VMs.", logging.info)
md5_set = set()
for vm, info in mapping.items():
session = info['session']
for fs in info['filesystems']:
shared_data = fs['guest_file']
error_context.context(
"%s: Get the md5 of %s." % (vm, shared_data), logging.info)
if os_type == "linux":
cmd_md5_vm = cmd_md5 % shared_data
else:
guest_file_win = shared_data.replace("/", "\\")
cmd_md5_vm = cmd_md5 % (volume_letter, guest_file_win)
md5_guest = session.cmd(cmd_md5_vm,
io_timeout).strip().split()[0]
logging.info(md5_guest)
md5_set.add(md5_guest)
if os_type == "linux":
error_context.context(
"%s: Umount the viriofs target %s." %
(vm, fs['fs_target']), logging.info)
utils_disk.umount(fs['fs_target'],
fs['fs_dest'],
'virtiofs',
session=session)
if len(md5_set) != 1:
test.fail('The md5 values are different among VMs.')
def run(test, params, env):
"""
Test virtio-fs by sharing the data between host and guest.
Steps:
1. Create shared directories on the host.
2. Run virtiofsd daemons on the host.
3. Boot a guest on the host with virtiofs options.
4. Log into guest then mount the virtiofs targets.
5. Generate files or run stress on the mount points inside guest.
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def get_viofs_exe(session):
"""
Get viofs.exe from virtio win iso,such as E:\viofs\2k19\amd64
"""
media_type = params["virtio_win_media_type"]
try:
get_drive_letter = getattr(virtio_win, "drive_letter_%s" % media_type)
get_product_dirname = getattr(virtio_win,
"product_dirname_%s" % media_type)
get_arch_dirname = getattr(virtio_win, "arch_dirname_%s" % media_type)
except AttributeError:
test.error("Not supported virtio win media type '%s'", media_type)
viowin_ltr = get_drive_letter(session)
if not viowin_ltr:
test.error("Could not find virtio-win drive in guest")
guest_name = get_product_dirname(session)
if not guest_name:
test.error("Could not get product dirname of the vm")
guest_arch = get_arch_dirname(session)
if not guest_arch:
test.error("Could not get architecture dirname of the vm")
exe_middle_path = ("{name}\\{arch}" if media_type == "iso"
else "{arch}\\{name}").format(name=guest_name,
arch=guest_arch)
exe_file_name = "virtiofs.exe"
exe_find_cmd = 'dir /b /s %s\\%s | findstr "\\%s\\\\"'
exe_find_cmd %= (viowin_ltr, exe_file_name, exe_middle_path)
exe_path = session.cmd(exe_find_cmd).strip()
logging.info("Found exe file '%s'", exe_path)
return exe_path
# data io config
cmd_dd = params.get('cmd_dd')
cmd_md5 = params.get('cmd_md5')
# pjdfs test config
cmd_pjdfstest = params.get('cmd_pjdfstest')
cmd_unpack = params.get('cmd_unpack')
cmd_yum_deps = params.get('cmd_yum_deps')
cmd_autoreconf = params.get('cmd_autoreconf')
cmd_configure = params.get('cmd_configure')
cmd_make = params.get('cmd_make')
pjdfstest_pkg = params.get('pjdfstest_pkg')
username = params.get('username')
password = params.get('password')
port = params.get('file_transfer_port')
# fio config
fio_options = params.get('fio_options')
io_timeout = params.get_numeric('io_timeout')
os_type = params.get("os_type")
vm = env.get_vm(params.get("main_vm"))
vm.verify_alive()
session = vm.wait_for_login()
host_addr = vm.get_address()
if os_type == "windows":
cmd_timeout = params.get_numeric("cmd_timeout", 120)
driver_name = params["driver_name"]
install_path = params["install_path"]
check_installed_cmd = params["check_installed_cmd"] % install_path
# Check whether windows driver is running,and enable driver verifier
session = utils_test.qemu.windrv_check_running_verifier(session,
vm, test,
driver_name)
# install winfsp tool
error_context.context("Install winfsp for windows guest.",
logging.info)
installed = session.cmd_status(check_installed_cmd) == 0
if installed:
logging.info("Winfsp tool is already installed.")
else:
install_cmd = utils_misc.set_winutils_letter(session,
params["install_cmd"])
session.cmd(install_cmd, cmd_timeout)
if not utils_misc.wait_for(lambda: not session.cmd_status(
check_installed_cmd), 60):
test.error("Winfsp tool is not installed.")
for fs in params.objects("filesystems"):
fs_params = params.object_params(fs)
fs_target = fs_params.get("fs_target")
fs_dest = fs_params.get("fs_dest")
fs_source = fs_params.get("fs_source_dir")
base_dir = fs_params.get("fs_source_base_dir",
data_dir.get_data_dir())
if not os.path.isabs(fs_source):
fs_source = os.path.join(base_dir, fs_source)
host_data = os.path.join(fs_source, 'fs_test')
if os_type == "linux":
error_context.context("Create a destination directory %s "
"inside guest." % fs_dest, logging.info)
utils_misc.make_dirs(fs_dest, session)
error_context.context("Mount virtiofs target %s to %s inside"
" guest." % (fs_target, fs_dest),
logging.info)
utils_disk.mount(fs_target, fs_dest, 'virtiofs', session=session)
else:
error_context.context("Start virtiofs service in guest.", logging.info)
exe_path = get_viofs_exe(session)
start_vfs_cmd = params["start_vfs_cmd"] % exe_path
session.sendline(start_vfs_cmd)
error_context.context("Check if virtiofs service is started.",
logging.info)
check_virtiofs_cmd = params["check_virtiofs_cmd"]
if not utils_misc.wait_for(lambda: re.search("virtiofs",
session.cmd_output(
check_virtiofs_cmd),
re.IGNORECASE), 30):
test.fail("Virtiofs service is failed to start.")
virtio_fs_disk_label = fs_target
error_context.context("Get Volume letter of virtio fs target, the disk"
"lable is %s." % virtio_fs_disk_label,
logging.info)
vol_con = "VolumeName='%s'" % virtio_fs_disk_label
vol_func = utils_misc.get_win_disk_vol(session, condition=vol_con)
volume_letter = utils_misc.wait_for(lambda: vol_func, cmd_timeout)
fs_dest = "%s:" % volume_letter
guest_file = os.path.join(fs_dest, 'fs_test')
logging.info("The guest file in shared dir is %s" % guest_file)
try:
if cmd_dd:
logging.info("Creating file under %s inside guest." % fs_dest)
session.cmd(cmd_dd % guest_file, io_timeout)
if os_type == "linux":
cmd_md5_vm = cmd_md5 % guest_file
else:
guest_file_win = guest_file.replace("/", "\\")
cmd_md5_vm = cmd_md5 % (volume_letter, guest_file_win)
md5_guest = session.cmd_output(cmd_md5_vm, io_timeout).strip().split()[0]
logging.info(md5_guest)
md5_host = process.run("md5sum %s" % host_data,
io_timeout).stdout_text.strip().split()[0]
if md5_guest != md5_host:
test.fail('The md5 value of host is not same to guest.')
if fio_options:
error_context.context("Run fio on %s." % fs_dest, logging.info)
fio = generate_instance(params, vm, 'fio')
try:
fio.run(fio_options % guest_file, io_timeout)
finally:
fio.clean()
vm.verify_dmesg()
if cmd_pjdfstest:
error_context.context("Run pjdfstest on %s." % fs_dest, logging.info)
host_path = os.path.join(data_dir.get_deps_dir('pjdfstest'), pjdfstest_pkg)
scp_to_remote(host_addr, port, username, password, host_path, fs_dest)
session.cmd(cmd_unpack.format(fs_dest), 180)
session.cmd(cmd_yum_deps, 180)
session.cmd(cmd_autoreconf % fs_dest, 180)
session.cmd(cmd_configure.format(fs_dest), 180)
session.cmd(cmd_make % fs_dest, io_timeout)
session.cmd(cmd_pjdfstest % fs_dest, io_timeout)
finally:
if os_type == "linux":
utils_disk.umount(fs_target, fs_dest, 'virtiofs', session=session)
utils_misc.safe_rmdir(fs_dest, session=session)
def run(test, params, env):
"""
Block performance test with fio
Steps:
1) boot up guest with one data disk on specified backend and pin qemu-kvm
process to the last numa node on host
2) pin guest vcpu and vhost threads to cpus of last numa node repectively
3) format data disk and run fio in guest
4) collect fio results and host info
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment
"""
def fio_thread():
"""
run fio command in guest
"""
# generate instance with fio
fio = generate_instance(params, vm, 'fio')
try:
fio.run(run_fio_options)
finally:
fio.clean()
def clean_tmp_files(session, os_type, guest_result_file, timeout):
"""
Clean temporary test result file inside guest
:param session: VM session
:param os_type: guest typet
:param guest_result_file: fio result file in guest
:param timeout: Timeout in seconds
"""
if os_type == "linux":
session.cmd("rm -rf %s" % guest_result_file, timeout)
elif os_type == "windows":
session.cmd("del /f/s/q %s" % guest_result_file, timeout)
def _pin_vm_threads(node):
"""
pin guest vcpu and vhost threads to cpus of a numa node repectively
:param node: which numa node to pin
"""
if node:
if not isinstance(node, utils_misc.NumaNode):
node = utils_misc.NumaNode(int(node))
utils_test.qemu.pin_vm_threads(vm, node)
# login virtual machine
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
login_timeout = int(params.get("login_timeout", 360))
session = vm.wait_for_login(timeout=login_timeout)
process.system_output("numactl --hardware")
process.system_output("numactl --show")
_pin_vm_threads(params.get("numa_node"))
# get parameter from dictionary
fio_options = params["fio_options"]
rw = params["rw"]
block_size = params["block_size"]
iodepth = params["iodepth"]
threads = params["threads"]
cmd_timeout = int(params.get("cmd_timeout", 1200))
order_list = params["order_list"]
driver_format = params.get("drive_format")
kvm_ver_chk_cmd = params.get("kvm_ver_chk_cmd")
guest_ver_cmd = params["guest_ver_cmd"]
pattern = params["pattern"]
pre_cmd = params["pre_cmd"]
guest_result_file = params["guest_result_file"]
format = params.get("format")
os_type = params.get("os_type", "linux")
drop_cache = params.get("drop_cache")
num_disk = params.get("num_disk")
result_path = utils_misc.get_path(test.resultsdir, "fio_result.RHS")
result_file = open(result_path, "w")
# scratch host and windows guest version info
get_version(session, result_file, kvm_ver_chk_cmd, guest_ver_cmd, os_type,
driver_format, cmd_timeout)
# online disk
if os_type == "windows":
for num in range(1, int(num_disk) + 1):
disks = check_disk_status(session, cmd_timeout, num)
diskstatus = re.findall(r"Disk\s+\d+\s+(\w+).*?\s+\d+",
disks[0])[0]
if diskstatus == "Offline":
online_disk_cmd = params.get("online_disk_cmd")
online_disk_run = online_disk_cmd % num
(s, o) = session.cmd_status_output(online_disk_run,
timeout=cmd_timeout)
if s:
test.fail("Failed to online disk: %s" % o)
for fs in params.objects("filesystems"):
fs_params = params.object_params(fs)
fs_target = fs_params.get("fs_target")
fs_dest = fs_params.get("fs_dest")
fs_source = fs_params.get("fs_source_dir")
error_context.context(
"Create a destination directory %s "
"inside guest." % fs_dest, logging.info)
utils_misc.make_dirs(fs_dest, session)
error_context.context(
"Mount virtiofs target %s to %s inside"
" guest." % (fs_target, fs_dest), logging.info)
if not utils_disk.mount(
fs_target, fs_dest, 'virtiofs', session=session):
test.fail('Mount virtiofs target failed.')
# format disk
if format == "True":
session.cmd(pre_cmd, cmd_timeout)
# get order_list
order_line = ""
for order in order_list.split():
order_line += "%s|" % format_result(order)
# get result tested by each scenario
for io_pattern in rw.split():
result_file.write("Category:%s\n" % io_pattern)
result_file.write("%s\n" % order_line.rstrip("|"))
for bs in block_size.split():
for io_depth in iodepth.split():
for numjobs in threads.split():
line = ""
line += "%s|" % format_result(bs[:-1])
line += "%s|" % format_result(io_depth)
line += "%s|" % format_result(numjobs)
if format == "True" or params.objects("filesystems"):
file_name = io_pattern + "_" + bs + "_" + io_depth
run_fio_options = fio_options % (
io_pattern, bs, io_depth, file_name, numjobs)
else:
run_fio_options = fio_options % (io_pattern, bs,
io_depth, numjobs)
logging.info("run_fio_options are: %s", run_fio_options)
if os_type == "linux":
(s, o) = session.cmd_status_output(drop_cache,
timeout=cmd_timeout)
if s:
test.fail("Failed to free memory: %s" % o)
cpu_file = os.path.join(data_dir.get_tmp_dir(), "cpus")
io_exits_b = int(
process.system_output(
"cat /sys/kernel/debug/kvm/exits"))
fio_t = threading.Thread(target=fio_thread)
fio_t.start()
process.system_output("mpstat 1 60 > %s" % cpu_file,
shell=True)
fio_t.join()
io_exits_a = int(
process.system_output(
"cat /sys/kernel/debug/kvm/exits"))
vm.copy_files_from(guest_result_file,
data_dir.get_tmp_dir())
fio_result_file = os.path.join(data_dir.get_tmp_dir(),
"fio_result")
o = process.system_output("egrep '(read|write)' %s" %
fio_result_file).decode()
results = re.findall(pattern, o)
o = process.system_output("egrep 'lat' %s" %
fio_result_file).decode()
laten = re.findall(
r"\s{5}lat\s\((\wsec)\).*?avg=[\s]?(\d+(?:[\.][\d]+)?).*?",
o)
bw = float(utils_numeric.normalize_data_size(
results[0][1]))
iops = float(
utils_numeric.normalize_data_size(results[0][0],
order_magnitude="B",
factor=1000))
if os_type == "linux" and not params.objects(
"filesystems"):
o = process.system_output("egrep 'util' %s" %
fio_result_file).decode()
util = float(
re.findall(r".*?util=(\d+(?:[\.][\d]+))%", o)[0])
lat = float(laten[0][1]) / 1000 if laten[0][0] == "usec" \
else float(laten[0][1])
if re.findall("rw", io_pattern):
bw = bw + float(
utils_numeric.normalize_data_size(results[1][1]))
iops = iops + float(
utils_numeric.normalize_data_size(
results[1][0],
order_magnitude="B",
factor=1000))
lat1 = float(laten[1][1]) / 1000 \
if laten[1][0] == "usec" else float(laten[1][1])
lat = lat + lat1
ret = process.system_output("tail -n 1 %s" % cpu_file)
idle = float(ret.split()[-1])
iowait = float(ret.split()[5])
cpu = 100 - idle - iowait
normal = bw / cpu
io_exits = io_exits_a - io_exits_b
for result in bw, iops, lat, cpu, normal:
line += "%s|" % format_result(result)
if os_type == "windows":
line += "%s" % format_result(io_exits)
if os_type == "linux":
if not params.objects("filesystems"):
line += "%s|" % format_result(io_exits)
line += "%s" % format_result(util)
else:
line += "%s" % format_result(io_exits)
result_file.write("%s\n" % line)
# del temporary files in guest os
clean_tmp_files(session, os_type, guest_result_file, cmd_timeout)
result_file.close()
for fs in params.objects("filesystems"):
fs_params = params.object_params(fs)
fs_target = fs_params.get("fs_target")
fs_dest = fs_params.get("fs_dest")
utils_disk.umount(fs_target, fs_dest, 'virtiofs', session=session)
utils_misc.safe_rmdir(fs_dest, session=session)
session.close()
def run(test, params, env):
"""
Test virtio-fs by sharing the data between host and guest.
Steps:
1. Create shared directories on the host.
2. Run virtiofsd daemons on the host.
3. Boot a guest on the host with virtiofs options.
4. Log into guest then mount the virtiofs targets.
5. Generate files or run stress on the mount points inside guest.
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
cmd_dd = params.get('cmd_dd')
cmd_md5 = params.get('cmd_md5')
cmd_pjdfstest = params.get('cmd_pjdfstest')
cmd_unpack = params.get('cmd_unpack')
cmd_yum_deps = params.get('cmd_yum_deps')
cmd_autoreconf = params.get('cmd_autoreconf')
cmd_configure = params.get('cmd_configure')
cmd_make = params.get('cmd_make')
pjdfstest_pkg = params.get('pjdfstest_pkg')
fio_options = params.get('fio_options')
io_timeout = params.get_numeric('io_timeout')
username = params.get('username')
password = params.get('password')
port = params.get('file_transfer_port')
vm = env.get_vm(params.get("main_vm"))
vm.verify_alive()
session = vm.wait_for_login()
host_addr = vm.get_address()
for fs in params.objects("filesystems"):
fs_params = params.object_params(fs)
fs_target = fs_params.get("fs_target")
fs_dest = fs_params.get("fs_dest")
fs_source = fs_params.get("fs_source_dir")
base_dir = fs_params.get("fs_source_base_dir", data_dir.get_data_dir())
if not os.path.isabs(fs_source):
fs_source = os.path.join(base_dir, fs_source)
guest_data = os.path.join(fs_dest, 'fs_test')
host_data = os.path.join(fs_source, 'fs_test')
error_context.context(
"Create a destination directory %s "
"inside guest." % fs_dest, logging.info)
utils_misc.make_dirs(fs_dest, session)
error_context.context(
"Mount virtiofs target %s to %s inside guest." %
(fs_target, fs_dest), logging.info)
utils_disk.mount(fs_target, fs_dest, 'virtiofs', session=session)
try:
if cmd_dd:
logging.info("Creating file under %s inside guest." % fs_dest)
session.cmd(cmd_dd % guest_data, io_timeout)
logging.info("Compare the md5 between guest and host.")
md5_guest = session.cmd(cmd_md5 % guest_data,
io_timeout).strip().split()[0]
logging.info(md5_guest)
md5_host = process.run(
cmd_md5 % host_data,
io_timeout).stdout_text.strip().split()[0]
if md5_guest != md5_host:
test.fail('The md5 value of host is not same to guest.')
if fio_options:
error_context.context("Run fio on %s." % fs_dest, logging.info)
fio = generate_instance(params, vm, 'fio')
try:
fio.run(fio_options % guest_data, io_timeout)
finally:
fio.clean()
vm.verify_dmesg()
if cmd_pjdfstest:
error_context.context("Run pjdfstest on %s." % fs_dest,
logging.info)
host_path = os.path.join(data_dir.get_deps_dir('pjdfstest'),
pjdfstest_pkg)
scp_to_remote(host_addr, port, username, password, host_path,
fs_dest)
session.cmd(cmd_unpack.format(fs_dest), 180)
session.cmd(cmd_yum_deps, 180)
session.cmd(cmd_autoreconf % fs_dest, 180)
session.cmd(cmd_configure.format(fs_dest), 180)
session.cmd(cmd_make % fs_dest, io_timeout)
session.cmd(cmd_pjdfstest % fs_dest, io_timeout)
finally:
utils_disk.umount(fs_target, fs_dest, 'virtiofs', session=session)
utils_misc.safe_rmdir(fs_dest, session=session)
def run(test, params, env):
"""
Test to virtio-fs with the multiple VMs and virtiofs daemons.
Steps:
1. Create shared directories on the host.
2. Run virtiofs daemons on the host.
3. Boot guests on the host with virtiofs options.
4. Log into guest then mount the virtiofs targets.
5. Generate files on the mount points inside guests.
6. Compare the md5 among guests if multiple virtiofs
daemons share the source.
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
cmd_dd = params.get('cmd_dd')
cmd_md5 = params.get('cmd_md5')
io_timeout = params.get_numeric('io_timeout')
shared_fs_source_dir = params.get('shared_fs_source_dir')
sessions = []
vms = env.get_all_vms()
for vm in vms:
vm.verify_alive()
sessions.append(vm.wait_for_login())
mapping = {}
for vm, session in zip(params.objects('vms'), sessions):
vm_params = params.object_params(vm)
mapping[vm] = {'session': session, 'filesystems': []}
for fs in vm_params.objects('filesystems'):
fs_params = vm_params.object_params(fs)
fs_target = fs_params.get("fs_target")
fs_dest = fs_params.get("fs_dest")
guest_data = os.path.join(fs_dest, 'fs_test')
mapping[vm]['filesystems'].append({'fs_target': fs_target,
'fs_dest': fs_dest,
'guest_data': guest_data})
error_context.context(
"%s: Create a destination directory %s inside guest." %
(vm, fs_dest), logging.info)
utils_misc.make_dirs(fs_dest, session)
error_context.context(
"%s: Mount the virtiofs target %s to %s inside guest." %
(vm, fs_target, fs_dest), logging.info)
utils_disk.mount(fs_target, fs_dest, 'virtiofs', session=session)
if cmd_dd:
logging.info("Creating file under %s inside guest." % fs_dest)
session.cmd(cmd_dd % guest_data, io_timeout)
if shared_fs_source_dir:
continue
error_context.context("%s: Umount the viriofs target %s." %
(vm, fs_target), logging.info)
utils_disk.umount(fs_target, fs_dest, 'virtiofs', session=session)
if shared_fs_source_dir:
error_context.context("Compare the md5 among VMs.", logging.info)
md5_set = set()
for vm, info in mapping.items():
session = info['session']
for fs in info['filesystems']:
shared_data = fs['guest_data']
error_context.context("%s: Get the md5 of %s." %
(vm, shared_data), logging.info)
val = session.cmd(cmd_md5 % shared_data).strip().split()[0]
logging.info(val)
md5_set.add(val)
error_context.context("%s: Umount the viriofs target %s." %
(vm, fs['fs_target']), logging.info)
utils_disk.umount(fs['fs_target'], fs['fs_dest'],
'virtiofs', session=session)
if len(md5_set) != 1:
test.fail('The md5 values are different among VMs.')
