def run(test, params, env):
"""
Check guest time monotonicity during migration:
1) Log into a guest.
2) Take time from guest.
3) Migrate the guest.
4) Keep guest running for a period after migration,
and record the time log.
5) Analyse log if it is exist.
:param test: QEMU test object.
:param params: Dictionary with test parameters.
:param env: Dictionary with the test environment.
"""
def get_time(cmd, test_time, session):
if os.path.isfile(host_path):
os.remove(host_path)
lasttv = "0"
cmd_timeout = int(params.get("cmd_timeout"))
start_time = time.time()
while (time.time() - start_time) < test_time:
tv = session.cmd_output(cmd, timeout=cmd_timeout)
if params.get("os_type") == 'windows':
list = re.split('[:]', tv)
tv = str(int(list[0]) * 3600 + int(
list[1]) * 60 + float(list[2]))
if float(tv) < float(lasttv):
p_tv = "time value = " + tv + "\n"
p_lasttv = "last time value = " + lasttv + "\n"
with open(host_path, 'a') as time_log:
time_log.write("time went backwards:\n" + p_tv + p_lasttv)
lasttv = tv
time.sleep(0.1)
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
boot_option_added = params.get("boot_option_added")
boot_option_removed = params.get("boot_option_removed")
if boot_option_added or boot_option_removed:
utils_test.update_boot_option(vm,
args_removed=boot_option_removed,
args_added=boot_option_added)
timeout = int(params.get("login_timeout", 360))
session1 = vm.wait_for_login(timeout=timeout)
host_path = params.get("host_path")
cmd = params.get("cmd_get_time")
test_time = int(params.get("time_linger", "60"))
try:
# take time
logging.info("Start take guest time")
bg = utils_misc.InterruptedThread(get_time, (cmd, test_time, session1))
bg.start()
# migration
logging.info("Start migration")
vm.migrate()
# log in
logging.info("Logging in after migration...")
session2 = vm.wait_for_login(timeout=timeout)
if not session2:
test.fail("Could not log in after migration")
logging.info("Logged in after migration")
# linger a while
time.sleep(test_time)
# analyse the result
if os.path.isfile(host_path):
log_dir = os.path.join(test.outputdir,
"timedrift-monotonicity-result.txt")
shutil.copyfile(host_path, log_dir)
with open(host_path, 'r') as myfile:
for line in myfile:
if "time went backwards" in line:
test.fail("Failed Time Monotonicity testing, "
"Please check log %s" % host_path)
finally:
session1.close()
# remove flags add for this test.
if boot_option_added or boot_option_removed:
utils_test.update_boot_option(vm,
args_removed=boot_option_added,
args_added=boot_option_removed)
def run(test, params, env):
"""
KVM migration test:
1) Get a live VM and clone it.
2) Verify that the source VM supports migration. If it does, proceed with
the test.
3) Reboot the VM
4) Send a migration command to the source VM and wait until it's finished.
5) Kill off the source VM.
6) Log into the destination VM after the migration is finished.
:param test: kvm test object.
:param params: Dictionary with test parameters.
:param env: Dictionary with the test environment.
"""
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
login_timeout = int(params.get("login_timeout", 360))
session = vm.wait_for_login(timeout=login_timeout)
mig_timeout = float(params.get("mig_timeout", "3600"))
mig_protocol = params.get("migration_protocol", "tcp")
mig_cancel_delay = int(params.get("mig_cancel") == "yes") * 2
migration_exec_cmd_src = params.get("migration_exec_cmd_src")
migration_exec_cmd_dst = params.get("migration_exec_cmd_dst")
pre_migrate = migration.get_functions(params.get("pre_migrate"),
migration.__dict__)
post_migrate = migration.get_functions(params.get("post_migrate"),
migration.__dict__)
if migration_exec_cmd_src and "%s" in migration_exec_cmd_src:
mig_file = os.path.join(
tempfile.mkdtemp(prefix="migrate", dir=test.workdir),
"migrate_file")
migration_exec_cmd_src %= mig_file
migration_exec_cmd_dst %= mig_file
try:
# Reboot the VM in the background
bg = utils_misc.InterruptedThread(vm.reboot,
kwargs={
'session': session,
'timeout': login_timeout
})
bg.start()
try:
while bg.isAlive():
for func in pre_migrate:
func(vm, params, test)
vm.migrate(mig_timeout,
mig_protocol,
mig_cancel_delay,
env=env,
migration_exec_cmd_src=migration_exec_cmd_src,
migration_exec_cmd_dst=migration_exec_cmd_dst)
# run some functions after migrate finish.
for func in post_migrate:
func(vm, params, test)
except Exception:
# If something bad happened in the main thread, ignore exceptions
# raised in the background thread
bg.join(suppress_exception=True)
raise
else:
session = bg.join()
finally:
session.close()
def run(test, params, env):
"""
Nic bonding test in guest.
1) Start guest with four nic models.
2) Setup bond0 in guest by script nic_bonding_guest.py.
3) Execute file transfer test between guest and host.
4) Repeatedly put down/up interfaces by set_link
5) Execute file transfer test between guest and host.
:param test: Kvm test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
timeout = int(params.get("login_timeout", 1200))
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session_serial = vm.wait_for_serial_login(timeout=timeout)
ifnames = [utils_net.get_linux_ifname(session_serial,
vm.get_mac_address(vlan))
for vlan, nic in enumerate(vm.virtnet)]
ssh_login_cmd = (
"echo LoginGraceTime 5m >> /etc/ssh/sshd_config &&"
" systemctl restart sshd.service || service sshd restart")
session_serial.cmd_output_safe(ssh_login_cmd)
# get params of bonding
nm_stop_cmd = "pidof NetworkManager && service NetworkManager stop; true"
session_serial.cmd_output_safe(nm_stop_cmd)
modprobe_cmd = "modprobe bonding"
bonding_params = params.get("bonding_params")
if bonding_params:
modprobe_cmd += " %s" % bonding_params
session_serial.cmd_output_safe(modprobe_cmd)
session_serial.cmd_output_safe("ifconfig bond0 up")
setup_cmd = "ifenslave bond0 " + " ".join(ifnames)
session_serial.cmd_output_safe(setup_cmd)
# do a pgrep to check if dhclient has already been running
pgrep_cmd = "pgrep dhclient"
try:
session_serial.cmd_output_safe(pgrep_cmd)
# if dhclient is there, killl it
except aexpect.ShellCmdError:
logging.info("it's safe to run dhclient now")
else:
logging.info("dhclient is already running, kill it")
session_serial.cmd_output_safe("killall -9 dhclient")
time.sleep(1)
session_serial.cmd_output_safe("dhclient bond0")
# get_bonding_nic_mac and ip
try:
logging.info("Test file transferring:")
utils_test.run_file_transfer(test, params, env)
logging.info("Failover test with file transfer")
transfer_thread = utils_misc.InterruptedThread(
utils_test.run_file_transfer, (test, params, env))
transfer_thread.start()
try:
while transfer_thread.isAlive():
for vlan, nic in enumerate(vm.virtnet):
device_id = nic.device_id
if not device_id:
test.error("Could not find peer device for"
" nic device %s" % nic)
vm.set_link(device_id, up=False)
time.sleep(random.randint(1, 30))
vm.set_link(device_id, up=True)
time.sleep(random.randint(1, 30))
except Exception:
transfer_thread.join(suppress_exception=True)
raise
else:
transfer_thread.join()
logging.info("Failover test 2 with file transfer")
transfer_thread = utils_misc.InterruptedThread(
utils_test.run_file_transfer, (test, params, env))
transfer_thread.start()
try:
nic_num = len(vm.virtnet)
up_index = 0
while transfer_thread.isAlive():
up_index = up_index % nic_num
for num in range(nic_num):
device_id = vm.virtnet[num].device_id
if not device_id:
test.error("Could not find peer device for"
" nic device %s" % nic)
if num == up_index:
vm.set_link(device_id, up=True)
else:
vm.set_link(device_id, up=False)
time.sleep(random.randint(1, 5))
up_index += 1
except Exception:
transfer_thread.join(suppress_exception=True)
raise
else:
transfer_thread.join()
finally:
session_serial.sendline("ifenslave -d bond0 " + " ".join(ifnames))
session_serial.sendline("kill -9 `pgrep dhclient`")
session_serial.sendline("sed -i '$ d' /etc/ssh/sshd_config")
def run_bg_test(target, args=(), kwargs={}):
""" Run the test background. """
error_context.context(target.__doc__, logging.info)
thread = utils_misc.InterruptedThread(target, args, kwargs)
thread.start()
return thread
def run(test, params, env):
"""
iperf testing with multicast/multiqueue.
1) Boot up VM
2) Prepare the iperf environment
3) Select guest or host to start iperf server/client
4) Execute iperf tests, analyze the result
5) Finish test and cleanup host environment
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def iperf_compile(src_path, dst_path, session=None):
"""Compile iperf and return its binary file path."""
iperf_version = params["iperf_version"]
iperf_source_path = os.path.join(dst_path, iperf_version)
compile_cmd = params["linux_compile_cmd"] % (src_path, dst_path,
iperf_source_path)
try:
if session:
logging.info("Compiling %s in guest..." % iperf_version)
session.cmd(compile_cmd)
else:
logging.info("Compiling %s in host..." % iperf_version)
process.run(compile_cmd, shell=True, verbose=False)
except (process.CmdError, ShellCmdError) as err_msg:
logging.error(err_msg)
test.error("Failed to compile iperf")
else:
iperf_bin_name = re.sub(r'[-2]', '', iperf_version.split('.')[0])
return os.path.join(iperf_source_path, 'src', iperf_bin_name)
def iperf_start(session, iperf_path, options, catch_data):
"""Start iperf session, analyze result if catch_data."""
iperf_cmd = iperf_path + options
try:
info_text = "Start iperf session in %s with cmd: %s"
if session:
logging.info(info_text % ("guest", iperf_cmd))
data_info = session.cmd_output(iperf_cmd, timeout=120)
else:
logging.info(info_text % ("host", iperf_cmd))
data_info = process.system_output(iperf_cmd, timeout=120,
verbose=False).decode()
except Exception as err_msg:
logging.error(str(err_msg))
test.error("Failed to start iperf session")
else:
if catch_data:
logging.debug("Full connection log:\n%s" % data_info)
parallel_cur = len(re.findall(catch_data, data_info))
parallel_exp = int(params.get("parallel_num", 0))
if not parallel_cur:
test.fail("iperf client not connected to server")
elif parallel_exp and parallel_cur != parallel_exp:
test.fail("Number of parallel threads running(%d) is "
"inconsistent with expectations(%d)"
% (parallel_cur, parallel_exp))
logging.info("iperf client successfully connected to server")
def is_iperf_running(name_pattern, session=None):
if session:
check_iperf_cmd = params["check_iperf_cmd"] % name_pattern
status = serial_session.cmd_status(check_iperf_cmd)
else:
status = process.system("pgrep -f %s" % name_pattern,
ignore_status=True, verbose=False)
return status == 0
os_type = params["os_type"]
login_timeout = int(params.get("login_timeout", 360))
fw_stop_cmd = params["fw_stop_cmd"]
tmp_dir = params.get("tmp_dir", "/tmp/")
iperf_test_duration = int(params["iperf_test_duration"])
iperf_deps_dir = data_dir.get_deps_dir("iperf")
host_iperf_file = params["host_iperf_file"]
guest_iperf_file = params.get('guest_iperf_file', host_iperf_file)
host_iperf_src_path = os.path.join(iperf_deps_dir, host_iperf_file)
guest_iperf_remote_path = os.path.join(iperf_deps_dir, guest_iperf_file)
guest_iperf_path = params.get('guest_iperf_path', tmp_dir)
guest_iperf_src_path = os.path.join(guest_iperf_path, guest_iperf_file)
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
serial_session = vm.wait_for_serial_login(timeout=login_timeout)
guest_session = vm.wait_for_login(timeout=login_timeout)
guest_session.cmd(fw_stop_cmd, ignore_all_errors=True)
vm.copy_files_to(guest_iperf_remote_path, guest_iperf_path)
host_ip_addr = utils_net.get_host_ip_address(params)
guest_ip_addr = vm.get_address()
host_iperf_bin = iperf_compile(host_iperf_src_path, tmp_dir)
if os_type == 'linux':
if not utils_package.package_install("gcc-c++", guest_session):
test.cancel("Please install gcc-c++ to proceed")
guest_iperf__bin = iperf_compile(guest_iperf_src_path, tmp_dir, guest_session)
else:
guest_iperf__bin = guest_iperf_src_path
iperf_deplist = params.get("iperf_deplist")
if iperf_deplist:
for d_name in iperf_deplist.split(','):
dep_path = os.path.join(data_dir.get_deps_dir("iperf"), d_name)
vm.copy_files_to(dep_path, guest_iperf_path)
search_pattern = {'host': host_iperf_bin.replace('src/', 'src/.*'),
'linux': guest_iperf__bin.replace('src/', 'src/.*'),
'windows': guest_iperf_file}
if params.get("iperf_server") == params["main_vm"]:
s_ip = params.get("multicast_addr", guest_ip_addr)
s_info = [search_pattern[os_type], s_ip, guest_session, guest_iperf__bin]
c_info = [search_pattern["host"], host_ip_addr, None, host_iperf_bin]
else:
s_ip = params.get("multicast_addr", host_ip_addr)
s_info = [search_pattern["host"], s_ip, None, host_iperf_bin]
c_info = [search_pattern[os_type], guest_ip_addr, guest_session, guest_iperf__bin]
s_catch_data = params["catch_data"] % (s_info[1], c_info[1])
s_options = params["iperf_server_options"]
c_options = params["iperf_client_options"] % (s_info[1], c_info[1],
iperf_test_duration)
s_info.extend([s_options, s_catch_data])
c_info.extend([c_options, None])
try:
s_start_args = tuple(s_info[-4:])
c_start_args = tuple(c_info[-4:])
bg_server = utils_misc.InterruptedThread(iperf_start, s_start_args)
bg_client = utils_misc.InterruptedThread(iperf_start, c_start_args)
bg_server.start()
if not utils_misc.wait_for(lambda: is_iperf_running(s_info[0],
s_info[2]), 5, 2):
test.error("Failed to start iperf server.")
error_context.context("iperf server has started.", logging.info)
bg_client.start()
if not utils_misc.wait_for(lambda: is_iperf_running(c_info[0],
c_info[2]), 5):
test.error("Failed to start iperf client.")
error_context.context("iperf client has started.", logging.info)
utils_misc.wait_for(lambda: not is_iperf_running(c_info[0], c_info[2]),
iperf_test_duration, 0, 5,
"Waiting for iperf test to finish.")
bg_server.join(timeout=60)
bg_client.join(timeout=60)
finally:
logging.info("Cleanup host environment...")
if is_iperf_running(search_pattern["host"]):
process.run('pkill -9 -f %s' % search_pattern["host"],
verbose=False, ignore_status=True)
shutil.rmtree(host_iperf_bin.rsplit('/', 2)[0], ignore_errors=True)
guest_session.close()
serial_session.close()
def run(test, params, env):
"""
Run NTttcp on Windows guest
1) Install NTttcp in server/client side by Autoit
2) Start NTttcp in server/client side
3) Get test results
:param test: kvm test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
login_timeout = int(params.get("login_timeout", 360))
results_path = os.path.join(test.resultsdir,
'raw_output_%s' % test.iteration)
platform = "x86"
if "64" in params["vm_arch_name"]:
platform = "x64"
buffers = params.get("buffers").split()
buf_num = params.get("buf_num", 200000)
session_num = params.get("session_num")
timeout = int(params.get("timeout")) * int(session_num)
driver_name = params.get("driver_name", "netkvm")
vm_sender = env.get_vm(params["main_vm"])
vm_sender.verify_alive()
# verify driver
_verify_vm_driver(vm_sender, test, driver_name)
logging.debug(
process.system("numactl --hardware", ignore_status=True, shell=True))
logging.debug(
process.system("numactl --show", ignore_status=True, shell=True))
# pin guest vcpus/memory/vhost threads to last numa node of host by default
if params.get('numa_node'):
numa_node = int(params.get('numa_node'))
node = utils_misc.NumaNode(numa_node)
utils_test.qemu.pin_vm_threads(vm_sender, node)
vm_receiver = env.get_vm("vm2")
vm_receiver.verify_alive()
_verify_vm_driver(vm_receiver, test, driver_name)
try:
sess = None
sess = vm_receiver.wait_for_login(timeout=login_timeout)
receiver_addr = vm_receiver.get_address()
if not receiver_addr:
test.error("Can't get receiver(%s) ip address" % vm_receiver.name)
if params.get('numa_node'):
utils_test.qemu.pin_vm_threads(vm_receiver, node)
finally:
if sess:
sess.close()
@error_context.context_aware
def install_ntttcp(session):
""" Install ntttcp through a remote session """
logging.info("Installing NTttcp ...")
try:
# Don't install ntttcp if it's already installed
error_context.context("NTttcp directory already exists")
session.cmd(params.get("check_ntttcp_cmd"))
except aexpect.ShellCmdError:
ntttcp_install_cmd = params.get("ntttcp_install_cmd")
ntttcp_install_cmd = utils_misc.set_winutils_letter(
session, ntttcp_install_cmd)
error_context.context("Installing NTttcp on guest")
session.cmd(ntttcp_install_cmd % (platform, platform), timeout=200)
def receiver():
""" Receive side """
logging.info("Starting receiver process on %s", receiver_addr)
session = vm_receiver.wait_for_login(timeout=login_timeout)
install_ntttcp(session)
ntttcp_receiver_cmd = params.get("ntttcp_receiver_cmd")
global _receiver_ready
f = open(results_path + ".receiver", 'a')
for b in buffers:
utils_misc.wait_for(lambda: not _wait(), timeout)
_receiver_ready = True
rbuf = params.get("fixed_rbuf", b)
cmd = ntttcp_receiver_cmd % (session_num, receiver_addr, rbuf,
buf_num)
r = session.cmd_output(cmd,
timeout=timeout,
print_func=logging.debug)
f.write("Send buffer size: %s\n%s\n%s" % (b, cmd, r))
f.close()
session.close()
def _wait():
""" Check if receiver is ready """
global _receiver_ready
if _receiver_ready:
return _receiver_ready
return False
def sender():
""" Send side """
logging.info("Sarting sender process ...")
session = vm_sender.wait_for_login(timeout=login_timeout)
install_ntttcp(session)
ntttcp_sender_cmd = params.get("ntttcp_sender_cmd")
f = open(results_path + ".sender", 'a')
try:
global _receiver_ready
for b in buffers:
cmd = ntttcp_sender_cmd % (session_num, receiver_addr, b,
buf_num)
# Wait until receiver ready
utils_misc.wait_for(_wait, timeout)
r = session.cmd_output(cmd,
timeout=timeout,
print_func=logging.debug)
_receiver_ready = False
f.write("Send buffer size: %s\n%s\n%s" % (b, cmd, r))
finally:
f.close()
session.close()
def parse_file(resultfile):
""" Parse raw result files and generate files with standard format """
fileobj = open(resultfile, "r")
lst = []
found = False
for line in fileobj.readlines():
o = re.findall(r"Send buffer size: (\d+)", line)
if o:
bfr = o[0]
if "Total Throughput(Mbit/s)" in line:
found = True
if found:
fields = line.split()
if len(fields) == 0:
continue
try:
[float(i) for i in fields]
lst.append([bfr, fields[-1]])
except ValueError:
continue
found = False
return lst
try:
bg = utils_misc.InterruptedThread(receiver, ())
bg.start()
if bg.is_alive():
sender()
bg.join(suppress_exception=True)
else:
test.error("Can't start backgroud receiver thread")
finally:
for i in glob.glob("%s.receiver" % results_path):
f = open("%s.RHS" % results_path, "w")
raw = " buf(k)| throughput(Mbit/s)"
logging.info(raw)
f.write("#ver# %s\n#ver# host kernel: %s\n" %
(process.system_output("rpm -q qemu-kvm",
shell=True,
verbose=False,
ignore_status=True), os.uname()[2]))
desc = """#desc# The tests are sessions of "NTttcp", send buf"
" number is %s. 'throughput' was taken from ntttcp's report.
#desc# How to read the results:
#desc# - The Throughput is measured in Mbit/sec.
#desc#
""" % (buf_num)
f.write(desc)
f.write(raw + "\n")
for j in parse_file(i):
raw = "%8s| %8s" % (j[0], j[1])
logging.info(raw)
f.write(raw + "\n")
f.close()
def run(test, params, env):
"""
Test failover by team driver
1) Boot a vm with 4 nics.
2) inside guest, configure the team driver.
3) inside guest, ping host
4) inside guest, repeated down the slaves one by one.
5) check ping_result.
:param test: Kvm test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def team_port_add(ifnames, team_if):
"""Team0 add ports and return the ip link result for debuging"""
for port in ifnames:
session_serial.cmd_output_safe(params["clearip_cmd"] % port)
session_serial.cmd_output_safe(params["setdown_cmd"] % port)
session_serial.cmd_output_safe(params["addport_cmd"] % port)
output_teamnl = session_serial.cmd_output_safe(params["portchk_cmd"])
ports = re.findall(r"%s" % params["ptn_teamnl"], output_teamnl)
for port in ifnames:
if port not in ports:
test.fail("Add %s to %s failed." % (port, team_if))
session_serial.cmd_output_safe(params["killdhclient_cmd"])
output = session_serial.cmd_output_safe(params["getip_cmd"],
timeout=300)
team_ip = re.search(r"%s" % params["ptn_ipv4"], output).group()
if not team_ip:
test.fail("Failed to get ip address of %s" % team_if)
return ports, team_ip
def failover(ifnames, timeout):
"""func for failover"""
time.sleep(3)
starttime = time.time()
while True:
pid_ping = session_serial.cmd_output_safe("pidof ping")
pid = re.findall(r"(\d+)", pid_ping)
if not pid:
break
# if ping finished, will break the loop.
for port in ifnames:
session_serial.cmd_output_safe(params["setdown_cmd"] % port)
time.sleep(random.randint(5, 30))
session_serial.cmd_output_safe(params["setup_cmd"] % port)
endtime = time.time()
timegap = endtime - starttime
if timegap > timeout:
break
def check_ping(status, output):
""" ratio <5% is acceptance."""
if status != 0:
test.fail("Ping failed, staus:%s, output:%s" % (status, output))
# if status != 0 the ping process seams hit issue.
ratio = utils_test.get_loss_ratio(output)
if ratio == -1:
test.fail("The ratio is %s, and status is %s, "
"output is %s" % (ratio, status, output))
elif ratio > int(params["failed_ratio"]):
test.fail("The loss raito is %s, test failed" % ratio)
logging.info("ping pass with loss raito:%s, that less than %s" %
(ratio, params["failed_ratio"]))
def team_if_exist():
""" judge if team is alive well."""
team_exists_cmd = params.get("team_if_exists_cmd")
return session_serial.cmd_status(team_exists_cmd) == 0
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
timeout = int(params.get("login_timeout", 1200))
session_serial = vm.wait_for_serial_login(timeout=timeout)
ifnames = [
utils_net.get_linux_ifname(session_serial, vm.get_mac_address(vlan))
for vlan, nic in enumerate(vm.virtnet)
]
session_serial.cmd_output_safe(params["nm_stop_cmd"])
team_if = params.get("team_if")
# initial
error_context.context("Step1: Configure the team environment",
logging.info)
# steps of building the teaming environment starts
modprobe_cmd = "modprobe team"
session_serial.cmd_output_safe(modprobe_cmd)
session_serial.cmd_output_safe(params["createteam_cmd"])
# this cmd is to create the team0 and correspoding userspace daemon
if not team_if_exist():
test.fail("Interface %s is not created." % team_if)
# check if team0 is created successfully
ports, team_ip = team_port_add(ifnames, team_if)
logging.debug("The list of the ports that added to %s : %s" %
(team_if, ports))
logging.debug("The ip address of %s : %s" % (team_if, team_ip))
output = session_serial.cmd_output_safe(params["team_debug_cmd"])
logging.debug("team interface configuration: %s" % output)
route_cmd = session_serial.cmd_output_safe(params["route_cmd"])
logging.debug("The route table of guest: %s" % route_cmd)
# this is not this case checkpoint, just to check if route works fine
# steps of building finished
try:
error_context.context("Login in guest via ssh", logging.info)
# steps of testing this case starts
session = vm.wait_for_login(timeout=timeout)
dest = utils_net.get_ip_address_by_interface(params["netdst"])
count = params.get("count")
timeout = float(count) * 2
error_context.context("Step2: Check if guest can ping out:",
logging.info)
status, output = utils_test.ping(dest=dest,
count=10,
interface=team_if,
timeout=30,
session=session)
check_ping(status, output)
# small ping check if the team0 works w/o failover
error_context.context(
"Step3: Start failover testing until "
"ping finished", logging.info)
failover_thread = utils_misc.InterruptedThread(failover,
(ifnames, timeout))
failover_thread.start()
# start failover loop until ping finished
error_context.context("Step4: Start ping host for %s counts" % count,
logging.info)
if failover_thread.is_alive():
status, output = utils_test.ping(dest=dest,
count=count,
interface=team_if,
timeout=float(count) * 1.5,
session=session)
error_context.context("Step5: Check if ping succeeded",
logging.info)
check_ping(status, output)
else:
test.error("The failover thread is not alive")
time.sleep(3)
try:
timeout = timeout * 1.5
failover_thread.join(timeout)
except Exception:
test.error("Failed to join the failover thread")
# finish the main steps and check the result
session_serial.cmd_output_safe(params["killteam_cmd"])
if team_if_exist():
test.fail("Remove %s failed" % team_if)
logging.info("%s removed" % team_if)
# remove the team0 and the daemon, check if succeed
finally:
if session:
session.close()
def run(test, params, env):
"""
KVM migration test:
1) Get a live VM and clone it.
2) Verify that the source VM supports migration. If it does, proceed with
the test.
3) Transfer file from host to guest.
4) Repeatedly migrate VM and wait until transfer's finished.
5) Transfer file from guest back to host.
6) Repeatedly migrate VM and wait until transfer's finished.
:param test: QEMU test object.
:param params: Dictionary with test parameters.
:param env: Dictionary with the test environment.
"""
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
login_timeout = int(params.get("login_timeout", 360))
session = vm.wait_for_login(timeout=login_timeout)
mig_timeout = float(params.get("mig_timeout", "3600"))
mig_protocol = params.get("migration_protocol", "tcp")
mig_cancel_delay = int(params.get("mig_cancel") == "yes") * 2
host_path = "/tmp/file-%s" % utils_misc.generate_random_string(6)
host_path_returned = "%s-returned" % host_path
guest_path = params.get("guest_path", "/tmp/file")
file_size = params.get("file_size", "500")
transfer_timeout = int(params.get("transfer_timeout", "240"))
migrate_between_vhost_novhost = params.get("migrate_between_vhost_novhost")
try:
process.run("dd if=/dev/urandom of=%s bs=1M count=%s" %
(host_path, file_size))
def run_and_migrate(bg):
bg.start()
try:
while bg.is_alive():
logging.info(
"File transfer not ended, starting a round of "
"migration...")
if migrate_between_vhost_novhost == "yes":
vhost_status = vm.params.get("vhost")
if vhost_status == "vhost=on":
vm.params["vhost"] = "vhost=off"
elif vhost_status == "vhost=off":
vm.params["vhost"] = "vhost=on"
vm.migrate(mig_timeout,
mig_protocol,
mig_cancel_delay,
env=env)
except Exception:
# If something bad happened in the main thread, ignore
# exceptions raised in the background thread
bg.join(suppress_exception=True)
raise
else:
bg.join()
error_context.context("transferring file to guest while migrating",
logging.info)
bg = utils_misc.InterruptedThread(
vm.copy_files_to, (host_path, guest_path),
dict(verbose=True, timeout=transfer_timeout))
run_and_migrate(bg)
error_context.context("transferring file back to host while migrating",
logging.info)
bg = utils_misc.InterruptedThread(
vm.copy_files_from, (guest_path, host_path_returned),
dict(verbose=True, timeout=transfer_timeout))
run_and_migrate(bg)
# Make sure the returned file is identical to the original one
error_context.context("comparing hashes", logging.info)
orig_hash = crypto.hash_file(host_path)
returned_hash = crypto.hash_file(host_path_returned)
if orig_hash != returned_hash:
test.fail("Returned file hash (%s) differs from "
"original one (%s)" % (returned_hash, orig_hash))
error_context.context()
finally:
session.close()
if os.path.isfile(host_path):
os.remove(host_path)
if os.path.isfile(host_path_returned):
os.remove(host_path_returned)
def run(test, params, env):
"""
Qemu guest pxe boot test:
1). check npt/ept function enable, then boot vm
2). execute query/info cpus in loop
3). verify vm not paused during pxe booting
params:
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
def stopVMS(params, env):
"""
Kill all VMS for relaod kvm_intel/kvm_amd module;
"""
for vm in env.get_all_vms():
if vm:
vm.destroy()
env.unregister_vm(vm.name)
qemu_bin = os.path.basename(params["qemu_binary"])
process.run("killall -g %s" % qemu_bin, ignore_status=True)
time.sleep(5)
enable_mmu_cmd = None
check_mmu_cmd = None
restore_mmu_cmd = None
error_context.context("Enable ept(npt)", logging.info)
try:
flag = list(
filter(lambda x: x in utils_misc.get_cpu_flags(),
['ept', 'npt']))[0]
except IndexError:
logging.warn("Host doesn't support ept(npt)")
else:
enable_mmu_cmd = params["enable_mmu_cmd_%s" % flag]
check_mmu_cmd = params["check_mmu_cmd_%s" % flag]
status = process.system(check_mmu_cmd,
timeout=120,
ignore_status=True,
shell=True)
if status != 0:
stopVMS(params, env)
process.run(enable_mmu_cmd, shell=True)
restore_mmu_cmd = params["restore_mmu_cmd_%s" % flag]
params["start_vm"] = "yes"
params["kvm_vm"] = "yes"
params["paused_after_start_vm"] = "yes"
env_process.preprocess_vm(test, params, env, params["main_vm"])
bg = utils_misc.InterruptedThread(utils_test.run_virt_sub_test,
args=(
test,
params,
env,
),
kwargs={"sub_type": "pxe_boot"})
count = 0
try:
bg.start()
error_context.context("Query cpus in loop", logging.info)
vm = env.get_vm(params["main_vm"])
vm.resume()
while True:
count += 1
try:
vm.monitor.info("cpus")
vm.verify_status("running")
if not bg.is_alive():
break
except qemu_monitor.MonitorSocketError:
test.fail("Qemu looks abnormally, please read the log")
logging.info("Execute info/query cpus %d times", count)
finally:
bg.join()
if restore_mmu_cmd:
stopVMS(params, env)
process.run(restore_mmu_cmd, shell=True)
def run(test, params, env):
"""
MULTI_QUEUE chang queues number test
1) Boot up VM, and login guest
2) Enable the queues in guest
3) Run netperf_and_ping test
4) Change queues number repeatedly during netperf_and_ping stress testing
5) Reboot VM
6) Repeat above 1-4 steps
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def change_queues_number(ifname, q_number, queues_status=None):
"""
change queues number
"""
if not queues_status:
queues_status = get_queues_status(ifname)
mq_set_cmd = "ethtool -L %s combined %d" % (ifname, q_number)
output = session_serial.cmd_output_safe(mq_set_cmd)
cur_queues_status = get_queues_status(ifname)
err_msg = ""
expect_q_number = q_number
if q_number != queues_status[1] and q_number <= queues_status[0]:
if (cur_queues_status[1] != q_number
or cur_queues_status[0] != queues_status[0]):
err_msg = "Param is valid, but change queues failed, "
elif cur_queues_status != queues_status:
if q_number != queues_status[1]:
err_msg = "Param is invalid, "
err_msg += "Current queues value is not expected, "
expect_q_number = queues_status[1]
if len(err_msg) > 0:
err_msg += "current queues set is %s, " % cur_queues_status[1]
err_msg += "max allow queues set is %s, " % cur_queues_status[0]
err_msg += "when run cmd: '%s', " % mq_set_cmd
err_msg += "expect queues are %s," % expect_q_number
err_msg += "expect max allow queues are %s, " % queues_status[0]
err_msg += "output: '%s'" % output
test.fail(err_msg)
return cur_queues_status
def get_queues_status(ifname):
"""
Get queues status
"""
mq_get_cmd = "ethtool -l %s" % ifname
nic_mq_info = session_serial.cmd_output_safe(mq_get_cmd)
queues_reg = re.compile(r"Combined:\s+(\d)", re.I)
queues_info = queues_reg.findall(" ".join(nic_mq_info.splitlines()))
if len(queues_info) != 2:
err_msg = "Oops, get guest queues info failed, "
err_msg += "make sure your guest support MQ.\n"
err_msg += "Check cmd is: '%s', " % mq_get_cmd
err_msg += "Command output is: '%s'." % nic_mq_info
test.cancel(err_msg)
return [int(x) for x in queues_info]
def ping_test(dest_ip, ping_time, ping_lost_ratio):
"""
ping guest from host,until change queues finished.
"""
_, output = utils_net.ping(dest=dest_ip, timeout=ping_time)
packets_lost = utils_test.get_loss_ratio(output)
if packets_lost > ping_lost_ratio:
err = " %s%% packages lost during ping. " % packets_lost
err += "Ping command log:\n %s" % "\n".join(
output.splitlines()[-3:])
test.fail(err)
def netperf_test():
"""
Netperf stress test for nic option.
"""
try:
n_server.start()
# Run netperf with message size defined in range.
netperf_test_duration = params.get_numeric("netperf_test_duration")
test_protocols = params.get("test_protocols", "TCP_STREAM")
netperf_output_unit = params.get("netperf_output_unit")
test_option = params.get("test_option", "")
test_option += " -l %s" % netperf_test_duration
if netperf_output_unit in "GMKgmk":
test_option += " -f %s" % netperf_output_unit
t_option = "%s -t %s" % (test_option, test_protocols)
n_client.bg_start(utils_net.get_host_ip_address(params),
t_option,
params.get_numeric("netperf_para_sessions"),
params.get("netperf_cmd_prefix", ""),
package_sizes=params.get("netperf_sizes"))
if utils_misc.wait_for(n_client.is_netperf_running, 10, 0, 1,
"Wait netperf test start"):
logging.info("Netperf test start successfully.")
else:
test.error("Can not start netperf client.")
utils_misc.wait_for(
lambda: not n_client.is_netperf_running(),
netperf_test_duration, 0, 5,
"Wait netperf test finish %ss" % netperf_test_duration)
finally:
n_server.stop()
login_timeout = params.get_numeric("login_timeout", 360)
netperf_stress = params.get("run_bgstress")
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
vm.wait_for_serial_login(timeout=login_timeout)
guest_ip = vm.get_address()
n_client = utils_netperf.NetperfClient(
guest_ip,
params.get("client_path"),
netperf_source=os.path.join(data_dir.get_deps_dir("netperf"),
params.get("netperf_client_link")),
client=params.get("shell_client"),
username=params.get("username"),
password=params.get("password"),
compile_option=params.get("compile_option", ""))
n_server = utils_netperf.NetperfServer(
utils_net.get_host_ip_address(params),
params.get("server_path", "/var/tmp"),
netperf_source=os.path.join(data_dir.get_deps_dir("netperf"),
params.get("netperf_server_link")),
password=params.get("hostpassword"),
compile_option=params.get("compile_option", ""))
wait_time = params.get_numeric("wait_bg_time")
ping_lost_ratio = params.get_numeric("background_ping_package_lost_ratio",
5)
ping_time = params.get_numeric("background_ping_time")
required_reboot = True
bg_test = True
try:
while bg_test:
session_serial = vm.wait_for_serial_login(timeout=login_timeout)
n_client.session = session_serial
error_context.context("Enable multi queues in guest.",
logging.info)
for nic in vm.virtnet:
ifname = utils_net.get_linux_ifname(session_serial, nic.mac)
queues = int(nic.queues)
change_queues_number(ifname, queues)
error_context.context("Run test %s background" % netperf_stress,
logging.info)
stress_thread = utils_misc.InterruptedThread(netperf_test)
stress_thread.start()
utils_misc.wait_for(lambda: wait_time, 0, 5,
"Wait %s start background" % netperf_stress)
# ping test
error_context.context("Ping guest from host", logging.info)
args = (guest_ip, ping_time, ping_lost_ratio)
bg_ping = utils_misc.InterruptedThread(ping_test, args)
bg_ping.start()
error_context.context("Change queues number repeatedly",
logging.info)
repeat_counts = params.get_numeric("repeat_counts")
for nic in vm.virtnet:
queues = int(nic.queues)
if queues == 1:
logging.info("Nic with single queue, skip and continue")
continue
ifname = utils_net.get_linux_ifname(session_serial, nic.mac)
change_list = params.get("change_list").split(",")
for repeat_num in range(repeat_counts):
error_context.context(
"Change queues number -- %sth" % repeat_num,
logging.info)
queues_status = get_queues_status(ifname)
for q_number in change_list:
queues_status = change_queues_number(
ifname, int(q_number), queues_status)
logging.info("wait for background test finish")
try:
stress_thread.join()
except Exception as err:
err_msg = "Run %s test background error!\n "
err_msg += "Error Info: '%s'"
test.error(err_msg % (netperf_stress, err))
logging.info("Wait for background ping test finish.")
try:
bg_ping.join()
except Exception as err:
txt = "Fail to wait background ping test finish. "
txt += "Got error message %s" % err
test.fail(txt)
if required_reboot:
logging.info("Rebooting guest ...")
vm.reboot()
required_reboot = False
else:
bg_test = False
finally:
n_server.cleanup(True)
n_client.cleanup(True)
if session_serial:
session_serial.close()
def test(vt_test, test_params, env):
"""Playback of audio stream tests for remote-viewer.
Parameters
----------
vt_test : avocado.core.plugins.vt.VirtTest
QEMU test object.
test_params : virttest.utils_params.Params
Dictionary with the test parameters.
env : virttest.utils_env.Env
Dictionary with test environment.
Raises
------
TestFail
Test fails for expected behaviour.
"""
test = stest.ClientGuestTest(vt_test, test_params, env)
cfg = test.cfg
act.x_active(test.vmi_c)
act.x_active(test.vmi_g)
ssn_c = act.new_ssn(test.vmi_c)
ssn_g = act.new_ssn(test.vmi_g)
# Get default sink at the client.
cmd = r"pacmd stat | grep 'Default sink name' | " \
r"sed -e 's/^.*[[:space:]]//'"
try:
def_sink = ssn_c.cmd(cmd).rstrip('\r\n')
except aexpect.ShellCmdError as excp:
raise utils.SpiceTestFail(test, "Test failed: %s" % str(excp))
logging.info("Default sink at client is: %s", def_sink)
# Create RV session
env_local = {}
if cfg.rv_record:
env_local["PULSE_SOURCE"] = "%s.monitor" % def_sink
ssn = act.new_ssn(test.vmi_c)
act.rv_connect(test.vmi_c, ssn, env=env_local)
ret, out = commands.getstatusoutput(MAKE_WAV)
if ret:
errmsg = "Cannot generate specimen WAV file: %s" % out
raise utils.SpiceTestFail(test, errmsg)
play_cmd = "aplay %s &> /dev/null &" % cfg.audio_tgt
rec_cmd = "arecord -d %s -f cd %s" % (cfg.audio_time, cfg.audio_rec)
# Check test type
if cfg.rv_record:
logging.info("Recording test. Player is client. Recorder is guest.")
player = ssn_c
recorder = ssn_g
vm_recorder = test.vm_g
vm_player = test.vm_c
else:
logging.info("Playback test. Player is guest. Recorder is client.")
env_var = "PULSE_SOURCE=%s.monitor" % def_sink
rec_cmd = env_var + " " + rec_cmd
player = ssn_g
recorder = ssn_c
vm_recorder = test.vm_c
vm_player = test.vm_g
vm_player.copy_files_to(SPECIMEN_FILE, cfg.audio_tgt)
time.sleep(2) # wait till everything is set up
player.cmd(play_cmd)
if cfg.config_test == "migration":
bguest = utils_misc.InterruptedThread(test.vm_g.migrate, kwargs={})
bguest.start()
try:
recorder.cmd(rec_cmd, timeout=500)
except aexpect.ShellCmdError as excp:
raise utils.SpiceTestFail(test, str(excp))
if cfg.config_test == "migration":
bguest.join()
vm_recorder.copy_files_from(cfg.audio_rec, RECORDED_FILE)
if not verify_recording(RECORDED_FILE, cfg):
raise utils.SpiceTestFail(test, "Cannot verify recorded file.")
if cfg.rv_reconnect:
act.rv_disconnect(test.vmi_c)
act.rv_connect(test.vmi_c, ssn)
try:
recorder.cmd(rec_cmd, timeout=500)
except aexpect.ShellCmdError as excp:
raise utils.SpiceTestFail(test, str(excp))
vm_recorder.copy_files_from(cfg.audio_rec, RECORDED_FILE)
if not verify_recording(RECORDED_FILE, cfg):
raise utils.SpiceTestFail(test, "Cannot verify recorded file.")
def run(test, params, env):
"""
Multicast test using iperf.
1) Boot up VM(s)
2) Prepare the test environment in server/client/host,install iperf
3) Execute iperf tests, analyze the results
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def server_start(cmd, catch_data):
"""
Start the iperf server in host, and check whether the guest have connected
this server through multicast address of the server
"""
try:
process.run(cmd)
except process.CmdError as e:
if not re.findall(catch_data, str(e)):
test.fail("Client not connected '%s'" % str(e))
logging.info("Client multicast test pass "
% re.findall(catch_data, str(e)))
os_type = params.get("os_type")
win_iperf_url = params.get("win_iperf_url")
linux_iperf_url = params.get("linux_iperf_url")
iperf_version = params.get("iperf_version", "2.0.5")
transfer_timeout = int(params.get("transfer_timeout", 360))
login_timeout = int(params.get("login_timeout", 360))
dir_name = test.tmpdir
tmp_dir = params.get("tmp_dir", "/tmp/")
host_path = os.path.join(dir_name, "iperf")
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session = vm.wait_for_login(timeout=login_timeout)
clean_cmd = ""
client_ip = vm.get_address(0)
try:
error_context.context("Test Env setup")
iperf_downloaded = 0
iperf_url = linux_iperf_url
app_check_cmd = params.get("linux_app_check_cmd", "false")
app_check_exit_status = int(params.get("linux_app_check_exit_status",
"0"))
exit_status = process.system(app_check_cmd, ignore_status=True,
shell=True)
# Install iperf in host if not available
default_install_cmd = "tar zxvf %s; cd iperf-%s;"
default_install_cmd += " ./configure; make; make install"
install_cmd = params.get("linux_install_cmd", default_install_cmd)
if not exit_status == app_check_exit_status:
error_context.context("install iperf in host", logging.info)
download.get_file(iperf_url, host_path)
iperf_downloaded = 1
process.system(install_cmd % (host_path, iperf_version),
shell=True)
# The guest may not be running Linux, see if we should update the
# app_check variables
if not os_type == "linux":
app_check_cmd = params.get("win_app_check_cmd", "false")
app_check_exit_status = int(params.get("win_app_check_exit_status",
"0"))
# Install iperf in guest if not available
if not session.cmd_status(app_check_cmd) == app_check_exit_status:
error_context.context("install iperf in guest", logging.info)
if not iperf_downloaded:
download.get_file(iperf_url, host_path)
if os_type == "linux":
guest_path = (tmp_dir + "iperf.tgz")
clean_cmd = "rm -rf %s iperf-%s" % (guest_path, iperf_version)
else:
guest_path = (tmp_dir + "iperf.exe")
iperf_url = win_iperf_url
download.get_file(iperf_url, host_path)
clean_cmd = "del %s" % guest_path
vm.copy_files_to(host_path, guest_path, timeout=transfer_timeout)
if os_type == "linux":
session.cmd(install_cmd % (guest_path, iperf_version))
muliticast_addr = params.get("muliticast_addr", "225.0.0.3")
multicast_port = params.get("multicast_port", "5001")
step_msg = "Start iperf server, bind host to multicast address %s "
error_context.context(step_msg % muliticast_addr, logging.info)
server_start_cmd = ("iperf -s -u -B %s -p %s " %
(muliticast_addr, multicast_port))
default_flag = "%s port %s connected with %s"
connected_flag = params.get("connected_flag", default_flag)
catch_data = connected_flag % (muliticast_addr, multicast_port,
client_ip)
t = utils_misc.InterruptedThread(server_start,
(server_start_cmd, catch_data))
t.start()
if not _process_is_alive("iperf"):
test.error("Start iperf server failed cmd: %s" % server_start_cmd)
logging.info("Server start successfully")
step_msg = "In client try to connect server and transfer file "
step_msg += " through multicast address %s"
error_context.context(step_msg % muliticast_addr, logging.info)
if os_type == "linux":
client_cmd = "iperf"
else:
client_cmd = guest_path
start_cmd = params.get("start_client_cmd", "%s -c %s -u -p %s")
start_client_cmd = start_cmd % (client_cmd, muliticast_addr,
multicast_port)
session.cmd(start_client_cmd)
logging.info("Client start successfully")
error_context.context("Test finish, check the result", logging.info)
process.system("pkill -2 iperf")
t.join()
finally:
if _process_is_alive("iperf"):
process.system("killall -9 iperf")
process.system("rm -rf %s" % host_path)
if session:
if clean_cmd:
session.cmd(clean_cmd)
session.close()
def run(test, params, env):
"""
Nic teaming test in guest.
1) Start guest with four nic devices.
2) Setup Team in guest.
3) Execute file transfer from host to guest.
4) Repeatedly set enable/disable interfaces by 'netsh interface set'
5) Execute file transfer from guest to host.
:param test: Kvm test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
tmp_dir = params["tmp_dir"]
filesize = params.get_numeric("filesize")
dd_cmd = params["dd_cmd"]
delete_cmd = params["delete_cmd"]
login_timeout = params.get_numeric("login_timeout", 1200)
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session_serial = vm.wait_for_serial_login(timeout=login_timeout)
nics = params.objects("nics")
ifnames = ()
for i in range(len(nics)):
mac = vm.get_mac_address(i)
connection_id = utils_net.get_windows_nic_attribute(
session_serial, "macaddress", mac, "netconnectionid")
ifnames += (connection_id, )
# get params of teaming
setup_cmd = params["setup_cmd"]
status, output = session_serial.cmd_status_output(setup_cmd % ifnames)
if status:
test.fail("Failed to setup team nic from powershell,"
"status=%s, output=%s" % (status, output))
# prepare test data
guest_path = (tmp_dir + "src-%s" % utils_misc.generate_random_string(8))
host_path = os.path.join(test.tmpdir,
"tmp-%s" % utils_misc.generate_random_string(8))
logging.info("Test setup: Creating %dMB file on host", filesize)
process.run(dd_cmd % host_path, shell=True)
try:
netsh_set_cmd = "netsh interface set interface \"%s\" %s"
# transfer data
original_md5 = crypto.hash_file(host_path, algorithm="md5")
logging.info("md5 value of data original: %s", original_md5)
logging.info("Failover test with file transfer")
transfer_thread = utils_misc.InterruptedThread(vm.copy_files_to,
(host_path, guest_path))
transfer_thread.start()
try:
while transfer_thread.is_alive():
for ifname in ifnames:
session_serial.cmd(netsh_set_cmd % (ifname, "disable"))
time.sleep(random.randint(1, 30))
session_serial.cmd(netsh_set_cmd % (ifname, "enable"))
time.sleep(random.randint(1, 30))
except aexpect.ShellProcessTerminatedError:
transfer_thread.join(suppress_exception=True)
raise
else:
transfer_thread.join()
os.remove(host_path)
logging.info('Cleaning temp file on host')
logging.info("Failover test 2 with file transfer")
transfer_thread = utils_misc.InterruptedThread(vm.copy_files_from,
(guest_path, host_path))
transfer_thread.start()
try:
nic_num = len(ifnames)
index = 0
while transfer_thread.is_alive():
index = index % nic_num
for i in range(nic_num):
session_serial.cmd(netsh_set_cmd % (ifnames[i], "enable"))
for j in range(nic_num):
if i != j:
session_serial.cmd(netsh_set_cmd %
(ifnames[j], "disable"))
time.sleep(random.randint(1, 5))
index += 1
except aexpect.ShellProcessTerminatedError:
transfer_thread.join(suppress_exception=True)
raise
else:
transfer_thread.join()
current_md5 = crypto.hash_file(host_path, algorithm="md5")
logging.info("md5 value of data current: %s", current_md5)
if original_md5 != current_md5:
test.fail("File changed after transfer host -> guest "
"and guest -> host")
finally:
os.remove(host_path)
session_serial.cmd(delete_cmd % guest_path,
timeout=login_timeout,
ignore_all_errors=True)
session_serial.close()
def _run_backgroud(args):
thread_session = vm.wait_for_login(timeout=360)
thread = utils_misc.InterruptedThread(thread_session.cmd, args)
thread.start()
def run(test, params, env):
"""
Qemu guest irqbalance inactive/active test:
1) Setup host for sr-iov test.
2) Boot VM with sr-iov vf/pf assigned and multi vcpu.
3) Update irqbalance service status in guest. stop/start this server
according to request.
4) Get available network interface name in guest.
5) Start background network stress in guest.
6) Get irq number assigned to attached vfs/pfs.
7) Get the cpu number the irq running.
8) Check specified IRQ count grow on specified cpu.
9) Repeat step 7 for every 10s.
10) Balance IRQs generated by vfs/pfs to different vcpus (optional)
e.g.
echo 4 > /proc/irq/num/smp_affinity
11) Repeat step 6, 7
12) Check that specified IRQ count grow on every cpu. (optional)
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
timeout = int(params.get("login_timeout", 360))
session = vm.wait_for_login(timeout=timeout)
irqbalance_check_count = int(params.get("irqbalance_check_count", 36))
nic_interface_filter = params["nic_interface_filter"]
error_context.context("Make sure that guest have at least 2 vCPUs.",
logging.info)
cpu_count = vm.get_cpu_count()
if cpu_count < 2:
test.cancel("Test requires at least 2 vCPUs.")
msg = "Update irqbalance service status in guest if not match request."
error_context.context(msg, logging.info)
irqbalance_status = params.get("irqbalance_status", "active")
status = utils_misc.get_guest_service_status(session=session,
service="irqbalance")
service_cmd = ""
if status == "active" and irqbalance_status == "inactive":
service_cmd = "service irqbalance stop"
elif status == "inactive" and irqbalance_status == "active":
service_cmd = "service irqbalance start"
if service_cmd:
status, output = session.cmd_status_output(service_cmd)
if status:
msg = "Fail to update irqbalance service status in guest."
msg += " Command output in guest: %s" % output
test.error(msg)
error_context.context("Get first network interface name in guest.",
logging.info)
devname = get_first_network_devname(test, session, nic_interface_filter)
error_context.context("Start background network stress in guest.",
logging.info)
host_ip = utils_net.get_ip_address_by_interface(params.get('netdst'))
ping_cmd = "ping %s -f -q" % host_ip
ping_timeout = irqbalance_check_count * 10 + 100
ping_session = vm.wait_for_login(timeout=timeout)
bg_stress = utils_misc.InterruptedThread(utils_test.raw_ping,
kwargs={'command': ping_cmd,
'timeout': ping_timeout,
'session': ping_session,
'output_func': None})
bg_stress.start()
try:
error_context.context("Get irq number assigned to attached "
"VF/PF in guest", logging.info)
irq_nums_dict = get_guest_irq_info(test, session, devname, cpu_count)
if irq_nums_dict:
irqs = irq_nums_dict.keys()
msg = "Check specified IRQ count grow on specified cpu."
error_context.context(msg, logging.info)
check_irqbalance(test, session, devname, cpu_count, irqs)
irq_cpus_dict = {}
for irq in irqs:
cpus = get_irq_smp_affinity(test, session, irq)
irq_cpus_dict[irq] = cpus
if irqbalance_status == "inactive":
msg = "balance IRQs generated by vfs/pfs to different vcpus."
error_context.context(msg, logging.info)
post_irq_cpus_dict = {}
for irq in irq_cpus_dict:
balance_cpu_count = 1
cpus = []
for cpu in range(cpu_count):
if cpu not in irq_cpus_dict[irq]:
cpus.append(cpu)
if len(cpus) == balance_cpu_count:
break
set_irq_smp_affinity(test, session, irq, cpus)
post_irq_cpus_dict[irq] = cpus
for irq in irqs:
cpus = get_irq_smp_affinity(test, session, irq)
msg = "Fail to balance IRQs generated by vf/pf to different cpu"
if cpus != post_irq_cpus_dict[irq]:
test.fail(msg)
msg = "Check specified IRQ count grow on specified cpu."
error_context.context(msg, logging.info)
check_irqbalance(test, session, devname,
cpu_count, irqs,
count=irqbalance_check_count)
if irqbalance_status == "active":
msg = "Check that specified IRQ count grow on every cpu."
error_context.context(msg, logging.info)
post_irq_nums_dict = get_guest_irq_info(test, session, devname,
cpu_count)
for irq in irqs:
if irq not in post_irq_nums_dict.keys():
post_irqs = post_irq_nums_dict.keys()
msg = "Different irq detected: '%s' and '%s'." % (irqs,
post_irqs)
test.error(msg)
for cpu in range(cpu_count):
if (int(irq_nums_dict[irq][cpu]) >=
int(post_irq_nums_dict[irq][cpu])):
msg = "'Cpu%s' did not handle more interrupt" % cpu
msg += "for irq '%s'." % irq
msg += "IRQ balance information for IRQ '%s'\n" % irq
msg += "First time: %s\n" % irq_nums_dict
msg += "Just now: %s" % post_irq_nums_dict
test.fail(msg)
finally:
if bg_stress.isAlive():
bg_stress.join(suppress_exception=True)
else:
logging.warn("Background stress test already finished")
def run(test, params, env):
"""
Special hardware test case.
FC host: ibm-x3650m4-05.lab.eng.pek2.redhat.com
Disk serial name: scsi-360050763008084e6e0000000000001a4
# multipath -ll
mpathb (360050763008084e6e0000000000001a8) dm-4 IBM,2145
size=100G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=50 status=active
| `- 2:0:1:0 sde 8:64 active ready running
`-+- policy='service-time 0' prio=10 status=enabled
`- 2:0:0:0 sdd 8:48 active ready running
mpatha (360050763008084e6e0000000000001a4) dm-3 IBM,2145
size=100G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=50 status=active
| `- 1:0:1:0 sdc 8:32 active ready running
`-+- policy='service-time 0' prio=10 status=enabled
`- 1:0:0:0 sdb 8:16 active ready running
Customer Bug ID: 1741937 1673546
Test if VM paused/resume when fc storage offline/online.
1) pass-through /dev/mapper/mpatha
2) install guest on pass-through disk
3) Disconnect the storage during installation
4) Check if VM status is 'paused'
5) Connect the storage, Wait until the storage is accessible again
6) resume the vm
7) Check if VM status is 'running'
8) installation completed successfully
9) re-pass-through /dev/mapper/mpatha
10) fio test on pass-through disk
11) Disconnect any path of multipath during fio testing
12) Check if VM status is 'paused'
13) Connect the storage, Wait until the storage is accessible again
14) resume the vm
15) fio testing completed successfully
:param test: kvm test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def check_vm_status(vm, status):
"""
Check if VM has the given status or not.
:param vm: VM object.
:param status: String with desired status.
:return: True if VM status matches our desired status.
:return: False if VM status does not match our desired status.
"""
try:
current_status = vm.monitor.get_status()
vm.verify_status(status)
except (virt_vm.VMStatusError, qemu_monitor.MonitorLockError):
logging.info("Failed to check vm status, it is '%s' "
"instead of '%s'" % (current_status, status))
return False
except Exception as e:
logging.info("Failed to check vm status: %s" % six.text_type(e))
logging.info("vm status is '%s' instead of"
" '%s'" % (current_status, status))
return False
else:
logging.info("Check vm status successfully. It is '%s'" % status)
return True
def get_multipath_disks(mpath_name="mpatha"):
"""
Get all disks of multiple paths.
multipath like below:
mpatha (360050763008084e6e0000000000001a4) dm-3 IBM,2145
size=100G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=50 status=active
| `- 1:0:1:0 sdc 8:32 active ready running
`-+- policy='service-time 0' prio=10 status=enabled
`- 1:0:0:0 sdb 8:16 active ready running
:param mpath_name: multi-path name.
:return: a list. if get disks successfully or raise a error
"""
logging.info("get_multipath_disks:" + mpath_name)
disks = []
disk_str = []
outputs = process.run("multipath -ll " + mpath_name,
shell=True).stdout.decode()
outputs = outputs.split(mpath_name)[-1]
disk_str.append("active ready running")
disk_str.append("active faulty offline")
disk_str.append("failed faulty offline")
disk_str.append("failed ready running")
for line in outputs.splitlines():
if disk_str[0] in line or disk_str[1] in line or disk_str[2] \
in line or disk_str[3] in line:
disks.append(line.split()[-5])
if not disks:
test.fail("Failed to get disks by 'multipath -ll'")
else:
return disks
def get_multipath_disks_status(mpath_name="mpatha"):
"""
Get status of multiple paths.
multipath like below:
mpatha (360050763008084e6e0000000000001a4) dm-3 IBM,2145
size=100G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=50 status=active
| `- 1:0:1:0 sdc 8:32 active ready running
`-+- policy='service-time 0' prio=10 status=enabled
`- 1:0:0:0 sdb 8:16 active ready running
:param mpath_name: multi-path name.
:return: a dict. e.g. {"sdb": "running", "sdc": "running"}
"""
disks = get_multipath_disks(mpath_name)
disks_status = {}
outputs = process.run("multipath -ll " + mpath_name,
shell=True).stdout.decode()
outputs = outputs.split(mpath_name)[-1]
for line in outputs.splitlines():
for i in range(len(disks)):
if disks[i] in line:
disks_status[disks[i]] = line.strip().split()[-1]
break
if not disks_status or len(disks_status) != len(disks):
logging.info("Failed to get disks status by 'multipath -ll'")
return {}
else:
return disks_status
def compare_onepath_status(status, disk):
"""
Compare status whether equal to the given status.
This function just focus on all paths are running or all are offline.
:param status: the state of disks.
:param disk: disk kname.
:return: True, if equal to the given status or False
"""
status_dict = get_multipath_disks_status(mpath_name)
logging.debug("compare_onepath_status disk:", disk, status_dict,
status)
if disk in status_dict.keys() and status == status_dict[disk]:
return True
else:
return False
def compare_multipath_status(status, mpath_name="mpatha"):
"""
Compare status whether equal to the given status.
This function just focus on all paths are running or all are offline.
:param status: the state of disks.
:param mpath_name: multi-path name.
:return: True, if equal to the given status or False
"""
status_dict = get_multipath_disks_status(mpath_name)
logging.debug("compare_multipath_status mpath_name:", mpath_name,
status_dict, status)
if len(set(
status_dict.values())) == 1 and status in status_dict.values():
return True
else:
return False
def set_disk_status_to_online_offline(disk, status):
"""
set disk state to online/offline.
multipath like below:
mpatha (360050763008084e6e0000000000001a4) dm-3 IBM,2145
size=100G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=50 status=active
| `- 1:0:1:0 sdc 8:32 active ready running
`-+- policy='service-time 0' prio=10 status=enabled
`- 1:0:0:0 sdb 8:16 failed faulty offline
:param disk: disk name.
:param status: the state of disk.
:return: by default
"""
error_context.context("Set disk '%s' to status '%s'." % (disk, status),
logging.info)
process.run("echo %s > /sys/block/%s/device/state" % (status, disk),
shell=True)
def set_multipath_disks_status(disks, status):
"""
set multiple paths to same status. all disks online or offline.
multipath like below:
mpatha (360050763008084e6e0000000000001a4) dm-3 IBM,2145
size=100G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='service-time 0' prio=50 status=active
| `- 1:0:1:0 sdc 8:32 active ready running
`-+- policy='service-time 0' prio=10 status=enabled
`- 1:0:0:0 sdb 8:16 failed faulty offline
:param disks: disk list.
:param status: the state of disk. online/offline
:return: by default
"""
for disk in disks:
set_disk_status_to_online_offline(disk, status)
time.sleep(2)
if len(disks) == 1:
wait.wait_for(lambda: compare_onepath_status(status, disks[0]),
first=wait_time,
step=3,
timeout=60)
else:
wait.wait_for(lambda: compare_multipath_status(status),
first=wait_time,
step=3,
timeout=60)
def get_lvm_dm_name(blkdevs_used):
"""
Get dm name for lvm. such as rhel_ibm--x3650m4--05-root in below
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 278.9G 0 disk
├─sda1 8:1 0 1G 0 part /boot
└─sda2 8:2 0 277.9G 0 part
├─rhel_ibm--x3650m4--05-root 253:0 0 50G 0 lvm /
├─rhel_ibm--x3650m4--05-swap 253:1 0 15.7G 0 lvm [SWAP]
└─rhel_ibm--x3650m4--05-home 253:2 0 212.2G 0 lvm /home
# ls /dev/mapper/* -l
crw-------. 1 root root 10, 236 Oct 25 02:07 /dev/mapper/control
lrwxrwxrwx. 1 root root 7 Oct 25 09:26 /dev/mapper/mpatha -> ../dm-3
lrwxrwxrwx. 1 root root 7 Oct 25 09:26 /dev/mapper/mpatha1 -> ../dm-5
lrwxrwxrwx. 1 root root 7 Oct 25 09:26 /dev/mapper/mpatha2 -> ../dm-6
lrwxrwxrwx. 1 root root 7 Oct 25 06:49 /dev/mapper/mpathb -> ../dm-4
lrwxrwxrwx. 1 root root 7 Oct 25 09:17 /dev/mapper/rhel_bootp--73--199--5-home -> ../dm-8
lrwxrwxrwx. 1 root root 7 Oct 25 09:17 /dev/mapper/rhel_bootp--73--199--5-root -> ../dm-9
lrwxrwxrwx. 1 root root 7 Oct 25 09:17 /dev/mapper/rhel_bootp--73--199--5-swap -> ../dm-7
lrwxrwxrwx. 1 root root 7 Oct 25 02:07 /dev/mapper/rhel_ibm--x3650m4--05-home -> ../dm-2
lrwxrwxrwx. 1 root root 7 Oct 25 02:07 /dev/mapper/rhel_ibm--x3650m4--05-root -> ../dm-0
lrwxrwxrwx. 1 root root 7 Oct 25 02:07 /dev/mapper/rhel_ibm--x3650m4--05-swap -> ../dm-1
-rw-r--r--. 1 root root 0 Oct 25 07:52 /dev/mapper/vg_raid10-lv_home
# dmsetup info -c -o name,blkdevs_used
Name BlkDevNamesUsed
rhel_bootp--73--199--5-home dm-6
mpathb sdd,sde
mpatha sdb,sdc
mpatha2 dm-3
rhel_bootp--73--199--5-swap dm-6
rhel_bootp--73--199--5-root dm-6
mpatha1 dm-3
rhel_ibm--x3650m4--05-home sda2
rhel_ibm--x3650m4--05-swap sda2
rhel_ibm--x3650m4--05-root sda2
:param blkdevs_used: block name, e.g. sda2
:return: a list contains all dm name for one blkdev
"""
dm_list = []
logging.info("Get dm name for '%s'" % blkdevs_used)
output = process.run("ls /dev/mapper/* -l", shell=True).stdout.decode()
for line in output.splitlines():
if blkdevs_used in line:
dm_name = line.split("/")[-1]
break
output = process.run("dmsetup info -c -o name,blkdevs_used",
shell=True).stdout.decode()
for line in output.splitlines():
if dm_name == line.split()[-1]:
dm_list.append(line.split()[0])
return dm_list
def delete_lvm_on_multipath(mpath_name="mpatha"):
"""
Delete lvm on the given multipath.
:param mpath_name: multi-path name.
:return: by default.
"""
output = process.run("pvscan", shell=True).stdout.decode()
pv_list = []
vg_list = []
lv_list = []
for line in output.splitlines():
if mpath_name in line:
if line.split()[1] not in pv_list:
pv_list.append(line.split()[1])
if line.split()[3] not in vg_list:
vg_list.append(line.split()[3])
output = process.run("lvscan", shell=True).stdout.decode()
for line in output.splitlines():
for vg in vg_list:
lv = "/dev/%s/" % vg
if lv in line and line.split("'")[1] not in lv_list:
lv_list.append(line.split("'")[1])
logging.info("pv list: %s" % pv_list)
logging.info("vg list: %s" % vg_list)
logging.info("lv list: %s" % lv_list)
for lv in lv_list:
logging.info("Remove lvm '%s'." % lv)
process.run("lvremove -y %s" % lv, ignore_status=True, shell=True)
for vg in vg_list:
logging.info("Remove vg '%s'." % vg)
process.run("vgremove -y %s" % vg, ignore_status=True, shell=True)
for pv in pv_list:
pv_name = pv.split("/")[-1]
for dm in get_lvm_dm_name(pv_name):
process.run("dmsetup remove %s" % dm,
ignore_status=True,
shell=True)
logging.info("Remove pv '%s'." % pv)
process.run("pvremove -y %s" % pv, ignore_status=True, shell=True)
def delete_partition_on_host(did):
"""
Delete partitions on the given disk.
:param did: disk ID. disk kname. e.g. 'sdb', 'nvme0n1'
:return: by default.
"""
# process.run("partprobe /dev/%s" % did, shell=True)
list_disk_cmd = "lsblk -o KNAME,MOUNTPOINT"
output = process.run(list_disk_cmd, shell=True).stdout.decode()
regex_str = did + "\w*(\d+)"
rm_cmd = 'parted -s "/dev/%s" rm %s'
for line in output.splitlines():
partition = re.findall(regex_str, line, re.I | re.M)
if partition:
if "/" in line.split()[-1]:
process.run("umount %s" % line.split()[-1], shell=True)
list_partition_number = "parted -s /dev/%s print|awk '/^ / {print $1}'"
partition_numbers = process.run(list_partition_number % did,
ignore_status=True,
shell=True).stdout.decode()
ignore_err_msg = "unrecognised disk label"
if ignore_err_msg in partition_numbers:
logging.info("no partition to delete on %s" % did)
else:
partition_numbers = partition_numbers.splitlines()
for number in partition_numbers:
logging.info("remove partition %s on %s" % (number, did))
process.run(rm_cmd % (did, number),
ignore_status=True,
shell=True)
process.run("partprobe /dev/%s" % did, shell=True)
def delete_multipath_partition_on_host(mpath_name="mpatha"):
"""
Delete partitions on the given multipath.
:param mpath_name: multi-path name.
:return: by default.
"""
# process.run("partprobe /dev/mapper/%s" % mpath_name, shell=True)
output = process.run("lsblk", shell=True).stdout.decode()
mpath_dict = {}
pattern = r'%s(\d+)' % mpath_name
rm_cmd = 'parted -s "/dev/mapper/%s" rm %s'
for line in output.splitlines():
for part_num in re.findall(pattern, line, re.I | re.M):
if part_num not in mpath_dict.keys():
mpath_dict[part_num] = line.split()[-1]
for key, value in mpath_dict.items():
if "/" in value:
process.run("umount %s" % value, shell=True)
logging.info("remove partition %s on %s" % (key, mpath_name))
process.run(rm_cmd % (mpath_name, key),
ignore_status=True,
shell=True)
output = process.run("dmsetup ls", shell=True).stdout.decode()
for line in output.splitlines():
for key in mpath_dict.keys():
if (mpath_name + key) in line:
process.run("dmsetup remove %s%s" % (mpath_name, key),
ignore_status=True,
shell=True)
process.run("partprobe /dev/mapper/%s" % mpath_name, shell=True)
def clean_partition_on_host(mpath_name="mpatha"):
"""
Delete partitions on multi-path disks.
:param mpath_name: multi-path name.
:return: by default
"""
delete_multipath_partition_on_host(mpath_name)
disks = get_multipath_disks(mpath_name)
for disk in disks:
delete_partition_on_host(disk)
def _run_fio_background(session, filename):
"""run fio testing by thread"""
logging.info("Start fio in background.")
cmd = fio_multipath.cfg.fio_path + params['fio_options'] % filename
logging.info(cmd)
session.cmdline(cmd)
# args = (params['fio_options'] % filename, 3600)
# fio_thread = utils_misc.InterruptedThread(fio_multipath.run, args)
# fio_thread.start()
# if not utils_misc.wait_for(lambda: fio_thread.is_alive, 60):
# test.fail("Failed to start fio thread.")
# return fio_thread
# def _run_fio_background(filename):
# """run fio testing by thread"""
#
# logging.info("Start fio in background.")
# args = (params['fio_options'] % filename, 3600)
# fio_thread = utils_misc.InterruptedThread(fio_multipath.run, args)
# fio_thread.start()
# if not utils_misc.wait_for(lambda: fio_thread.is_alive, 60):
# test.fail("Failed to start fio thread.")
# return fio_thread
def _get_windows_disks_index(session, image_size):
"""
Get all disks index which show in 'diskpart list disk'.
except for system disk.
in diskpart: if disk size < 8GB: it displays as MB
else: it displays as GB
:param session: session object to guest.
:param image_size: image size. e.g. 40M
:return: a list with all disks index except for system disk.
"""
disk = "disk_" + ''.join(
random.sample(string.ascii_letters + string.digits, 4))
disk_indexs = []
list_disk_cmd = "echo list disk > " + disk
list_disk_cmd += " && echo exit >> " + disk
list_disk_cmd += " && diskpart /s " + disk
list_disk_cmd += " && del /f " + disk
disks = session.cmd_output(list_disk_cmd)
size_type = image_size[-1] + "B"
if size_type == "MB":
disk_size = image_size[:-1] + " MB"
elif size_type == "GB" and int(image_size[:-1]) < 8:
disk_size = str(int(image_size[:-1]) * 1024) + " MB"
else:
disk_size = image_size[:-1] + " GB"
regex_str = 'Disk (\d+).*?%s' % disk_size
for disk in disks.splitlines():
if disk.startswith(" Disk"):
o = re.findall(regex_str, disk, re.I | re.M)
if o:
disk_indexs.append(o[0])
return disk_indexs
def resume_vm_plus(vm, timeout=120):
"""resume vm when it is paused.
:param vm: VM object.
:param timeout: re-try times, if it is still paused after resume
:return: True or None.
If resume successfully return True or return None
"""
logging.info("Try to resume vm within %s seconds." % timeout)
try:
vm_status = vm.resume(timeout=timeout)
except Exception as e:
logging.error("Failed to resume vm: %s" % six.text_type(e))
return vm_status
def resume_vm(vm, n_repeat=2):
"""resume vm when it is paused.
:param vm: VM object.
:param n_repeat: re-try times, if it is still paused after resume
:return: True or False.
If resume successfully return True or return False
"""
for i in range(1, n_repeat + 1):
logging.info("Try to resume vm %s time(s)" % i)
try:
vm.resume()
time.sleep(wait_time * 15)
except Exception as e:
logging.error("Failed to resume vm: %s" % six.text_type(e))
finally:
if check_vm_status(vm, "running"):
return True
if vm.is_paused() and i == 3:
return False
error_context.context("Get FC host name:", logging.info)
hostname = process.run("hostname", shell=True).stdout.decode().strip()
if hostname != params["special_host"]:
test.cancel("The special host is not '%s', cancel the test." %
params["special_host"])
error_context.context("Get FC disk serial name:", logging.info)
outputs = process.run("multipath -ll",
shell=True).stdout.decode().splitlines()
stg_serial_name = params["stg_serial_name"]
image_name_stg = params["image_name_stg"]
mpath_name = image_name_stg.split("/")[-1]
for output in outputs:
if stg_serial_name in output and mpath_name in output:
break
else:
test.cancel("The special disk is not '%s', cancel the test." %
stg_serial_name)
wait_time = float(params.get("sub_test_wait_time", 0))
repeat_times = int(params.get("repeat_times", 2))
multi_disks = get_multipath_disks(mpath_name)
error_context.context(
"Get all disks for '%s': %s" % (mpath_name, multi_disks), logging.info)
error_context.context(
"Verify all paths are running for %s before"
"start vm." % mpath_name, logging.info)
if compare_multipath_status("running", mpath_name):
logging.info("All paths are running for %s." % mpath_name)
else:
logging.info("Not all paths are running for %s, set "
"them to running." % mpath_name)
set_multipath_disks_status(multi_disks, "running")
error_context.context(
"Delete lvm on multipath disks on host "
"before testing.", logging.info)
delete_lvm_on_multipath(mpath_name)
error_context.context("Delete partitions on host before testing.",
logging.info)
clean_partition_on_host(mpath_name)
vm = env.get_vm(params["main_vm"])
try:
if params.get("need_install") == "yes":
error_context.context("Install guest on passthrough disk:",
logging.info)
args = (test, params, env)
bg = utils_misc.InterruptedThread(
utils_test.run_virt_sub_test, args,
{"sub_type": "unattended_install"})
bg.start()
utils_misc.wait_for(bg.is_alive, timeout=10)
time.sleep(random.uniform(60, 180))
else:
vm.create(params=params)
session = vm.wait_for_login(
timeout=int(params.get("timeout", 240)))
fio_multipath = generate_instance(params, vm, 'fio')
image_size_stg = params["image_size_stg"]
image_num_stg = int(params["image_num_stg"])
os_type = params["os_type"]
except Exception as e:
test.error("failed to create VM: %s" % six.text_type(e))
try:
error_context.context("Make sure guest is running before test",
logging.info)
if vm.is_paused():
vm.resume()
vm.verify_status("running")
if "fio_multipath" in locals().keys():
if os_type == "windows":
error_context.context(
"Get windows disk index that to "
"be formatted", logging.info)
disks = _get_windows_disks_index(session, image_size_stg)
if len(disks) != image_num_stg:
test.fail(
"Failed to list all disks by image size. The expected "
"is %s, actual is %s" % (image_num_stg, len(disks)))
error_context.context(
"Clear readonly for all disks and online "
"them in windows guest.", logging.info)
if not utils_disk.update_windows_disk_attributes(
session, disks):
test.fail("Failed to update windows disk attributes.")
else:
error_context.context("Get linux disk that to be "
"formatted", logging.info)
disks = sorted(utils_disk.get_linux_disks(session).keys())
if len(disks) != image_num_stg:
test.fail(
"Failed to list all disks by image size. The expected "
"is %s, actual is %s" % (image_num_stg, len(disks)))
file_system = [_.strip() for _ in params["file_system"].split()]
labeltype = params.get("labeltype", "gpt")
for n, disk in enumerate(disks):
error_context.context("Format disk in guest: '%s'" % disk,
logging.info)
# Random select one file system from file_system
index = random.randint(0, (len(file_system) - 1))
fstype = file_system[index].strip()
partitions = utils_disk.configure_empty_disk(
session,
disk,
image_size_stg,
os_type,
fstype=fstype,
labeltype=labeltype)
if not partitions:
test.fail("Fail to format disks.")
fio_file_name = params["fio_file_name"] % partitions[0]
# fio_thread = _run_fio_background(fio_file_name)
_run_fio_background(session, fio_file_name)
# disk = random.sample(multi_disks, 1)
for disk in multi_disks:
error_context.context(
"Disable disk %s during guest running" % disk, logging.info)
set_multipath_disks_status([disk], "offline")
time.sleep(wait_time * 15)
if vm.is_paused():
logging.info("vm is paused, will resume it.")
if not resume_vm(vm, repeat_times):
test.fail("Failed to resume guest after disable one disk")
logging.info("Has resumed vm already. Then verify it running.")
if not utils_misc.wait_for(
lambda: check_vm_status(vm, "running"), 60):
test.fail("Guest is not running after disable one disk")
error_context.context("Enable disk %s during guest running" % disk,
logging.info)
set_multipath_disks_status([disk], "running")
time.sleep(wait_time * 15)
error_context.context(
"Disable multipath '%s' during guest "
"running." % mpath_name, logging.info)
set_multipath_disks_status(multi_disks, "offline")
time.sleep(wait_time * 15)
error_context.context("Check if VM status is 'paused'", logging.info)
if not utils_misc.wait_for(lambda: check_vm_status(vm, "paused"), 120):
test.fail("Guest is not paused after all disks offline")
error_context.context(
"Re-connect fc storage, wait until the "
"storage is accessible again", logging.info)
set_multipath_disks_status(multi_disks, "running")
time.sleep(wait_time * 15)
error_context.context("vm is paused, resume it.", logging.info)
if not resume_vm(vm, repeat_times):
test.fail("Failed to resume guest after enable multipath.")
logging.info("Has resumed vm already. Then verify it running.")
time.sleep(wait_time * 15)
error_context.context("Check if VM status is 'running'", logging.info)
if not utils_misc.wait_for(lambda: check_vm_status(vm, "running"),
120):
test.fail("Guest is not running after all disks online")
if "bg" in locals().keys() and bg.is_alive and not vm.is_paused():
bg.join()
# wq comment ,why need wait fio
# if "fio_thread" in locals().keys() and fio_thread.is_alive \
# and not vm.is_paused():
# fio_thread.join()
error_context.context(
"Verify Host and guest kernel no error "
"and call trace", logging.info)
vm.verify_kernel_crash()
# except Exception as e:
# logging.error(e)
finally:
logging.info("Finally, clean environment.")
def _transfer_data(session, host_cmd, guest_cmd, timeout, sender):
"""
Send transfer data command and check result via output
:param session: guest session
:param host_cmd: host script command
:param guest_cmd: guest script command
:param timeout: timeout for data transfer
:param sender: who send data file
:return: True if pass, False and error message if check fail
"""
md5_host = "1"
md5_guest = "2"
def check_output(sender, output):
if sender == "both":
if "Md5MissMatch" in output:
err = "Data lost during file transfer. Md5 miss match."
err += " Script output:\n%s" % output
return False, err
return True, 0
else:
md5_re = "md5_sum = (\\w{32})"
try:
md5 = re.findall(md5_re, output)[0]
except Exception:
err = "Fail to get md5, script may fail."
err += " Script output:\n%s" % output
return False, err
return True, md5
try:
kwargs = {'cmd': host_cmd,
'shell': True,
'timeout': timeout}
error_context.context("Send host command: %s"
% host_cmd, logging.info)
host_thread = utils_misc.InterruptedThread(process.getoutput,
kwargs=kwargs)
host_thread.start()
time.sleep(3)
g_output = session.cmd_output(guest_cmd, timeout=timeout)
result = check_output(sender, g_output)
if result[0] is False:
return result
else:
md5_guest = result[1]
finally:
if host_thread:
output = host_thread.join(10)
result = check_output(sender, output)
if result[0] is False:
return result
else:
md5_host = result[1]
if sender != "both" and md5_host != md5_guest:
err = "Data lost during file transfer. Md5 miss match."
err += " Guest script output:\n %s" % g_output
err += " Host script output:\n%s" % output
return False, err
return True
def run(test, params, env):
"""
Test Step:
1. boot up three virtual machine
2. transfer file from guest1 to guest2, check md5
3. in guest 3 try to capture the packets(guest1 <-> guest2)
Params:
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
def _is_process_finished(session, process_name):
"""
Check whether the target process is finished running
param session: a guest session to send command
param process_name: the target process name
return: True if process does not exists,
False if still exists
"""
check_proc_cmd = check_proc_temp % process_name
status, output = session.cmd_status_output(check_proc_cmd)
if status:
return False
return process_name not in output
def data_mon(session, cmd, timeout):
try:
session.cmd(cmd, timeout)
except ShellCmdError as e:
if re.findall(catch_date % (addresses[1], addresses[0]), str(e)):
test.fail("God! Capture the transfet data:'%s'" % str(e))
logging.info("Guest3 catch data is '%s'", str(e))
timeout = int(params.get("login_timeout", '360'))
password = params.get("password")
username = params.get("username")
shell_port = params.get("shell_port")
tmp_dir = params.get("tmp_dir", "/tmp/")
clean_cmd = params.get("clean_cmd", "rm -f")
filesize = int(params.get("filesize", '100'))
wireshark_name = params.get("wireshark_name")
check_proc_temp = params.get("check_proc_temp")
tcpdump_cmd = params.get("tcpdump_cmd")
dd_cmd = params.get("dd_cmd")
catch_date = params.get("catch_data", "%s.* > %s.ssh")
md5_check = params.get("md5_check", "md5sum %s")
mon_process_timeout = int(params.get("mon_process_timeout", "1200"))
sessions = []
addresses = []
vms = []
error_context.context("Init boot the vms")
for vm_name in params.get("vms", "vm1 vm2 vm3").split():
vms.append(env.get_vm(vm_name))
for vm in vms:
vm.verify_alive()
sessions.append(vm.wait_for_login(timeout=timeout))
addresses.append(vm.get_address())
mon_session = vms[2].wait_for_login(timeout=timeout)
mon_macaddr = vms[2].get_mac_address()
src_file = (tmp_dir + "src-%s" % utils_misc.generate_random_string(8))
dst_file = (tmp_dir + "dst-%s" % utils_misc.generate_random_string(8))
try:
# Before transfer, run tcpdump to try to catche data
error_msg = "In guest3, try to capture the packets(guest1 <-> guest2)"
error_context.context(error_msg, logging.info)
if params.get("os_type") == "linux":
if_func = utils_net.get_linux_ifname
args = (mon_session, mon_macaddr)
else:
if_func = utils_net.get_windows_nic_attribute
args = (mon_session, "macaddress", mon_macaddr, "netconnectionid")
error_context.context("Install wireshark", logging.info)
install_wireshark_cmd = params.get("install_wireshark_cmd")
install_wireshark_cmd = utils_misc.set_winutils_letter(
sessions[2], install_wireshark_cmd)
status, output = sessions[2].cmd_status_output(
install_wireshark_cmd, timeout=timeout)
if status:
test.error(
"Failed to install wireshark, status=%s, output=%s" %
(status, output))
logging.info("Wait for wireshark installation to complete")
utils_misc.wait_for(
lambda: _is_process_finished(sessions[2], wireshark_name),
timeout, 20, 3)
logging.info("Wireshark is already installed")
interface_name = if_func(*args)
tcpdump_cmd = tcpdump_cmd % (addresses[1], addresses[0],
interface_name)
dthread = utils_misc.InterruptedThread(
data_mon, (sessions[2], tcpdump_cmd, mon_process_timeout))
logging.info("Tcpdump mon start ...")
logging.info("Creating %dMB file on guest1", filesize)
sessions[0].cmd(dd_cmd % (src_file, filesize), timeout=timeout)
dthread.start()
error_context.context("Transferring file guest1 -> guest2",
logging.info)
if params.get("os_type") == "windows":
cp_cmd = params["copy_cmd"]
cp_cmd = cp_cmd % (addresses[1], params['file_transfer_port'],
src_file, dst_file)
sessions[0].cmd_output(cp_cmd)
else:
remote.scp_between_remotes(addresses[0], addresses[1], shell_port,
password, password, username, username,
src_file, dst_file)
error_context.context("Check the src and dst file is same",
logging.info)
src_md5 = sessions[0].cmd_output(md5_check % src_file).split()[0]
dst_md5 = sessions[1].cmd_output(md5_check % dst_file).split()[0]
if dst_md5 != src_md5:
debug_msg = "Files md5sum mismatch!"
debug_msg += "source file md5 is '%s', after transfer md5 is '%s'"
test.fail(debug_msg % (src_md5, dst_md5), logging.info)
logging.info("Files md5sum match, file md5 is '%s'", src_md5)
error_context.context("Checking network private", logging.info)
tcpdump_check_cmd = params["tcpdump_check_cmd"]
tcpdump_kill_cmd = params["tcpdump_kill_cmd"]
tcpdump_check_cmd = re.sub("ADDR0", addresses[0], tcpdump_check_cmd)
tcpdump_check_cmd = re.sub("ADDR1", addresses[1], tcpdump_check_cmd)
status = mon_session.cmd_status(tcpdump_check_cmd)
if status:
test.error("Tcpdump process terminate exceptly")
mon_session.cmd(tcpdump_kill_cmd)
dthread.join()
finally:
sessions[0].cmd(" %s %s " % (clean_cmd, src_file))
sessions[1].cmd(" %s %s " % (clean_cmd, src_file))
if mon_session:
mon_session.close()
for session in sessions:
if session:
session.close()
def run(test, params, env):
"""
Qemu host nic bonding test:
1) Load bonding module with mode 802.3ad
2) Bring up bond interface
3) Add nics to bond interface
4) Add a new bridge and add bond interface to it
5) Get ip address for bridge
6) Boot up guest with the bridge
7) Checking guest netowrk via ping
8) Start file transfer between guest and host
9) Disable and enable physical interfaces during file transfer
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
bond_iface = params.get("bond_iface", "bond0")
bond_br_name = params.get("bond_br_name", "br_bond0")
timeout = int(params.get("login_timeout", 240))
remote_host = params.get("dsthost")
ping_timeout = int(params.get("ping_timeout", 240))
bonding_timeout = int(params.get("bonding_timeout", 1))
bonding_mode = params.get("bonding_mode", "1")
bonding_miimon = params.get("bonding_miimon", "100")
params['netdst'] = bond_br_name
host_bridges = utils_net.Bridge()
error_context.context("Load bonding module with mode 802.3ad",
logging.info)
if not process.system("lsmod|grep bonding", ignore_status=True,
shell=True):
process.system("modprobe -r bonding")
process.system(
"modprobe bonding mode=%s miimon=%s" %
(bonding_mode, bonding_miimon))
error_context.context("Bring up %s" % bond_iface, logging.info)
host_ifaces = utils_net.get_host_iface()
if bond_iface not in host_ifaces:
test.error("Can not find bond0 in host")
bond_iface = utils_net.Interface(bond_iface)
bond_iface.up()
bond_mac = bond_iface.get_mac()
host_ph_iface_pre = params.get("host_ph_iface_prefix", "en")
host_iface_bonding = int(params.get("host_iface_bonding", 2))
ph_ifaces = [_ for _ in host_ifaces if re.match(host_ph_iface_pre, _)]
host_ph_ifaces = [_ for _ in ph_ifaces if utils_net.Interface(_).is_up()]
ifaces_in_use = host_bridges.list_iface()
host_ph_ifaces_un = list(set(host_ph_ifaces) - set(ifaces_in_use))
if (len(host_ph_ifaces_un) < 2 or
len(host_ph_ifaces_un) < host_iface_bonding):
test.cancel("Host need %s nics at least." % host_iface_bonding)
error_context.context("Add nics to %s" % bond_iface.name, logging.info)
host_ifaces_bonding = host_ph_ifaces_un[:host_iface_bonding]
ifenslave_cmd = "ifenslave %s" % bond_iface.name
op_ifaces = []
for host_iface_bonding in host_ifaces_bonding:
op_ifaces.append(utils_net.Interface(host_iface_bonding))
ifenslave_cmd += " %s" % host_iface_bonding
process.system(ifenslave_cmd)
error_context.context("Add a new bridge and add %s to it."
% bond_iface.name, logging.info)
if bond_br_name not in host_bridges.list_br():
host_bridges.add_bridge(bond_br_name)
host_bridges.add_port(bond_br_name, bond_iface.name)
error_context.context("Get ip address for bridge", logging.info)
process.system("dhclient -r; dhclient %s" % bond_br_name, shell=True)
error_context.context("Boot up guest with bridge %s" % bond_br_name,
logging.info)
params["start_vm"] = "yes"
vm_name = params.get("main_vm")
env_process.preprocess_vm(test, params, env, vm_name)
vm = env.get_vm(vm_name)
session = vm.wait_for_login(timeout=timeout)
error_context.context("Checking guest netowrk via ping.", logging.info)
ping_cmd = params.get("ping_cmd")
ping_cmd = re.sub("REMOTE_HOST", remote_host, ping_cmd)
session.cmd(ping_cmd, timeout=ping_timeout)
error_context.context("Start file transfer", logging.info)
f_transfer = utils_misc.InterruptedThread(
utils_test.run_virt_sub_test,
args=(test, params, env,),
kwargs={"sub_type": "file_transfer"})
f_transfer.start()
utils_misc.wait_for(
lambda: process.system_output("pidof scp", ignore_status=True), 30)
error_context.context("Disable and enable physical "
"interfaces in %s" % bond_br_name, logging.info)
while True:
for op_iface in op_ifaces:
logging.debug("Turn down %s" % op_iface.name)
op_iface.down()
time.sleep(bonding_timeout)
logging.debug("Bring up %s" % op_iface.name)
op_iface.up()
time.sleep(bonding_timeout)
if not f_transfer.is_alive():
break
f_transfer.join()
def run(test, params, env):
"""
Test nic driver load/unload.
1) Boot a VM.
2) Get the NIC driver name.
3) Multi-session TCP transfer on test interface.
4) Repeatedly unload/load NIC driver during file transfer.
5) Check whether the test interface should still work.
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def reset_guest_udevrules(session, rules_file, rules_content):
"""
Write guest udev rules, then reboot the guest and
return the new session
"""
set_cmd = "echo '%s' > %s" % (rules_content, rules_file)
session.cmd_output_safe(set_cmd)
return vm.reboot()
def all_threads_done(threads):
"""
Check whether all threads have finished
"""
for thread in threads:
if thread.is_alive():
return False
else:
continue
return True
def all_threads_alive(threads):
"""
Check whether all threads is alive
"""
for thread in threads:
if not thread.is_alive():
return False
else:
continue
return True
timeout = int(params.get("login_timeout", 360))
transfer_timeout = int(params.get("transfer_timeout", 1000))
filesize = int(params.get("filesize", 512))
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session = vm.wait_for_login(timeout=timeout)
vm_mac_address = vm.get_mac_address()
udev_rules_file = "/etc/udev/rules.d/70-persistent-net.rules"
rules = params.get("rules")
if not session.cmd_status("[ -e %s ]" % udev_rules_file):
if not rules:
test.cancel("You must set udev rules before test")
rules = rules % vm_mac_address
session = reset_guest_udevrules(session, udev_rules_file, rules)
error_context.base_context("Test env prepare")
error_context.context("Get NIC interface name in guest.", logging.info)
ethname = utils_net.get_linux_ifname(session, vm.get_mac_address(0))
# get ethernet driver from '/sys' directory.
# ethtool can do the same thing and doesn't care about os type.
# if we make sure all guests have ethtool, we can make a change here.
sys_path = params.get("sys_path") % (ethname)
# readlink in RHEL4.8 doesn't have '-e' param, should use '-f' in RHEL4.8.
readlink_cmd = params.get("readlink_command", "readlink -e")
driver = os.path.basename(
session.cmd("%s %s" % (readlink_cmd, sys_path)).strip())
logging.info("The guest interface %s using driver %s", ethname, driver)
error_context.context(
"Host test file prepare, create %dMB file on host" % filesize,
logging.info)
tmp_dir = data_dir.get_tmp_dir()
host_path = os.path.join(
tmp_dir, "host_file_%s" % utils_misc.generate_random_string(8))
guest_path = os.path.join(
"/home", "guest_file_%s" % utils_misc.generate_random_string(8))
cmd = "dd if=/dev/zero of=%s bs=1M count=%d" % (host_path, filesize)
process.run(cmd)
file_checksum = crypto.hash_file(host_path, algorithm="md5")
error_context.context(
"Guest test file prepare, Copy file %s from host to "
"guest" % host_path, logging.info)
vm.copy_files_to(host_path, guest_path, timeout=transfer_timeout)
if session.cmd_status("md5sum %s | grep %s" % (guest_path, file_checksum)):
test.cancel("File MD5SUMs changed after copy to guest")
logging.info("Test env prepare successfully")
error_context.base_context("Nic driver load/unload testing", logging.info)
session_serial = vm.wait_for_serial_login(timeout=timeout)
try:
error_context.context("Transfer file between host and guest",
logging.info)
threads = []
file_paths = []
host_file_paths = []
for sess_index in range(int(params.get("sessions_num", "10"))):
sess_path = os.path.join("/home", "dst-%s" % sess_index)
host_sess_path = os.path.join(tmp_dir, "dst-%s" % sess_index)
thread1 = utils_misc.InterruptedThread(
vm.copy_files_to, (host_path, sess_path),
{"timeout": transfer_timeout})
thread2 = utils_misc.InterruptedThread(
vm.copy_files_from, (guest_path, host_sess_path),
{"timeout": transfer_timeout})
thread1.start()
threads.append(thread1)
thread2.start()
threads.append(thread2)
file_paths.append(sess_path)
host_file_paths.append(host_sess_path)
utils_misc.wait_for(lambda: all_threads_alive(threads), 60, 10, 1)
time.sleep(5)
error_context.context(
"Repeatedly unload/load NIC driver during file "
"transfer", logging.info)
while not all_threads_done(threads):
error_context.context("Shutdown the driver for NIC interface.",
logging.info)
session_serial.cmd_output_safe("ifconfig %s down" % ethname)
error_context.context("Unload NIC driver.", logging.info)
session_serial.cmd_output_safe("modprobe -r %s" % driver)
error_context.context("Load NIC driver.", logging.info)
session_serial.cmd_output_safe("modprobe %s" % driver)
error_context.context("Activate NIC driver.", logging.info)
session_serial.cmd_output_safe("ifconfig %s up" % ethname)
session_serial.cmd_output_safe("sleep %s" % random.randint(10, 60))
# files md5sums check
error_context.context("File transfer finished, checking files md5sums",
logging.info)
err_info = []
for copied_file in file_paths:
if session_serial.cmd_status("md5sum %s | grep %s" %
(copied_file, file_checksum)):
err_msg = "Guest file %s md5sum changed"
err_info.append(err_msg % copied_file)
for copied_file in host_file_paths:
if process.system("md5sum %s | grep %s" %
(copied_file, file_checksum),
shell=True):
err_msg = "Host file %s md5sum changed"
err_info.append(err_msg % copied_file)
if err_info:
test.error("files MD5SUMs changed after copying %s" % err_info)
except Exception:
for thread in threads:
thread.join(suppress_exception=True)
raise
else:
for thread in threads:
thread.join()
for copied_file in file_paths:
session_serial.cmd("rm -rf %s" % copied_file)
for copied_file in host_file_paths:
process.system("rm -rf %s" % copied_file)
session_serial.cmd("%s %s" % ("rm -rf", guest_path))
os.remove(host_path)
session.close()
session_serial.close()
def run(test, params, env):
"""
KVM guest stop test:
1) Log into a guest
2) Copy a file into guest
3) Stop guest
4) Check the status through monitor
5) Check the session
6) Migrat the vm to a file twice and compare them.
:param test: kvm test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
timeout = float(params.get("login_timeout", 240))
session = vm.wait_for_login(timeout=timeout)
save_path = params.get("save_path", "/tmp")
clean_save = params.get("clean_save") == "yes"
save1 = os.path.join(save_path, "save1")
save2 = os.path.join(save_path, "save2")
guest_path = params.get("guest_path", "/tmp")
file_size = params.get("file_size", "1000")
try:
process.run("dd if=/dev/zero of=/tmp/file bs=1M count=%s" % file_size)
# Transfer file from host to guest, we didn't expect the finish of
# transfer, we just let it to be a kind of stress in guest.
bg = utils_misc.InterruptedThread(vm.copy_files_to,
("/tmp/file", guest_path),
dict(verbose=True, timeout=60))
test.log.info("Start the background transfer")
bg.start()
try:
# wait for the transfer start
time.sleep(5)
test.log.info("Stop the VM")
vm.pause()
# check with monitor
test.log.info("Check the status through monitor")
if not vm.monitor.verify_status("paused"):
status = str(vm.monitor.info("status"))
test.fail("Guest did not pause after sending stop,"
" guest status is %s" % status)
# check through session
test.log.info("Check the session")
if session.is_responsive():
test.fail("Session still alive after sending stop")
# Check with the migration file
test.log.info("Save and check the state files")
for p in [save1, save2]:
vm.save_to_file(p)
time.sleep(1)
if not os.path.isfile(p):
test.fail("VM failed to save state file %s" % p)
# Fail if we see deltas
md5_save1 = crypto.hash_file(save1)
md5_save2 = crypto.hash_file(save2)
if md5_save1 != md5_save2:
test.fail("The produced state files differ")
finally:
bg.join(suppress_exception=True)
finally:
session.close()
if clean_save:
test.log.debug("Clean the state files")
if os.path.isfile(save1):
os.remove(save1)
if os.path.isfile(save2):
os.remove(save2)
vm.resume()
def run(test, params, env):
"""
Enable MULTI_QUEUE feature in guest
1) Boot up VM(s)
2) Login guests one by one
3) Enable MQ for all virtio nics by ethtool -L
4) Run netperf on guest
5) check vhost threads on host, if vhost is enable
6) check cpu affinity if smp == queues
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def get_virtio_queues_irq(session):
"""
Return multi queues input irq list
"""
guest_irq_info = session.cmd_output("cat /proc/interrupts")
virtio_queues_irq = re.findall(r"(\d+):.*virtio\d+-input.\d", guest_irq_info)
if not virtio_queues_irq:
test.error('Could not find any "virtio-input" interrupts')
return virtio_queues_irq
def get_cpu_affinity_hint(session, irq_number):
"""
Return the cpu affinity_hint of irq_number
"""
cmd_get_cpu_affinity = r"cat /proc/irq/%s/affinity_hint" % irq_number
return session.cmd_output(cmd_get_cpu_affinity).strip()
def get_cpu_index(cpu_id):
"""
Transfer cpu_id to cpu index
"""
cpu_used_index = []
for cpu_index in range(int(vm.cpuinfo.smp)):
if int(cpu_id) & (1 << cpu_index) != 0:
cpu_used_index.append(cpu_index)
return cpu_used_index
def set_cpu_affinity(session):
"""
Set cpu affinity
"""
cmd_set_cpu_affinity = r"echo $(cat /proc/irq/%s/affinity_hint)"
cmd_set_cpu_affinity += " > /proc/irq/%s/smp_affinity"
irq_list = get_virtio_queues_irq(session)
for irq in irq_list:
session.cmd(cmd_set_cpu_affinity % (irq, irq))
def get_cpu_irq_statistics(session, irq_number, cpu_id=None):
"""
Get guest interrupts statistics
"""
online_cpu_number_cmd = r"cat /proc/interrupts | head -n 1 | wc -w"
cmd = r"cat /proc/interrupts | sed -n '/^\s*%s:/p'" % irq_number
online_cpu_number = int(session.cmd_output_safe(online_cpu_number_cmd))
irq_statics = session.cmd_output(cmd)
irq_statics_list = list(map(int, irq_statics.split()[1:online_cpu_number]))
if irq_statics_list:
if cpu_id and cpu_id < len(irq_statics_list):
return irq_statics_list[cpu_id]
if not cpu_id:
return irq_statics_list
return []
login_timeout = int(params.get("login_timeout", 360))
bg_stress_run_flag = params.get("bg_stress_run_flag")
stress_thread = None
queues = int(params.get("queues", 1))
vms = params.get("vms").split()
if queues == 1:
test.log.info("No need to enable MQ feature for single queue")
return
for vm in vms:
vm = env.get_vm(vm)
vm.verify_alive()
session = vm.wait_for_login(timeout=login_timeout)
for i, nic in enumerate(vm.virtnet):
if "virtio" in nic['nic_model']:
ifname = utils_net.get_linux_ifname(session,
vm.get_mac_address(i))
session.cmd_output("ethtool -L %s combined %d" % (ifname,
queues))
o = session.cmd_output("ethtool -l %s" % ifname)
if len(re.findall(r"Combined:\s+%d\s" % queues, o)) != 2:
test.error("Fail to enable MQ feature of (%s)" %
nic.nic_name)
test.log.info("MQ feature of (%s) is enabled", nic.nic_name)
taskset_cpu = params.get("netperf_taskset_cpu")
if taskset_cpu:
taskset_cmd = "taskset -c %s " % " ".join(taskset_cpu)
params["netperf_cmd_prefix"] = taskset_cmd
check_cpu_affinity = params.get("check_cpu_affinity", 'no')
check_vhost = params.get("check_vhost_threads", 'yes')
if check_cpu_affinity == 'yes' and (vm.cpuinfo.smp == queues):
process.system("systemctl stop irqbalance.service")
session.cmd("systemctl stop irqbalance.service")
set_cpu_affinity(session)
bg_sub_test = params.get("bg_sub_test")
n_instance = int(params.get("netperf_para_sessions", queues))
try:
if bg_sub_test:
error_context.context("Run test %s background" % bg_sub_test,
test.log.info)
# Set flag, when the sub test really running, will change this
# flag to True
stress_thread = utils_misc.InterruptedThread(
utils_test.run_virt_sub_test, (test, params, env),
{"sub_type": bg_sub_test})
stress_thread.start()
if params.get("vhost") == 'vhost=on' and check_vhost == 'yes':
vhost_thread_pattern = params.get("vhost_thread_pattern",
r"\w+\s+(\d+)\s.*\[vhost-%s\]")
vhost_threads = vm.get_vhost_threads(vhost_thread_pattern)
time.sleep(120)
error_context.context("Check vhost threads on host",
test.log.info)
top_cmd = (r'top -n 1 -bis | tail -n +7 | grep -E "^ *%s "'
% ' |^ *'.join(map(str, vhost_threads)))
top_info = None
while session.cmd_status("ps -C netperf") == 0:
top_info = process.system_output(top_cmd, ignore_status=True,
shell=True).decode()
if top_info:
break
test.log.info(top_info)
vhost_re = re.compile(r"(0:00.\d{2}).*vhost-\d+[\d|+]")
invalid_vhost_thread = len(vhost_re.findall(top_info, re.I))
running_threads = (len(top_info.splitlines()) -
int(invalid_vhost_thread))
n_instance = min(n_instance, int(queues), int(vm.cpuinfo.smp))
if running_threads != n_instance:
err_msg = "Run %s netperf session, but %s queues works"
test.fail(err_msg % (n_instance, running_threads))
# check cpu affinity
if check_cpu_affinity == 'yes' and (vm.cpuinfo.smp == queues):
error_context.context("Check cpu affinity", test.log.info)
vectors = params.get("vectors", None)
enable_msix_vectors = params.get("enable_msix_vectors")
expect_vectors = 2 * int(queues) + 2
if (not vectors) and (enable_msix_vectors == "yes"):
vectors = expect_vectors
if vectors and (vectors >= expect_vectors) and taskset_cpu:
cpu_irq_affinity = {}
for irq in get_virtio_queues_irq(session):
cpu_id = get_cpu_affinity_hint(session, irq)
cpu_index = get_cpu_index(cpu_id)
if cpu_index:
for cpu in cpu_index:
cpu_irq_affinity["%s" % cpu] = irq
else:
test.error("Can not get the cpu")
irq_number = cpu_irq_affinity[taskset_cpu]
irq_ori = get_cpu_irq_statistics(session, irq_number)
test.log.info("Cpu irq info: %s", irq_ori)
time.sleep(10)
irq_cur = get_cpu_irq_statistics(session, irq_number)
test.log.info("After 10s, cpu irq info: %s", irq_cur)
irq_change_list = [x[0] - x[1] for x in zip(irq_cur, irq_ori)]
cpu_affinity = irq_change_list.index(max(irq_change_list))
if cpu_affinity != int(taskset_cpu):
err_msg = "Error, taskset on cpu %s, "
err_msg += "but queues use cpu %s"
test.fail(err_msg % (taskset_cpu, cpu_affinity))
if bg_sub_test and stress_thread:
env[bg_stress_run_flag] = False
try:
stress_thread.join()
except Exception as e:
err_msg = "Run %s test background error!\n "
err_msg += "Error Info: '%s'"
test.error(err_msg % (bg_sub_test, e))
finally:
if session:
session.close()
if check_cpu_affinity == 'yes' and (vm.cpuinfo.smp == queues):
process.system("systemctl start irqbalance.service")
def run(test, params, env):
"""
Test Steps:
1. boot up guest with sndbuf=1048576 or other value.
2. Transfer file between host and guest.
3. Run netperf between host and guest.
4. During netperf testing, from an external host ping the host whitch
booting the guest.
Params:
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
dst_ses = None
try:
error_context.context("Transfer file between host and guest",
test.log.info)
utils_test.run_file_transfer(test, params, env)
dsthost = params.get("dsthost")
login_timeout = int(params.get("login_timeout", 360))
if dsthost:
params_host = params.object_params("dsthost")
dst_ses = remote.wait_for_login(params_host.get("shell_client"),
dsthost,
params_host.get("shell_port"),
params_host.get("username"),
params_host.get("password"),
params_host.get("shell_prompt"),
timeout=login_timeout)
else:
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
dst_ses = vm.wait_for_login(timeout=login_timeout)
dsthost = vm.get_address()
bg_stress_test = params.get("background_stress_test", 'netperf_stress')
error_context.context(
("Run subtest %s between host and guest." % bg_stress_test),
test.log.info)
wait_time = float(params.get("wait_bg_time", 60))
bg_stress_run_flag = params.get("bg_stress_run_flag")
env[bg_stress_run_flag] = False
stress_thread = utils_misc.InterruptedThread(
utils_test.run_virt_sub_test, (test, params, env),
{"sub_type": bg_stress_test})
stress_thread.start()
if not utils_misc.wait_for(lambda: env.get(bg_stress_run_flag),
wait_time, 0, 1,
"Wait %s test start" % bg_stress_test):
err = "Fail to start netperf test between guest and host"
test.error(err)
ping_timeout = int(params.get("ping_timeout", 60))
host_ip = utils_net.get_host_ip_address(params)
txt = "Ping %s from %s during netperf testing" % (host_ip, dsthost)
error_context.context(txt, test.log.info)
status, output = utils_test.ping(host_ip,
session=dst_ses,
timeout=ping_timeout)
if status != 0:
test.fail("Ping returns non-zero value %s" % output)
package_lost = utils_test.get_loss_ratio(output)
package_lost_ratio = float(params.get("package_lost_ratio", 5))
txt = "%s%% packeage lost when ping %s from %s." % (package_lost,
host_ip, dsthost)
if package_lost > package_lost_ratio:
test.fail(txt)
test.log.info(txt)
finally:
try:
stress_thread.join(60)
except Exception:
pass
if dst_ses:
dst_ses.close()
def run(test, params, env):
"""
MULTI_QUEUE chang queues number test
1) Boot up VM, and login guest
2) Check guest pci msi support and reset it as expection
3) Enable the queues in guest
4) Run bg_stress_test(pktgen, netperf or file copy) if needed
5) Change queues number repeatly during stress test running
6) Ping external host (local host, if external host not available)
:param test: QEMU test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
def change_queues_number(session, ifname, q_number, queues_status=None):
"""
Change queues number
"""
mq_set_cmd = "ethtool -L %s combined %d" % (ifname, q_number)
if not queues_status:
queues_status = get_queues_status(session, ifname)
if (q_number != queues_status[1] and q_number <= queues_status[0] and
q_number > 0):
expect_status = 0
else:
expect_status = 1
status, output = session.cmd_status_output(mq_set_cmd, safe=True)
cur_queues_status = get_queues_status(session, ifname)
if status != expect_status:
err_msg = "Change queues number failed, "
err_msg += "current queues set is %s, " % queues_status[1]
err_msg += "max allow queues set is %s, " % queues_status[0]
err_msg += "when run cmd: '%s', " % mq_set_cmd
err_msg += "expect exit status is: %s, " % expect_status
err_msg += "output: '%s'" % output
test.fail(err_msg)
if not status and cur_queues_status == queues_status:
test.fail("params is right, but change queues failed")
elif status and cur_queues_status != queues_status:
test.fail("No need change queues number")
return [int(_) for _ in cur_queues_status]
def get_queues_status(session, ifname, timeout=240):
"""
Get queues status
"""
mq_get_cmd = "ethtool -l %s" % ifname
nic_mq_info = session.cmd_output(mq_get_cmd, timeout=timeout,
safe=True)
queues_reg = re.compile(r"Combined:\s+(\d)", re.I)
queues_info = queues_reg.findall(" ".join(nic_mq_info.splitlines()))
if len(queues_info) != 2:
err_msg = "Oops, get guest queues info failed, "
err_msg += "make sure your guest support MQ.\n"
err_msg += "Check cmd is: '%s', " % mq_get_cmd
err_msg += "Command output is: '%s'." % nic_mq_info
test.cancel(err_msg)
return [int(x) for x in queues_info]
def enable_multi_queues(vm):
sess = vm.wait_for_serial_login(timeout=login_timeout)
error_context.context("Enable multi queues in guest.", logging.info)
for nic_index, nic in enumerate(vm.virtnet):
ifname = utils_net.get_linux_ifname(sess, nic.mac)
queues = int(nic.queues)
change_queues_number(sess, ifname, queues)
def ping_test(dest_ip, ping_time, lost_raito, session=None):
status, output = utils_test.ping(dest=dest_ip, timeout=ping_time,
session=session)
packets_lost = utils_test.get_loss_ratio(output)
if packets_lost > lost_raito:
err = " %s%% packages lost during ping. " % packets_lost
err += "Ping command log:\n %s" % "\n".join(output.splitlines()[-3:])
test.fail(err)
error_context.context("Init guest and try to login", logging.info)
login_timeout = int(params.get("login_timeout", 360))
bg_stress_test = params.get("run_bgstress")
bg_stress_run_flag = params.get("bg_stress_run_flag")
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
vm.wait_for_login(timeout=login_timeout)
if params.get("pci_nomsi", "no") == "yes":
error_context.context("Disable pci msi in guest", logging.info)
utils_test.update_boot_option(vm, args_added="pci=nomsi")
vm.wait_for_login(timeout=login_timeout)
enable_multi_queues(vm)
session_serial = vm.wait_for_serial_login(timeout=login_timeout)
s_session = None
bg_ping = params.get("bg_ping")
b_ping_lost_ratio = int(params.get("background_ping_package_lost_ratio", 5))
f_ping_lost_ratio = int(params.get("final_ping_package_lost_ratio", 5))
guest_ip = vm.get_address()
b_ping_time = int(params.get("background_ping_time", 60))
f_ping_time = int(params.get("final_ping_time", 60))
bg_test = None
try:
ifnames = []
for nic_index, nic in enumerate(vm.virtnet):
ifname = utils_net.get_linux_ifname(session_serial,
vm.virtnet[nic_index].mac)
ifnames.append(ifname)
if bg_stress_test:
error_context.context("Run test %s background" % bg_stress_test,
logging.info)
stress_thread = ""
wait_time = float(params.get("wait_bg_time", 60))
env[bg_stress_run_flag] = False
stress_thread = utils_misc.InterruptedThread(
utils_test.run_virt_sub_test, (test, params, env),
{"sub_type": bg_stress_test})
stress_thread.start()
if bg_stress_run_flag:
utils_misc.wait_for(lambda: env.get(bg_stress_run_flag),
wait_time, 0, 5,
"Wait %s start background" % bg_stress_test)
if bg_ping == "yes":
error_context.context("Ping guest from host", logging.info)
args = (guest_ip, b_ping_time, b_ping_lost_ratio)
bg_test = utils_misc.InterruptedThread(ping_test, args)
bg_test.start()
error_context.context("Change queues number repeatly", logging.info)
repeat_counts = int(params.get("repeat_counts", 10))
for nic_index, nic in enumerate(vm.virtnet):
if "virtio" not in nic['nic_model']:
continue
queues = int(vm.virtnet[nic_index].queues)
if queues == 1:
logging.info("Nic with single queue, skip and continue")
continue
ifname = ifnames[nic_index]
default_change_list = range(1, int(queues + 1))
change_list = params.get("change_list")
if change_list:
change_list = change_list.split(",")
else:
change_list = default_change_list
for repeat_num in range(1, repeat_counts + 1):
error_context.context("Change queues number -- %sth"
% repeat_num, logging.info)
try:
queues_status = get_queues_status(session_serial, ifname)
for q_number in change_list:
queues_status = change_queues_number(session_serial,
ifname,
int(q_number),
queues_status)
except aexpect.ShellProcessTerminatedError:
vm = env.get_vm(params["main_vm"])
session = vm.wait_for_serial_login(timeout=login_timeout)
session_serial = session
queues_status = get_queues_status(session_serial, ifname)
for q_number in change_list:
queues_status = change_queues_number(session_serial,
ifname,
int(q_number),
queues_status)
if params.get("ping_after_changing_queues", "yes") == "yes":
default_host = "www.redhat.com"
ext_host = utils_net.get_host_default_gateway()
if not ext_host:
# Fallback to a hardcode host, eg:
logging.warn("Can't get specified host,"
" Fallback to default host '%s'", default_host)
ext_host = default_host
s_session = vm.wait_for_login(timeout=login_timeout)
txt = "ping %s after changing queues in guest."
error_context.context(txt, logging.info)
ping_test(ext_host, f_ping_time, f_ping_lost_ratio, s_session)
if bg_stress_test:
env[bg_stress_run_flag] = False
if stress_thread:
error_context.context("wait for background test finish",
logging.info)
try:
stress_thread.join()
except Exception as err:
err_msg = "Run %s test background error!\n "
err_msg += "Error Info: '%s'"
test.error(err_msg % (bg_stress_test, err))
finally:
if bg_stress_test:
env[bg_stress_run_flag] = False
if session_serial:
session_serial.close()
if s_session:
s_session.close()
if bg_test:
error_context.context("Wait for background ping test finish.",
logging.info)
try:
bg_test.join()
except Exception as err:
txt = "Fail to wait background ping test finish. "
txt += "Got error message %s" % err
test.fail(txt)
def run(test, params, env):
"""
Qemu guest pxe boot test:
1). check npt/ept function enable, then boot vm
2). execute query/info cpus in loop
3). verify vm not paused during pxe booting
params:
:param test: QEMU test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
restore_mmu_cmd = None
pxe_timeout = int(params.get("pxe_timeout", 60))
error_context.context("Enable ept/npt", logging.info)
try:
flag = list(filter(lambda x: x in cpu.get_cpu_flags(),
['ept', 'npt']))[0]
except IndexError:
logging.info("Host doesn't support ept/npt, skip the configuration")
else:
enable_mmu_cmd = params["enable_mmu_cmd_%s" % flag]
check_mmu_cmd = params["check_mmu_cmd_%s" % flag]
restore_mmu_cmd = params["restore_mmu_cmd_%s" % flag]
status = process.system(check_mmu_cmd,
timeout=120,
ignore_status=True,
shell=True)
if status != 0:
_kill_vms(params, env)
process.run(enable_mmu_cmd, shell=True)
params["start_vm"] = "yes"
params["kvm_vm"] = "yes"
params["paused_after_start_vm"] = "yes"
error_context.context("Try to boot from NIC", logging.info)
env_process.preprocess_vm(test, params, env, params["main_vm"])
vm = env.get_vm(params["main_vm"])
bg = utils_misc.InterruptedThread(_capture_tftp, (test, vm, pxe_timeout))
count = 0
try:
bg.start()
error_context.context("Query cpus in loop", logging.info)
vm.resume()
while True:
count += 1
try:
vm.monitor.info("cpus", debug=False)
if params.get('machine_type').startswith("s390"):
if vm.monitor.get_status()['status'] in [
'running', 'guest-panicked'
]:
pass
else:
raise virt_vm.VMStatusError(
'Unexpected VM status: "%s"' %
vm.monitor.get_status())
else:
vm.verify_status("running")
if not bg.is_alive():
break
except qemu_monitor.MonitorSocketError:
test.fail("Qemu looks abnormally, please read the log")
logging.info("Execute info/query cpus %d times", count)
finally:
bg.join()
if restore_mmu_cmd:
_kill_vms(params, env)
process.run(restore_mmu_cmd, shell=True)
def run(test, params, env):
"""
Nic bonding test in guest.
1) Start guest with four nic devices.
2) Setup bond0 in guest.
3) Execute file transfer test between guest and host.
4) Repeatedly put down/up interfaces by 'ip link'
5) Execute file transfer test between guest and host.
6) Check md5 value after transfered.
:param test: Kvm test object.
:param params: Dictionary with the test parameters.
:param env: Dictionary with test environment.
"""
tmp_dir = params["tmp_dir"]
filesize = params.get_numeric("filesize")
dd_cmd = params["dd_cmd"]
login_timeout = params.get_numeric("login_timeout", 1200)
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session_serial = vm.wait_for_serial_login(timeout=login_timeout)
ifnames = utils_net.get_linux_ifname(session_serial)
ssh_login_cmd = ("echo LoginGraceTime 5m >> /etc/ssh/sshd_config &&"
" systemctl restart sshd.service || service sshd restart")
session_serial.cmd_output_safe(ssh_login_cmd)
# get params of bonding
nm_stop_cmd = "service NetworkManager stop; true"
session_serial.cmd_output_safe(nm_stop_cmd)
modprobe_cmd = "modprobe bonding"
bonding_params = params.get("bonding_params")
if bonding_params:
modprobe_cmd += " %s" % bonding_params
session_serial.cmd_output_safe(modprobe_cmd)
session_serial.cmd_output_safe("ifconfig bond0 up")
setup_cmd = "ifenslave bond0 " + " ".join(ifnames)
session_serial.cmd_output_safe(setup_cmd)
# do a pgrep to check if dhclient has already been running
pgrep_cmd = "pgrep dhclient"
try:
session_serial.cmd_output_safe(pgrep_cmd)
# if dhclient is there, killl it
except aexpect.ShellCmdError:
logging.info("it's safe to run dhclient now")
else:
logging.info("dhclient is already running, kill it")
session_serial.cmd_output_safe("killall -9 dhclient")
time.sleep(1)
session_serial.cmd_output_safe("dhclient bond0")
# prepare test data
guest_path = os.path.join(tmp_dir +
"dst-%s" % utils_misc.generate_random_string(8))
host_path = os.path.join(test.tmpdir,
"tmp-%s" % utils_misc.generate_random_string(8))
logging.info("Test setup: Creating %dMB file on host", filesize)
process.run(dd_cmd % host_path, shell=True)
# get_bonding_nic_mac and ip
try:
link_set_cmd = "ip link set dev %s %s"
# transfer data
original_md5 = crypto.hash_file(host_path, algorithm="md5")
logging.info("md5 value of data original: %s", original_md5)
logging.info("Failover test with file transfer")
transfer_thread = utils_misc.InterruptedThread(vm.copy_files_to,
(host_path, guest_path))
transfer_thread.start()
try:
while transfer_thread.is_alive():
for ifname in ifnames:
session_serial.cmd(link_set_cmd % (ifname, "down"))
time.sleep(random.randint(1, 30))
session_serial.cmd(link_set_cmd % (ifname, "up"))
time.sleep(random.randint(1, 30))
except aexpect.ShellProcessTerminatedError:
transfer_thread.join(suppress_exception=True)
raise
else:
transfer_thread.join()
logging.info('Cleaning temp file on host')
os.remove(host_path)
logging.info("Failover test 2 with file transfer")
transfer_thread = utils_misc.InterruptedThread(vm.copy_files_from,
(guest_path, host_path))
transfer_thread.start()
try:
nic_num = len(ifnames)
up_index = 0
while transfer_thread.is_alive():
nic_indexes = list(range(nic_num))
up_index = up_index % nic_num
session_serial.cmd(link_set_cmd % (ifnames[up_index], "up"))
nic_indexes.remove(up_index)
for num in nic_indexes:
session_serial.cmd(link_set_cmd % (ifnames[num], "down"))
time.sleep(random.randint(3, 5))
up_index += 1
except aexpect.ShellProcessTerminatedError:
transfer_thread.join(suppress_exception=True)
raise
else:
transfer_thread.join()
current_md5 = crypto.hash_file(host_path, algorithm="md5")
logging.info("md5 value of data current: %s", current_md5)
if original_md5 != current_md5:
test.fail("File changed after transfer host -> guest "
"and guest -> host")
finally:
session_serial.sendline("ifenslave -d bond0 " + " ".join(ifnames))
session_serial.sendline("kill -9 `pgrep dhclient`")
session_serial.sendline("sed -i '$ d' /etc/ssh/sshd_config")