class TestVhostUserLiveMigration(TestCase):
def set_up_all(self):
# verify at least two duts
self.verify(len(self.duts) >= 2, "Insufficient duts for live migration!!!")
# each dut required one ports
self.dut_ports = self.dut.get_ports()
# Verify that enough ports are available
self.verify(len(self.dut_ports) >= 1, "Insufficient ports for testing")
self.dut_port = self.dut_ports[0]
dut_ip = self.dut.crb['My IP']
self.host_tport = self.tester.get_local_port_bydut(self.dut_port, dut_ip)
self.host_tintf = self.tester.get_interface(self.host_tport)
self.backup_ports = self.duts[1].get_ports()
# Verify that enough ports are available
self.verify(len(self.backup_ports) >= 1, "Insufficient ports for testing")
self.backup_port = self.backup_ports[0]
# backup host ip will be used in migrate command
self.backup_dutip = self.duts[1].crb['My IP']
self.backup_tport = self.tester.get_local_port_bydut(self.backup_port, self.backup_dutip)
self.backup_tintf = self.tester.get_interface(self.backup_tport)
# Use testpmd as vhost-user application on host/backup server
self.vhost = "./%s/app/testpmd" % self.target
self.vm_testpmd = "./%s/app/testpmd -c 0x3 -n 4 -- -i" % self.target
self.virio_mac = "52:54:00:00:00:01"
# flag for environment
self.env_done = False
def set_up(self):
self.setup_vm_env()
pass
def bind_nic_driver(self, crb, ports, driver=""):
# modprobe vfio driver
if driver == "vfio-pci":
for port in ports:
netdev = crb.ports_info[port]['port']
driver = netdev.get_nic_driver()
if driver != 'vfio-pci':
netdev.bind_driver(driver='vfio-pci')
elif driver == "igb_uio":
# igb_uio should insmod as default, no need to check
for port in ports:
netdev = crb.ports_info[port]['port']
driver = netdev.get_nic_driver()
if driver != 'igb_uio':
netdev.bind_driver(driver='igb_uio')
else:
for port in ports:
netdev = crb.ports_info[port]['port']
driver_now = netdev.get_nic_driver()
if driver == "":
driver = netdev.default_driver
if driver != driver_now:
netdev.bind_driver(driver=driver)
def setup_vm_env(self, driver='default'):
"""
Create testing environment on Host and Backup
"""
if self.env_done:
return
# start vhost application on host and backup machines
self.logger.info("Start vhost on host and backup host")
for crb in self.duts[:2]:
self.bind_nic_driver(crb, [crb.get_ports()[0]], driver="igb_uio")
# start vhost app: testpmd, predict hugepage on both sockets
base_dir = crb.base_dir.replace('~', '/root')
crb.send_expect("rm -f %s/vhost-net" % base_dir, "# ")
crb.send_expect("%s -c f -n 4 --socket-mem 512,512 --vdev 'eth_vhost0,iface=./vhost-net,queues=1' -- -i" % self.vhost, "testpmd> ",60)
crb.send_expect("start", "testpmd> ")
try:
# set up host virtual machine
self.host_vm = QEMUKvm(self.duts[0], 'host', 'vhost_user_live_migration')
vhost_params = {}
vhost_params['driver'] = 'vhost-user'
# qemu command can't use ~
base_dir = self.dut.base_dir.replace('~', '/root')
vhost_params['opt_path'] = base_dir + '/vhost-net'
vhost_params['opt_mac'] = self.virio_mac
self.host_vm.set_vm_device(**vhost_params)
self.logger.info("Start virtual machine on host")
self.vm_host = self.host_vm.start()
if self.vm_host is None:
raise Exception("Set up host VM ENV failed!")
self.host_serial = self.host_vm.connect_serial_port(name='vhost_user_live_migration')
if self.host_serial is None:
raise Exception("Connect host serial port failed!")
self.logger.info("Start virtual machine on backup host")
# set up backup virtual machine
self.backup_vm = QEMUKvm(self.duts[1], 'backup', 'vhost_user_live_migration')
vhost_params = {}
vhost_params['driver'] = 'vhost-user'
# qemu command can't use ~
base_dir = self.dut.base_dir.replace('~', '/root')
vhost_params['opt_path'] = base_dir + '/vhost-net'
vhost_params['opt_mac'] = self.virio_mac
self.backup_vm.set_vm_device(**vhost_params)
# start qemu command
self.backup_vm.start()
except Exception as ex:
if ex is VirtDutInitException:
self.host_vm.stop()
self.host_vm = None
# no session created yet, call internal stop function
self.backup_vm._stop_vm()
self.backup_vm = None
else:
self.destroy_vm_env()
raise Exception(ex)
self.env_done = True
def destroy_vm_env(self):
# if environment has been destroyed, just skip
if self.env_done is False:
return
if getattr(self, 'host_serial', None):
if self.host_vm is not None:
self.host_vm.close_serial_port()
if getattr(self, 'backup_serial', None):
if self.backup_serial is not None and self.backup_vm is not None:
self.backup_vm.close_serial_port()
if getattr(self, 'vm_host', None):
if self.vm_host is not None:
self.host_vm.stop()
self.host_vm = None
self.logger.info("Stop virtual machine on backup host")
if getattr(self, 'vm_backup', None):
if self.vm_backup is not None:
self.vm_backup.kill_all()
# backup vm dut has been initialized, destroy backup vm
self.backup_vm.stop()
self.backup_vm = None
if getattr(self, 'backup_vm', None):
# only qemu start, no session created
if self.backup_vm is not None:
self.backup_vm.stop()
self.backup_vm = None
# after vm stopped, stop vhost testpmd
for crb in self.duts[:2]:
crb.kill_all()
for crb in self.duts[:2]:
self.bind_nic_driver(crb, [crb.get_ports()[0]], driver="igb_uio")
self.env_done = False
def send_pkts(self, intf, number=0):
"""
send packet from tester
"""
sendp_fmt = "sendp([Ether(dst='%(DMAC)s')/IP()/UDP()/Raw('x'*18)], iface='%(INTF)s', count=%(COUNT)d)"
sendp_cmd = sendp_fmt % {'DMAC': self.virio_mac, 'INTF': intf, 'COUNT': number}
self.tester.scapy_append(sendp_cmd)
self.tester.scapy_execute()
# sleep 10 seconds for heavy load with backup host
time.sleep(10)
def verify_dpdk(self, tester_port, serial_session):
num_pkts = 10
stats_pat = re.compile("RX-packets: (\d+)")
intf = self.tester.get_interface(tester_port)
serial_session.send_expect("stop", "testpmd> ")
serial_session.send_expect("set fwd rxonly", "testpmd> ")
serial_session.send_expect("clear port stats all", "testpmd> ")
serial_session.send_expect("start tx_first", "testpmd> ")
# send packets from tester
self.send_pkts(intf, number=num_pkts)
out = serial_session.send_expect("show port stats 0", "testpmd> ")
m = stats_pat.search(out)
if m:
num_received = int(m.group(1))
else:
num_received = 0
self.logger.info("Verified %s packets recevied" % num_received)
self.verify(num_received >= num_pkts, "Not receive packets as expected!!!")
def verify_kernel(self, tester_port, vm_dut):
"""
Function to verify packets received by virtIO
"""
intf = self.tester.get_interface(tester_port)
num_pkts = 10
# get host interface
vm_intf = vm_dut.ports_info[0]['port'].get_interface_name()
# start tcpdump the interface
vm_dut.send_expect("ifconfig %s up" % vm_intf, "# ")
direct_pat = re.compile(r"(\s+)\[ (\S+) in\|out\|inout \]")
vm_dut.send_expect("tcpdump -h", "# ")
out = vm_dut.get_session_output(timeout=1)
m = direct_pat.search(out)
if m:
direct_param = "-" + m.group(2)[1] + " in"
else:
direct_param = ""
vm_dut.send_expect("tcpdump -i %s %s -v" % (vm_intf, direct_param), "listening on", 120)
# wait for promisc on
time.sleep(3)
# send packets from tester
self.send_pkts(intf, number=num_pkts)
# killall tcpdump and verify packet received
out = vm_dut.get_session_output(timeout=1)
vm_dut.send_expect("^C", "# ")
num = out.count('UDP')
self.logger.info("Verified %s packets recevied" % num_pkts)
self.verify(num == num_pkts, "Not receive packets as expected!!!")
def test_migrate_with_kernel(self):
"""
Verify migrate virtIO device from host to backup host,
Verify before/in/after migration, device with kernel driver can receive packets
"""
# bind virtio-net back to virtio-pci
self.bind_nic_driver(self.vm_host, [self.vm_host.get_ports()[0]], driver="")
# verify host virtio-net work fine
self.verify_kernel(self.host_tport, self.vm_host)
self.logger.info("Migrate host VM to backup host")
# start live migration
ret = self.host_vm.start_migration(self.backup_dutip, self.backup_vm.migrate_port)
self.verify(ret, "Failed to migration, please check VM and qemu version")
# make sure still can receive packets in migration process
self.verify_kernel(self.host_tport, self.vm_host)
self.logger.info("Waiting migration process done")
# wait live migration done
self.host_vm.wait_migration_done()
# check vhost testpmd log after migration
out = self.duts[0].get_session_output(timeout=1)
self.verify("closed" in out, "Vhost Connection NOT closed on host")
out = self.duts[1].get_session_output(timeout=1)
self.verify("established" in out, "Device not ready on backup host")
self.logger.info("Migration process done, then go to backup VM")
# connected backup VM
self.vm_backup = self.backup_vm.migrated_start()
# make sure still can receive packets
self.verify_kernel(self.backup_tport, self.vm_backup)
def test_migrate_with_dpdk(self):
# bind virtio-net to igb_uio
self.bind_nic_driver(self.vm_host, [self.vm_host.get_ports()[0]], driver="igb_uio")
# start testpmd on host vm
base_dir = self.vm_host.base_dir.replace('~', '/root')
self.host_serial.send_expect('cd %s' % base_dir, "# ")
self.host_serial.send_expect(self.vm_testpmd, "testpmd> ", 120)
# verify testpmd receive packets
self.verify_dpdk(self.host_tport, self.host_serial)
self.logger.info("Migrate host VM to backup host")
# start live migration
ret = self.host_vm.start_migration(self.backup_dutip, self.backup_vm.migrate_port)
self.verify(ret, "Failed to migration, please check VM and qemu version")
# make sure still can receive packets in migration process
self.verify_dpdk(self.host_tport, self.host_serial)
self.logger.info("Waiting migration process done")
# wait live migration done
self.host_vm.wait_migration_done()
# check vhost testpmd log after migration
out = self.duts[0].get_session_output(timeout=1)
self.verify("closed" in out, "Vhost Connection NOT closed on host")
out = self.duts[1].get_session_output(timeout=1)
self.verify("established" in out, "Device not ready on backup host")
self.logger.info("Migration process done, then go to backup VM")
time.sleep(5)
# make sure still can receive packets
self.backup_serial = self.backup_vm.connect_serial_port(name='vhost_user_live_migration', first=False)
if self.backup_serial is None:
raise Exception("Connect backup host serial port failed!")
self.verify_dpdk(self.backup_tport, self.backup_serial)
# quit testpmd
self.backup_serial.send_expect("quit", "# ")
def tear_down(self):
self.destroy_vm_env()
pass
def tear_down_all(self):
pass
class TestVhostPmdXstats(TestCase):
def set_up_all(self):
"""
Run at the start of each test suite.
"""
self.dut_ports = self.dut.get_ports(self.nic)
self.unbind_ports = copy.deepcopy(self.dut_ports)
self.unbind_ports.remove(0)
self.dut.unbind_interfaces_linux(self.unbind_ports)
cores = self.dut.get_core_list("1S/4C/1T")
self.coremask = utils.create_mask(cores)
self.scapy_num = 0
self.dmac = self.dut.get_mac_address(self.dut_ports[0])
self.virtio1_mac = "52:54:00:00:00:01"
# build sample app
out = self.dut.build_dpdk_apps("./examples/vhost")
self.verify("Error" not in out, "compilation error 1")
self.verify("No such file" not in out, "compilation error 2")
def set_up(self):
"""
Run before each test case.
Launch vhost sample using default params
"""
self.dut.send_expect("rm -rf ./vhost.out", "#")
self.dut.send_expect("rm -rf ./vhost-net*", "#")
self.dut.send_expect("killall vhost-switch", "#")
self.dut.send_expect("killall qemu-system-x86_64", "#")
def vm_testpmd_start(self):
"""
Start testpmd in vm
"""
self.vm_testpmd = "./%s/app/testpmd -c 0x3 -n 4 -- -i --txqflags=0xf01" % self.target
if self.vm_dut is not None:
self.vm_dut.send_expect(self.vm_testpmd, "testpmd>", 60)
def vm_tx_first_start(self):
"""
Start tx_first
"""
if self.vm_dut is not None:
# Start tx_first
self.vm_dut.send_expect("set fwd mac", "testpmd>")
self.vm_dut.send_expect("start tx_first", "testpmd>")
def start_onevm(self):
"""
Start One VM with one virtio device
"""
self.vm_dut = None
self.vm = QEMUKvm(self.dut, 'vm0', 'vhost_pmd_xstats')
vm_params = {}
vm_params['driver'] = 'vhost-user'
vm_params['opt_path'] = './vhost-net'
vm_params['opt_mac'] = self.virtio1_mac
self.vm.set_vm_device(**vm_params)
try:
self.vm_dut = self.vm.start()
if self.vm_dut is None:
raise Exception("Set up VM ENV failed")
except Exception as e:
self.logger.error("Failure for %s" % str(e))
return True
def scapy_send_packet(self, pktsize, dmac, num=1):
"""
Send a packet to port
"""
self.scapy_num += 1
txport = self.tester.get_local_port(self.dut_ports[0])
self.txItf = self.tester.get_interface(txport)
pkt = Packet(pkt_type='TCP', pkt_len=pktsize)
pkt.config_layer('ether', {
'dst': dmac,
})
pkt.send_pkt(tx_port=self.txItf, count=num)
def send_verify(self, scope, mun):
"""
according the scope to check results
"""
out = self.dut.send_expect("show port xstats %s" % self.dut_ports[0],
"testpmd>", 60)
packet = re.search("rx_%s_packets:\s*(\d*)" % scope, out)
sum_packet = packet.group(1)
self.verify(
int(sum_packet) >= mun, "Insufficient the received package")
def prepare_start(self):
"""
prepare all of the conditions for start
"""
self.dut.send_expect(
"./%s/app/testpmd -c %s -n %s --socket-mem 1024,0 --vdev 'net_vhost0,iface=vhost-net,queues=1' -- -i --nb-cores=1"
% (self.target, self.coremask, self.dut.get_memory_channels()),
"testpmd>", 60)
self.start_onevm()
self.vm_testpmd_start()
self.dut.send_expect("set fwd mac", "testpmd>", 60)
self.dut.send_expect("start tx_first", "testpmd>", 60)
self.vm_tx_first_start()
def test_based_size(self):
"""
Verify receiving and transmitting packets correctly in the Vhsot PMD xstats
"""
self.prepare_start()
sizes = [64, 65, 128, 256, 513, 1025]
scope = ''
for pktsize in sizes:
if pktsize == 64:
scope = 'size_64'
elif 65 <= pktsize <= 127:
scope = 'size_65_to_127'
elif 128 <= pktsize <= 255:
scope = 'size_128_to_255'
elif 256 <= pktsize <= 511:
scope = 'size_256_to_511'
elif 512 <= pktsize <= 1023:
scope = 'size_512_to_1023'
elif 1024 <= pktsize:
scope = 'size_1024_to_max'
self.scapy_send_packet(pktsize, self.dmac, 10000)
self.send_verify(scope, 10000)
self.clear_port_xstats(scope)
def clear_port_xstats(self, scope):
self.dut.send_expect("clear port xstats all", "testpmd>", 60)
out = self.dut.send_expect("show port xstats %s" % self.dut_ports[0],
"testpmd>", 60)
packet = re.search("rx_%s_packets:\s*(\d*)" % scope, out)
sum_packet = packet.group(1)
self.verify(int(sum_packet) == 0, "Insufficient the received package")
def test_based_types(self):
"""
Verify different type of packets receiving and transmitting packets correctly in the Vhsot PMD xstats
"""
self.prepare_start()
types = ['ff:ff:ff:ff:ff:ff', '01:00:00:33:00:01']
scope = ''
for p in types:
if p == 'ff:ff:ff:ff:ff:ff':
scope = 'broadcast'
self.dmac = 'ff:ff:ff:ff:ff:ff'
elif p == '01:00:00:33:00:01':
scope = 'multicast'
self.dmac = '01:00:00:33:00:01'
self.scapy_send_packet(64, self.dmac, 10000)
self.send_verify(scope, 10000)
self.clear_port_xstats(scope)
def test_stability(self):
"""
Verify stability case with multiple queues for Vhsot PMD xstats
Send packets for 30 minutes, check the Xstatsa still can work correctly
"""
self.scapy_num = 0
self.prepare_start()
date_old = datetime.datetime.now()
date_new = date_old + datetime.timedelta(minutes=2)
while (1):
date_now = datetime.datetime.now()
self.scapy_send_packet(64, self.dmac, 1)
if date_now >= date_new:
break
out_0 = self.dut.send_expect("show port xstats %s" % self.dut_ports[0],
"testpmd>", 60)
rx_packet = re.search("rx_size_64_packets:\s*(\d*)", out_0)
rx_packets = rx_packet.group(1)
self.verify(
self.scapy_num == int(rx_packets),
"Error for rx_package:%s != tx_package :%s" %
(self.scapy_num, int(rx_packets)))
def tear_down(self):
"""
Run after each test case.
"""
self.vm._stop_vm()
self.dut.kill_all()
time.sleep(2)
def tear_down_all(self):
"""
Run after each test suite.
"""
self.dut.bind_interfaces_linux(nics_to_bind=self.unbind_ports)