def delete_framework_dir(node):
"""Delete framework directory in /tmp/ on given node.
:param node: Node to delete framework directory on.
:type node: dict
"""
logger.console(
f"Deleting framework directory on {node[u'type']} host {node[u'host']},"
f" port {node[u'port']} starts.")
exec_cmd_no_error(
node,
f"sudo rm -rf {con.REMOTE_FW_DIR}",
message=f"Framework delete failed at node {node[u'type']} "
f"host {node[u'host']}, port {node[u'port']}",
timeout=100,
include_reason=True)
logger.console(
f"Deleting framework directory on {node[u'type']} host {node[u'host']},"
f" port {node[u'port']} done.")
def start_iperf_server(node, namespace=None, port=5201, affinity=None):
"""Start iPerf3 server instance as a deamon.
:param node: Topology node running iPerf3 server.
:param namespace: Name of TG namespace to execute.
:param port: The server port for the server to listen on.
:param affinity: iPerf3 server affinity.
:type node: dict
:type namespace: str
:type port: int
:type affinity: str
"""
cmd = IPerf3Server.iperf3_cmdline(namespace=namespace,
port=port,
affinity=affinity)
exec_cmd_no_error(node,
cmd,
sudo=True,
message=u"Failed to start iPerf3 server!")
def start_hoststack_test_program(node, namespace, core_list, program):
"""Start the specified HostStack test program.
:param node: DUT node.
:param namespace: Net Namespace to run program in.
:param core_list: List of cpu's to pass to taskset to pin the test
program to a different set of cores on the same numa node as VPP.
:param program: Test program.
:type node: dict
:type namespace: str
:type core_list: str
:type program: dict
:returns: Process ID
:rtype: int
:raises RuntimeError: If node subtype is not a DUT or startup failed.
"""
if node[u"type"] != u"DUT":
raise RuntimeError(u"Node type is not a DUT!")
program_name = program[u"name"]
DUTSetup.kill_program(node, program_name, namespace)
if namespace == u"default":
shell_cmd = u"sh -c"
else:
shell_cmd = f"ip netns exec {namespace} sh -c"
env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
args = program[u"args"]
cmd = f"nohup {shell_cmd} \'{env_vars}taskset --cpu-list {core_list} " \
f"{program_name} {args} >/tmp/{program_name}_stdout.log " \
f"2>/tmp/{program_name}_stderr.log &\'"
try:
exec_cmd_no_error(node, cmd, sudo=True)
return DUTSetup.get_pid(node, program_name)[0]
except RuntimeError:
stdout_log, stderr_log = \
HoststackUtil.get_hoststack_test_program_logs(node,
program)
raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
f"{stderr_log}\nSTDOUT:\n{stdout_log}")
return None
def restart_service(node, service):
"""Restart the named service on node.
:param node: Node in the topology.
:param service: Service unit name.
:type node: dict
:type service: str
"""
command = f"supervisorctl restart {service}" \
if DUTSetup.running_in_container(node) \
else f"service {service} restart"
message = f"Node {node[u'host']} failed to restart service {service}"
exec_cmd_no_error(node,
command,
timeout=180,
sudo=True,
message=message)
DUTSetup.get_service_logs(node, service)
def get_service_logs(node, service):
"""Get specific service unit logs from node.
:param node: Node in the topology.
:param service: Service unit name.
:type node: dict
:type service: str
"""
command = u"cat /tmp/*supervisor*.log"\
if DUTSetup.running_in_container(node) \
else f"journalctl --no-pager _SYSTEMD_INVOCATION_ID=$(systemctl " \
f"show -p InvocationID --value {service})"
message = f"Node {node[u'host']} failed to get logs from unit {service}"
exec_cmd_no_error(node,
command,
timeout=30,
sudo=True,
message=message)
def vhost_user_affinity(node, pf_key, skip_cnt=0):
"""Set vhost-user affinity for the given node.
:param node: Topology node.
:param pf_key: Interface key to compute numa location.
:param skip_cnt: Skip first "skip_cnt" CPUs.
:type node: dict
:type pf_key: str
:type skip_cnt: int
"""
pids, _ = exec_cmd_no_error(
node, f"grep -h vhost /proc/*/comm | uniq | xargs pidof")
affinity = CpuUtils.get_affinity_vhost(node,
pf_key,
skip_cnt=skip_cnt,
cpu_cnt=len(pids.split(" ")))
for cpu, pid in zip(affinity, pids.split(" ")):
exec_cmd_no_error(node, f"taskset -pc {cpu} {pid}", sudo=True)
def get_cpu_info_from_all_nodes(nodes):
"""Assuming all nodes are Linux nodes, retrieve the following
cpu information from all nodes:
- cpu architecture
- cpu layout
:param nodes: DICT__nodes from Topology.DICT__nodes.
:type nodes: dict
:raises RuntimeError: If an ssh command retrieving cpu information
fails.
"""
for node in nodes.values():
stdout, _ = exec_cmd_no_error(node, u"uname -m")
node[u"arch"] = stdout.strip()
stdout, _ = exec_cmd_no_error(node, u"lscpu -p")
node[u"cpuinfo"] = list()
for line in stdout.split(u"\n"):
if line and line[0] != u"#":
node[u"cpuinfo"].append(
[CpuUtils.__str2int(x) for x in line.split(u",")])
def add_route(node, ip_addr, prefix, gateway, namespace=None):
"""Add route in namespace.
:param node: Node where to execute command.
:param ip_addr: Route destination IP address.
:param prefix: IP prefix.
:param namespace: Execute command in namespace. Optional.
:param gateway: Gateway address.
:type node: dict
:type ip_addr: str
:type prefix: int
:type gateway: str
:type namespace: str
"""
if namespace is not None:
cmd = 'ip netns exec {} ip route add {}/{} via {}'.format(
namespace, ip_addr, prefix, gateway)
else:
cmd = 'ip route add {}/{} via {}'.format(ip_addr, prefix, gateway)
exec_cmd_no_error(node, cmd, sudo=True)
def pci_driver_unbind(node, pci_addr):
"""Unbind PCI device from current driver on node.
:param node: DUT node.
:param pci_addr: PCI device address.
:type node: dict
:type pci_addr: str
:raises RuntimeError: If PCI device unbind failed.
"""
command = "sh -c "\
"'echo {pci} | tee /sys/bus/pci/devices/{pcie}/driver/unbind'".\
format(pci=pci_addr, pcie=pci_addr.replace(':', r'\:'))
message = 'Failed to unbind PCI device {pci} on {host}'.\
format(pci=pci_addr, host=node['host'])
exec_cmd_no_error(node,
command,
timeout=120,
sudo=True,
message=message)
def verify_vpp_started(node):
"""Verify that VPP is started on the specified topology node.
:param node: Topology node.
:type node: dict
"""
cmd = u"echo \"show pci\" | sudo socat - UNIX-CONNECT:/run/vpp/cli.sock"
exec_cmd_no_error(node,
cmd,
sudo=False,
message=u"VPP failed to start!",
retries=120)
cmd = u"vppctl show pci 2>&1 | fgrep -v \"Connection refused\" | " \
u"fgrep -v \"No such file or directory\""
exec_cmd_no_error(node,
cmd,
sudo=True,
message=u"VPP failed to start!",
retries=120)
def running_in_container(node):
"""This method tests if topology node is running inside container.
:param node: Topology node.
:type node: dict
:returns: True if running in docker container, false if not or failed
to detect.
:rtype: bool
"""
command = "fgrep docker /proc/1/cgroup"
message = 'Failed to get cgroup settings.'
try:
exec_cmd_no_error(node,
command,
timeout=30,
sudo=False,
message=message)
except RuntimeError:
return False
return True
def linux_del_bridge(node, br_name, set_down=True):
"""Delete bridge from linux node.
..note:: The network interface corresponding to the bridge must be
down before it can be deleted!
:param node: Node to delete bridge from.
:param br_name: Bridge name.
:param set_down: Change bridge interface state to down before delbr
command. Optional. Default: True.
:type node: dict
:type br_name: str
:type set_down: bool
"""
if set_down:
cmd = 'ip link set dev {0} down'.format(br_name)
exec_cmd_no_error(node, cmd, sudo=True)
cmd = 'brctl delbr {0}'.format(br_name)
exec_cmd_no_error(node, cmd, sudo=True)
def stop_hoststack_test_program(node, program, pid):
"""Stop the specified Hoststack test program.
:param node: DUT node.
:param program: Test program.
:param pid: Process ID of test program.
:type node: dict
:type program: dict
:type pid: int
"""
program_name = program[u"name"]
if program_name == u"nginx":
cmd = u"nginx -s quit"
errmsg = u"Quit nginx failed!"
else:
cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
f'then kill -s SIGTERM {pid}; fi'
errmsg = f"Kill {program_name} ({pid}) failed!"
exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)
def _qemu_qga_flush(self):
"""Flush the QGA parser state."""
command = f"(printf \"\xFF\"; sleep 1) | sudo -S socat " \
f"- UNIX-CONNECT:{self._temp.get(u'qga')}"
message = f"QGA flush failed on {self._node[u'host']}"
stdout, _ = exec_cmd_no_error(self._node,
command,
sudo=False,
message=message)
return json.loads(stdout.split(u"\n", 1)[0]) if stdout else dict()
def get_qemu_pids(self):
"""Get QEMU CPU pids.
:returns: List of QEMU CPU pids.
:rtype: list of str
"""
command = f"grep -rwl 'CPU' /proc/$(sudo cat " \
f"{self._temp.get(u'pidfile')})/task/*/comm "
command += r"| xargs dirname | sed -e 's/\/.*\///g' | uniq"
stdout, _ = exec_cmd_no_error(self._node, command)
return stdout.splitlines()
def pci_vf_driver_unbind(node, pf_pci_addr, vf_id):
"""Unbind Virtual Function from driver on node.
:param node: DUT node.
:param pf_pci_addr: PCI device address.
:param vf_id: Virtual Function ID.
:type node: dict
:type pf_pci_addr: str
:type vf_id: int
:raises RuntimeError: If Virtual Function unbind failed.
"""
vf_pci_addr = DUTSetup.get_virtfn_pci_addr(node, pf_pci_addr, vf_id)
pf_pci = pf_pci_addr.replace(u":", r"\:")
vf_path = f"/sys/bus/pci/devices/{pf_pci}/virtfn{vf_id}"
command = f"sh -c \"echo {vf_pci_addr} | tee {vf_path}/driver/unbind\""
message = f"Failed to unbind VF {vf_pci_addr} on {node[u'host']}"
exec_cmd_no_error(
node, command, timeout=120, sudo=True, message=message
)
def set_sriov_numvfs(node, pf_pci_addr, numvfs=0):
"""Init or reset SR-IOV virtual functions by setting its number on PCI
device on DUT. Setting to zero removes all VFs.
:param node: DUT node.
:param pf_pci_addr: Physical Function PCI device address.
:param numvfs: Number of VFs to initialize, 0 - removes the VFs.
:type node: dict
:type pf_pci_addr: str
:type numvfs: int
:raises RuntimeError: Failed to create VFs on PCI.
"""
pci = pf_pci_addr.replace(u":", r"\:")
command = f"sh -c \"echo {numvfs} | " \
f"tee /sys/bus/pci/devices/{pci}/sriov_numvfs\""
message = f"Failed to create {numvfs} VFs on {pf_pci_addr} device " \
f"on {node[u'host']}"
exec_cmd_no_error(
node, command, timeout=120, sudo=True, message=message
)
def add_linux_route(node, ip_addr, prefix, gateway, namespace=None):
"""Add linux route in namespace.
:param node: Node where to execute command.
:param ip_addr: Route destination IP address.
:param prefix: IP prefix.
:param namespace: Execute command in namespace. Optional.
:param gateway: Gateway address.
:type node: dict
:type ip_addr: str
:type prefix: int
:type gateway: str
:type namespace: str
"""
if namespace is not None:
cmd = f"ip netns exec {namespace} ip route add {ip_addr}/{prefix}" \
f" via {gateway}"
else:
cmd = f"ip route add {ip_addr}/{prefix} via {gateway}"
exec_cmd_no_error(node, cmd, sudo=True)
def setup_network_namespace(node, namespace_name, interface_name, ip_addr,
prefix):
"""Setup namespace on given node and attach interface and IP to
this namespace. Applicable also on TG node.
:param node: Node to set namespace on.
:param namespace_name: Namespace name.
:param interface_name: Interface name.
:param ip_addr: IP address of namespace's interface.
:param prefix: IP address prefix length.
:type node: dict
:type namespace_name: str
:type vhost_if: str
:type ip_addr: str
:type prefix: int
"""
cmd = ('ip netns add {0}'.format(namespace_name))
exec_cmd_no_error(node, cmd, sudo=True)
cmd = ('ip link set dev {0} up netns {1}'.format(
interface_name, namespace_name))
exec_cmd_no_error(node, cmd, sudo=True)
cmd = ('ip netns exec {0} ip addr add {1}/{2} dev {3}'.format(
namespace_name, ip_addr, prefix, interface_name))
exec_cmd_no_error(node, cmd, sudo=True)
def get_core_files_on_all_nodes(self, nodes, disable_on_success=True):
"""Compress all core files into single file and remove the original
core files on all nodes.
:param nodes: Nodes in the topology.
:param disable_on_success: If True, disable setting of core limit by
this instance of library. Default: True
:type nodes: dict
:type disable_on_success: bool
"""
for node in nodes.values():
uuid = str(time()).replace('.', '')
name = '{uuid}.tar.lzo.lrz.xz'.format(uuid=uuid)
command = ('[ -e {dir}/*.core ] && cd {dir} && '
'sudo tar c *.core | '
'lzop -1 | '
'lrzip -n -T -p 1 -w 5 | '
'xz -9e > {name} && '
'sudo rm -f *.core'.format(dir=Constants.CORE_DUMP_DIR,
name=name))
try:
exec_cmd_no_error(node, command, timeout=3600)
if disable_on_success:
self.set_core_limit_disabled()
except RuntimeError:
# If compress was not sucessfull ignore error and skip further
# processing.
continue
local_path = 'archive/{name}'.format(name=name)
remote_path = '{dir}/{name}'.format(dir=Constants.CORE_DUMP_DIR,
name=name)
try:
scp_node(node, local_path, remote_path, get=True, timeout=3600)
command = 'rm -f {dir}/{name}'\
.format(dir=Constants.CORE_DUMP_DIR, name=name)
exec_cmd_no_error(node, command, sudo=True)
except RuntimeError:
pass
def pci_driver_bind(node, pci_addr, driver):
"""Bind PCI device to driver on node.
:param node: DUT node.
:param pci_addr: PCI device address.
:param driver: Driver to bind.
:type node: dict
:type pci_addr: str
:type driver: str
:raises RuntimeError: If PCI device bind failed.
"""
message = f"Failed to bind PCI device {pci_addr} to {driver} " \
f"on host {node[u'host']}"
pci = pci_addr.replace(u":", r"\:")
command = f"sh -c \"echo {driver} | " \
f"tee /sys/bus/pci/devices/{pci}/driver_override\""
exec_cmd_no_error(
node, command, timeout=120, sudo=True, message=message
)
command = f"sh -c \"echo {pci_addr} | " \
f"tee /sys/bus/pci/drivers/{driver}/bind\""
exec_cmd_no_error(
node, command, timeout=120, sudo=True, message=message
)
command = f"sh -c \"echo | " \
f"tee /sys/bus/pci/devices/{pci}/driver_override\""
exec_cmd_no_error(
node, command, timeout=120, sudo=True, message=message
)
def qemu_start(self):
"""Start QEMU and wait until VM boot.
:returns: VM node info.
:rtype: dict
"""
cmd_opts = OptionString()
cmd_opts.add(f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch}")
cmd_opts.extend(self._params)
message = f"QEMU: Start failed on {self._node[u'host']}!"
try:
DUTSetup.check_huge_page(
self._node, u"/dev/hugepages", int(self._opt.get(u"mem")))
exec_cmd_no_error(
self._node, cmd_opts, timeout=300, sudo=True, message=message
)
self._wait_until_vm_boot()
except RuntimeError:
self.qemu_kill_all()
raise
return self._vm_info
def create_env_directory_at_node(node):
"""Create fresh virtualenv to a directory, install pip requirements.
:param node: Node to create virtualenv on.
:type node: dict
:returns: nothing
:raises RuntimeError: When failed to setup virtualenv.
"""
host = node['host']
logger.console(
'Virtualenv setup including requirements.txt on {0} starts.'.format(
host))
exec_cmd_no_error(
node,
'cd {0} && rm -rf env'
' && virtualenv --system-site-packages --never-download env'
' && source env/bin/activate && pip install -r requirements.txt'.
format(con.REMOTE_FW_DIR),
timeout=100,
include_reason=True,
message="Failed install at node {host}".format(host=host))
logger.console('Virtualenv setup on {0} done.'.format(host))
def stop_service(node, service):
"""Stop the named service on node.
:param node: Node in the topology.
:param service: Service unit name.
:type node: dict
:type service: str
"""
if DUTSetup.running_in_container(node):
command = 'supervisorctl stop {name}'.format(name=service)
else:
command = 'service {name} stop'.format(name=service)
message = 'Node {host} failed to stop service {name}'.\
format(host=node['host'], name=service)
exec_cmd_no_error(node,
command,
timeout=30,
sudo=True,
message=message)
DUTSetup.get_service_logs(node, service)
def get_the_test_result(dut_node):
"""
After executing exec_the_base_vs_epoll_test, use this
to get the test result.
:param dut_node: Node to get the test result on.
:type dut_node: dict
:returns: Word count of "send 50000" in the log.
:rtype: str
"""
cmd = 'cat {0}/{1}/log_run_dmm.txt | grep "send 50000" | wc -l' \
.format(con.REMOTE_FW_DIR, con.DMM_SCRIPTS)
(stdout, _) = exec_cmd_no_error(dut_node, cmd)
return stdout
def start_service(node, service):
"""Start up the named service on node.
:param node: Node in the topology.
:param service: Service unit name.
:type node: dict
:type service: str
"""
# TODO: change command to start once all parent function updated.
if DUTSetup.running_in_container(node):
command = 'supervisorctl restart {name}'.format(name=service)
else:
command = 'service {name} restart'.format(name=service)
message = 'Node {host} failed to start service {name}'.\
format(host=node['host'], name=service)
exec_cmd_no_error(node,
command,
timeout=180,
sudo=True,
message=message)
DUTSetup.get_service_logs(node, service)
def set_sriov_numvfs(node, pf_pci_addr, numvfs=0):
"""Init or reset SR-IOV virtual functions by setting its number on PCI
device on DUT. Setting to zero removes all VFs.
:param node: DUT node.
:param pf_pci_addr: Physical Function PCI device address.
:param numvfs: Number of VFs to initialize, 0 - removes the VFs.
:type node: dict
:type pf_pci_addr: str
:type numvfs: int
:raises RuntimeError: Failed to create VFs on PCI.
"""
command = "sh -c "\
"'echo {num} | tee /sys/bus/pci/devices/{pci}/sriov_numvfs'".\
format(num=numvfs, pci=pf_pci_addr.replace(':', r'\:'))
message = 'Failed to create {num} VFs on {pci} device on {host}'.\
format(num=numvfs, pci=pf_pci_addr, host=node['host'])
exec_cmd_no_error(node,
command,
timeout=120,
sudo=True,
message=message)
def get_service_logs(node, service):
"""Get specific service unit logs from node.
:param node: Node in the topology.
:param service: Service unit name.
:type node: dict
:type service: str
"""
if DUTSetup.running_in_container(node):
command = 'echo $(< /var/log/supervisord.log)'
else:
command = ('journalctl --no-pager --unit={name} '
'--since="$(echo `systemctl show -p '
'ActiveEnterTimestamp {name}` | '
'awk \'{{print $2 $3}}\')"'.format(name=service))
message = 'Node {host} failed to get logs from unit {name}'.\
format(host=node['host'], name=service)
exec_cmd_no_error(node,
command,
timeout=30,
sudo=True,
message=message)
def qemu_version(self):
"""Return Qemu version.
:returns: Qemu version.
:rtype: str
"""
command = f"{Constants.QEMU_BIN_PATH}/qemu-system-{self._arch} " \
f"--version"
try:
stdout, _ = exec_cmd_no_error(self._node, command, sudo=True)
return match(r"QEMU emulator version ([\d.]*)", stdout).group(1)
except RuntimeError:
self.qemu_kill_all()
raise
def set_linux_interface_ip(
node, interface, ip_addr, prefix, namespace=None):
"""Set IP address to interface in linux.
:param node: VPP/TG node.
:param interface: Interface in namespace.
:param ip_addr: IP to be set on interface.
:param prefix: IP prefix.
:param namespace: Execute command in namespace. Optional
:type node: dict
:type interface: str
:type ip_addr: str
:type prefix: int
:type namespace: str
:raises RuntimeError: IP could not be set.
"""
if namespace is not None:
cmd = f"ip netns exec {namespace} ip addr add {ip_addr}/{prefix}" \
f" dev {interface}"
else:
cmd = f"ip addr add {ip_addr}/{prefix} dev {interface}"
exec_cmd_no_error(node, cmd, timeout=5, sudo=True)
