class OvsDpdkContext(Context):
""" This class handles OVS standalone nodes - VM running on Non-Managed NFVi
Configuration: ovs_dpdk
"""
__context_type__ = "StandaloneOvsDpdk"
SUPPORTED_OVS_TO_DPDK_MAP = {
'2.6.0': '16.07.1',
'2.6.1': '16.07.2',
'2.7.0': '16.11.1',
'2.7.1': '16.11.2',
'2.7.2': '16.11.3',
'2.8.0': '17.05.2'
}
DEFAULT_OVS = '2.6.0'
PKILL_TEMPLATE = "pkill %s %s"
def __init__(self):
self.file_path = None
self.sriov = []
self.first_run = True
self.dpdk_nic_bind = ""
self.vm_names = []
self.name = None
self.nfvi_host = []
self.nodes = []
self.networks = {}
self.attrs = {}
self.vm_flavor = None
self.servers = None
self.helper = StandaloneContextHelper()
self.vnf_node = Server()
self.ovs_properties = {}
self.wait_for_vswitchd = 10
super(OvsDpdkContext, self).__init__()
def init(self, attrs):
"""initializes itself from the supplied arguments"""
self.name = attrs["name"]
self.file_path = attrs.get("file", "pod.yaml")
self.nodes, self.nfvi_host, self.host_mgmt = \
self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
self.attrs = attrs
self.vm_flavor = attrs.get('flavor', {})
self.servers = attrs.get('servers', {})
self.vm_deploy = attrs.get("vm_deploy", True)
self.ovs_properties = attrs.get('ovs_properties', {})
# add optional static network definition
self.networks = attrs.get("networks", {})
LOG.debug("Nodes: %r", self.nodes)
LOG.debug("NFVi Node: %r", self.nfvi_host)
LOG.debug("Networks: %r", self.networks)
def setup_ovs(self):
vpath = self.ovs_properties.get("vpath", "/usr/local")
xargs_kill_cmd = self.PKILL_TEMPLATE % ('-9', 'ovs')
create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
cmd_list = [
"chmod 0666 /dev/vfio/*",
"chmod a+x /dev/vfio",
"pkill -9 ovs",
xargs_kill_cmd,
"killall -r 'ovs*'",
"mkdir -p {0}/etc/openvswitch".format(vpath),
"mkdir -p {0}/var/run/openvswitch".format(vpath),
"rm {0}/etc/openvswitch/conf.db".format(vpath),
"ovsdb-tool create {0} {1}".format(create_from, create_to),
"modprobe vfio-pci",
"chmod a+x /dev/vfio",
"chmod 0666 /dev/vfio/*",
]
for cmd in cmd_list:
self.connection.execute(cmd)
bind_cmd = "{dpdk_nic_bind} --force -b {driver} {port}"
phy_driver = "vfio-pci"
for key, port in self.networks.items():
vpci = port.get("phy_port")
self.connection.execute(
bind_cmd.format(dpdk_nic_bind=self.dpdk_nic_bind,
driver=phy_driver,
port=vpci))
def start_ovs_serverswitch(self):
vpath = self.ovs_properties.get("vpath")
pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
ovs_sock_path = '/var/run/openvswitch/db.sock'
log_path = '/var/log/openvswitch/ovs-vswitchd.log'
pmd_mask = hex(sum(2**num for num in range(pmd_nums)) << 1)
socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
cmd_list = [
"mkdir -p /usr/local/var/run/openvswitch",
"mkdir -p {}".format(os.path.dirname(log_path)),
"ovsdb-server --remote=punix:/{0}/{1} --pidfile --detach".format(
vpath, ovs_sock_path),
ovs_other_config.format("--no-wait ", "dpdk-init=true"),
ovs_other_config.format(
"--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
detach_cmd.format(vpath, ovs_sock_path, log_path),
ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
]
for cmd in cmd_list:
LOG.info(cmd)
self.connection.execute(cmd)
time.sleep(self.wait_for_vswitchd)
def setup_ovs_bridge_add_flows(self):
dpdk_args = ""
dpdk_list = []
vpath = self.ovs_properties.get("vpath", "/usr/local")
version = self.ovs_properties.get('version', {})
ovs_ver = [
int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')
]
ovs_add_port = \
"ovs-vsctl add-port {br} {port} -- set Interface {port} type={type_}{dpdk_args}"
ovs_add_queue = "ovs-vsctl set Interface {port} options:n_rxq={queue}"
chmod_vpath = "chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*"
cmd_dpdk_list = [
"ovs-vsctl del-br br0",
"rm -rf {0}/var/run/openvswitch/dpdkvhostuser*".format(vpath),
"ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev",
]
ordered_network = OrderedDict(self.networks)
for index, (key, vnf) in enumerate(ordered_network.items()):
if ovs_ver >= [2, 7, 0]:
dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
dpdk_list.append(
ovs_add_port.format(br='br0',
port='dpdk%s' % vnf.get("port_num", 0),
type_='dpdk',
dpdk_args=dpdk_args))
dpdk_list.append(
ovs_add_queue.format(port='dpdk%s' % vnf.get("port_num", 0),
queue=self.ovs_properties.get(
"queues", 1)))
# Sorting the array to make sure we execute dpdk0... in the order
list.sort(dpdk_list)
cmd_dpdk_list.extend(dpdk_list)
# Need to do two for loop to maintain the dpdk/vhost ports.
for index, _ in enumerate(ordered_network):
cmd_dpdk_list.append(
ovs_add_port.format(br='br0',
port='dpdkvhostuser%s' % index,
type_='dpdkvhostuser',
dpdk_args=""))
for cmd in cmd_dpdk_list:
LOG.info(cmd)
self.connection.execute(cmd)
# Fixme: add flows code
ovs_flow = "ovs-ofctl add-flow br0 in_port=%s,action=output:%s"
network_count = len(ordered_network) + 1
for in_port, out_port in zip(range(1, network_count),
range(network_count, network_count * 2)):
self.connection.execute(ovs_flow % (in_port, out_port))
self.connection.execute(ovs_flow % (out_port, in_port))
self.connection.execute(chmod_vpath.format(vpath))
def cleanup_ovs_dpdk_env(self):
self.connection.execute("ovs-vsctl del-br br0")
self.connection.execute("pkill -9 ovs")
def check_ovs_dpdk_env(self):
self.cleanup_ovs_dpdk_env()
version = self.ovs_properties.get("version", {})
ovs_ver = version.get("ovs", self.DEFAULT_OVS)
dpdk_ver = version.get("dpdk", "16.07.2").split('.')
supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
if supported_version is None or supported_version.split(
'.')[:2] != dpdk_ver[:2]:
raise Exception(
"Unsupported ovs '{}'. Please check the config...".format(
ovs_ver))
status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" %
ovs_ver)[0]
if status:
deploy = OvsDeploy(self.connection, get_nsb_option("bin_path"),
self.ovs_properties)
deploy.ovs_deploy()
def deploy(self):
"""don't need to deploy"""
# Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
if not self.vm_deploy:
return
self.connection = ssh.SSH.from_node(self.host_mgmt)
self.dpdk_nic_bind = provision_tool(
self.connection,
os.path.join(get_nsb_option("bin_path"), "dpdk-devbind.py"))
# Check dpdk/ovs version, if not present install
self.check_ovs_dpdk_env()
# Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
StandaloneContextHelper.install_req_libs(self.connection)
self.networks = StandaloneContextHelper.get_nic_details(
self.connection, self.networks, self.dpdk_nic_bind)
self.setup_ovs()
self.start_ovs_serverswitch()
self.setup_ovs_bridge_add_flows()
self.nodes = self.setup_ovs_dpdk_context()
LOG.debug("Waiting for VM to come up...")
self.nodes = StandaloneContextHelper.wait_for_vnfs_to_start(
self.connection, self.servers, self.nodes)
def undeploy(self):
if not self.vm_deploy:
return
# Cleanup the ovs installation...
self.cleanup_ovs_dpdk_env()
# Bind nics back to kernel
bind_cmd = "{dpdk_nic_bind} --force -b {driver} {port}"
for key, port in self.networks.items():
vpci = port.get("phy_port")
phy_driver = port.get("driver")
self.connection.execute(
bind_cmd.format(dpdk_nic_bind=self.dpdk_nic_bind,
driver=phy_driver,
port=vpci))
# Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
for vm in self.vm_names:
Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
def _get_server(self, attr_name):
"""lookup server info by name from context
Keyword arguments:
attr_name -- A name for a server listed in nodes config file
"""
node_name, name = self.split_name(attr_name)
if name is None or self.name != name:
return None
matching_nodes = (n for n in self.nodes if n["name"] == node_name)
try:
# A clone is created in order to avoid affecting the
# original one.
node = dict(next(matching_nodes))
except StopIteration:
return None
try:
duplicate = next(matching_nodes)
except StopIteration:
pass
else:
raise ValueError("Duplicate nodes!!! Nodes: %s %s",
(node, duplicate))
node["name"] = attr_name
return node
def _get_network(self, attr_name):
if not isinstance(attr_name, collections.Mapping):
network = self.networks.get(attr_name)
else:
# Don't generalize too much Just support vld_id
vld_id = attr_name.get('vld_id', {})
# for standalone context networks are dicts
iter1 = (n for n in self.networks.values()
if n.get('vld_id') == vld_id)
network = next(iter1, None)
if network is None:
return None
result = {
# name is required
"name": network["name"],
"vld_id": network.get("vld_id"),
"segmentation_id": network.get("segmentation_id"),
"network_type": network.get("network_type"),
"physical_network": network.get("physical_network"),
}
return result
def configure_nics_for_ovs_dpdk(self):
portlist = OrderedDict(self.networks)
for key, ports in portlist.items():
mac = StandaloneContextHelper.get_mac_address()
portlist[key].update({'mac': mac})
self.networks = portlist
LOG.info("Ports %s" % self.networks)
def _enable_interfaces(self, index, vfs, cfg):
vpath = self.ovs_properties.get("vpath", "/usr/local")
vf = self.networks[vfs[0]]
port_num = vf.get('port_num', 0)
vpci = PciAddress.parse_address(vf['vpci'].strip(), multi_line=True)
# Generate the vpci for the interfaces
slot = index + port_num + 10
vf['vpci'] = \
"{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
Libvirt.add_ovs_interface(vpath, port_num, vf['vpci'], vf['mac'],
str(cfg))
def setup_ovs_dpdk_context(self):
nodes = []
self.configure_nics_for_ovs_dpdk()
for index, (key, vnf) in enumerate(OrderedDict(self.servers).items()):
cfg = '/tmp/vm_ovs_%d.xml' % index
vm_name = "vm_%d" % index
# 1. Check and delete VM if already exists
Libvirt.check_if_vm_exists_and_delete(vm_name, self.connection)
vcpu, mac = Libvirt.build_vm_xml(self.connection, self.vm_flavor,
cfg, vm_name, index)
# 2: Cleanup already available VMs
for idx, (vkey, vfs) in enumerate(
OrderedDict(vnf["network_ports"]).items()):
if vkey == "mgmt":
continue
self._enable_interfaces(index, vfs, cfg)
# copy xml to target...
self.connection.put(cfg, cfg)
# FIXME: launch through libvirt
LOG.info("virsh create ...")
Libvirt.virsh_create_vm(self.connection, cfg)
self.vm_names.append(vm_name)
# build vnf node details
nodes.append(
self.vnf_node.generate_vnf_instance(self.vm_flavor,
self.networks,
self.host_mgmt.get('ip'),
key, vnf, mac))
return nodes
class SriovContext(Context):
""" This class handles SRIOV standalone nodes - VM running on Non-Managed NFVi
Configuration: sr-iov
"""
__context_type__ = "StandaloneSriov"
def __init__(self):
self.file_path = None
self.sriov = []
self.first_run = True
self.dpdk_nic_bind = ""
self.vm_names = []
self.name = None
self.nfvi_host = []
self.nodes = []
self.networks = {}
self.attrs = {}
self.vm_flavor = None
self.servers = None
self.helper = StandaloneContextHelper()
self.vnf_node = Server()
self.drivers = []
super(SriovContext, self).__init__()
def init(self, attrs):
"""initializes itself from the supplied arguments"""
self.name = attrs["name"]
self.file_path = attrs.get("file", "pod.yaml")
self.nodes, self.nfvi_host, self.host_mgmt = \
self.helper.parse_pod_file(self.file_path, 'Sriov')
self.attrs = attrs
self.vm_flavor = attrs.get('flavor', {})
self.servers = attrs.get('servers', {})
self.vm_deploy = attrs.get("vm_deploy", True)
# add optional static network definition
self.networks = attrs.get("networks", {})
LOG.debug("Nodes: %r", self.nodes)
LOG.debug("NFVi Node: %r", self.nfvi_host)
LOG.debug("Networks: %r", self.networks)
def deploy(self):
"""don't need to deploy"""
# Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
if not self.vm_deploy:
return
self.connection = ssh.SSH.from_node(self.host_mgmt)
self.dpdk_nic_bind = provision_tool(
self.connection,
os.path.join(get_nsb_option("bin_path"), "dpdk_nic_bind.py"))
# Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
StandaloneContextHelper.install_req_libs(self.connection)
self.networks = StandaloneContextHelper.get_nic_details(
self.connection, self.networks, self.dpdk_nic_bind)
self.nodes = self.setup_sriov_context()
LOG.debug("Waiting for VM to come up...")
self.nodes = StandaloneContextHelper.wait_for_vnfs_to_start(
self.connection, self.servers, self.nodes)
def undeploy(self):
"""don't need to undeploy"""
if not self.vm_deploy:
return
# Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
for vm in self.vm_names:
Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
# Bind nics back to kernel
for ports in self.networks.values():
# enable VFs for given...
build_vfs = "echo 0 > /sys/bus/pci/devices/{0}/sriov_numvfs"
self.connection.execute(build_vfs.format(ports.get('phy_port')))
def _get_server(self, attr_name):
"""lookup server info by name from context
Keyword arguments:
attr_name -- A name for a server listed in nodes config file
"""
node_name, name = self.split_name(attr_name)
if name is None or self.name != name:
return None
matching_nodes = (n for n in self.nodes if n["name"] == node_name)
try:
# A clone is created in order to avoid affecting the
# original one.
node = dict(next(matching_nodes))
except StopIteration:
return None
try:
duplicate = next(matching_nodes)
except StopIteration:
pass
else:
raise ValueError("Duplicate nodes!!! Nodes: %s %s",
(node, duplicate))
node["name"] = attr_name
return node
def _get_network(self, attr_name):
if not isinstance(attr_name, collections.Mapping):
network = self.networks.get(attr_name)
else:
# Don't generalize too much Just support vld_id
vld_id = attr_name.get('vld_id', {})
# for standalone context networks are dicts
iter1 = (n for n in self.networks.values()
if n.get('vld_id') == vld_id)
network = next(iter1, None)
if network is None:
return None
result = {
# name is required
"name": network["name"],
"vld_id": network.get("vld_id"),
"segmentation_id": network.get("segmentation_id"),
"network_type": network.get("network_type"),
"physical_network": network.get("physical_network"),
}
return result
def configure_nics_for_sriov(self):
vf_cmd = "ip link set {0} vf 0 mac {1}"
for ports in self.networks.values():
host_driver = ports.get('driver')
if host_driver not in self.drivers:
self.connection.execute("rmmod %svf" % host_driver)
self.drivers.append(host_driver)
# enable VFs for given...
build_vfs = "echo 1 > /sys/bus/pci/devices/{0}/sriov_numvfs"
self.connection.execute(build_vfs.format(ports.get('phy_port')))
# configure VFs...
mac = StandaloneContextHelper.get_mac_address()
interface = ports.get('interface')
if interface is not None:
self.connection.execute(vf_cmd.format(interface, mac))
vf_pci = self._get_vf_data(ports.get('phy_port'), mac, interface)
ports.update({'vf_pci': vf_pci, 'mac': mac})
LOG.info('Ports %s', self.networks)
def _enable_interfaces(self, index, idx, vfs, cfg):
vf_spoofchk = "ip link set {0} vf 0 spoofchk off"
vf = self.networks[vfs[0]]
vpci = PciAddress(vf['vpci'].strip())
# Generate the vpci for the interfaces
slot = index + idx + 10
vf['vpci'] = \
"{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
Libvirt.add_sriov_interfaces(vf['vpci'], vf['vf_pci']['vf_pci'],
vf['mac'], str(cfg))
self.connection.execute("ifconfig %s up" % vf['interface'])
self.connection.execute(vf_spoofchk.format(vf['interface']))
def setup_sriov_context(self):
nodes = []
# 1 : modprobe host_driver with num_vfs
self.configure_nics_for_sriov()
for index, (key, vnf) in enumerate(OrderedDict(self.servers).items()):
cfg = '/tmp/vm_sriov_%s.xml' % str(index)
vm_name = "vm_%s" % str(index)
# 1. Check and delete VM if already exists
Libvirt.check_if_vm_exists_and_delete(vm_name, self.connection)
_, mac = Libvirt.build_vm_xml(self.connection, self.vm_flavor, cfg,
vm_name, index)
# 2: Cleanup already available VMs
for idx, (vkey, vfs) in enumerate(
OrderedDict(vnf["network_ports"]).items()):
if vkey == "mgmt":
continue
self._enable_interfaces(index, idx, vfs, cfg)
# copy xml to target...
self.connection.put(cfg, cfg)
# NOTE: launch through libvirt
LOG.info("virsh create ...")
Libvirt.virsh_create_vm(self.connection, cfg)
self.vm_names.append(vm_name)
# build vnf node details
nodes.append(
self.vnf_node.generate_vnf_instance(self.vm_flavor,
self.networks,
self.host_mgmt.get('ip'),
key, vnf, mac))
return nodes
def _get_vf_data(self, value, vfmac, pfif):
vf_data = {"mac": vfmac, "pf_if": pfif}
vfs = StandaloneContextHelper.get_virtual_devices(
self.connection, value)
for k, v in vfs.items():
m = PciAddress(k.strip())
m1 = PciAddress(value.strip())
if m.bus == m1.bus:
vf_data.update({"vf_pci": str(v)})
break
return vf_data
class StandaloneContextHelperTestCase(unittest.TestCase):
NODE_SAMPLE = "nodes_sample.yaml"
NODE_SRIOV_SAMPLE = "nodes_sriov_sample.yaml"
NETWORKS = {
'mgmt': {
'cidr': '152.16.100.10/24'
},
'private_0': {
'phy_port': "0000:05:00.0",
'vpci': "0000:00:07.0",
'cidr': '152.16.100.10/24',
'gateway_ip': '152.16.100.20'
},
'public_0': {
'phy_port': "0000:05:00.1",
'vpci': "0000:00:08.0",
'cidr': '152.16.40.10/24',
'gateway_ip': '152.16.100.20'
}
}
def setUp(self):
self.helper = StandaloneContextHelper()
def test___init__(self):
self.assertIsNone(self.helper.file_path)
def test_install_req_libs(self):
with mock.patch("yardstick.ssh.SSH") as ssh:
ssh_mock = mock.Mock(autospec=ssh.SSH)
ssh_mock.execute = \
mock.Mock(return_value=(1, "a", ""))
ssh.return_value = ssh_mock
status = StandaloneContextHelper.install_req_libs(ssh_mock)
self.assertIsNone(status)
def test_get_kernel_module(self):
with mock.patch("yardstick.ssh.SSH") as ssh:
ssh_mock = mock.Mock(autospec=ssh.SSH)
ssh_mock.execute = \
mock.Mock(return_value=(1, "i40e", ""))
ssh.return_value = ssh_mock
status = StandaloneContextHelper.get_kernel_module(
ssh_mock, "05:00.0", None)
self.assertEqual(status, "i40e")
@mock.patch(
'yardstick.benchmark.contexts.standalone.model.StandaloneContextHelper.get_kernel_module'
)
def test_get_nic_details(self, mock_get_kernel_module):
with mock.patch("yardstick.ssh.SSH") as ssh:
ssh_mock = mock.Mock(autospec=ssh.SSH)
ssh_mock.execute = \
mock.Mock(return_value=(1, "i40e ixgbe", ""))
ssh.return_value = ssh_mock
mock_get_kernel_module.return_value = "i40e"
status = StandaloneContextHelper.get_nic_details(
ssh_mock, self.NETWORKS, "dpdk-devbind.py")
self.assertIsNotNone(status)
def test_get_virtual_devices(self):
pattern = "PCI_SLOT_NAME=0000:05:00.0"
with mock.patch("yardstick.ssh.SSH") as ssh:
ssh_mock = mock.Mock(autospec=ssh.SSH)
ssh_mock.execute = \
mock.Mock(return_value=(1, pattern, ""))
ssh.return_value = ssh_mock
status = StandaloneContextHelper.get_virtual_devices(
ssh_mock, "0000:00:05.0")
self.assertIsNotNone(status)
def _get_file_abspath(self, filename):
curr_path = os.path.dirname(os.path.abspath(__file__))
file_path = os.path.join(curr_path, filename)
return file_path
def test_read_config_file(self):
self.helper.file_path = self._get_file_abspath(self.NODE_SAMPLE)
status = self.helper.read_config_file()
self.assertIsNotNone(status)
def test_parse_pod_file(self):
self.helper.file_path = self._get_file_abspath("dummy")
self.assertRaises(IOError, self.helper.parse_pod_file,
self.helper.file_path)
self.helper.file_path = self._get_file_abspath(self.NODE_SAMPLE)
self.assertRaises(TypeError, self.helper.parse_pod_file,
self.helper.file_path)
self.helper.file_path = self._get_file_abspath(self.NODE_SRIOV_SAMPLE)
self.assertIsNotNone(self.helper.parse_pod_file(self.helper.file_path))
def test_get_mac_address(self):
status = StandaloneContextHelper.get_mac_address()
self.assertIsNotNone(status)
@mock.patch('yardstick.ssh.SSH')
def test_get_mgmt_ip(self, mock_ssh):
with mock.patch("yardstick.ssh.SSH") as ssh:
ssh_mock = mock.Mock(autospec=ssh.SSH)
ssh_mock.execute = \
mock.Mock(return_value=(1, "1.2.3.4 00:00:00:00:00:01", ""))
ssh.return_value = ssh_mock
status = StandaloneContextHelper.get_mgmt_ip(ssh_mock,
"00:00:00:00:00:01",
"1.1.1.1/24", {})
self.assertIsNotNone(status)
@mock.patch('yardstick.ssh.SSH')
def test_get_mgmt_ip_no(self, mock_ssh):
with mock.patch("yardstick.ssh.SSH") as ssh:
ssh_mock = mock.Mock(autospec=ssh.SSH)
ssh_mock.execute = \
mock.Mock(return_value=(1, "", ""))
ssh.return_value = ssh_mock
model.WAIT_FOR_BOOT = 0
status = StandaloneContextHelper.get_mgmt_ip(ssh_mock, "99",
"1.1.1.1/24", {})
self.assertIsNone(status)