def __init__(self, fabrics=None):
self.acl_handler = EthtoolAclHandler()
self.eswitches = {}
self.pci_utils = pci_utils.pciUtils()
self.rm = ResourceManager()
self.devices = {
constants.VIF_TYPE_DIRECT: set(),
constants.VIF_TYPE_HOSTDEV: set()
}
if cfg.CONF.OF.start_of_agent:
self.of_handler = OfHandler()
if fabrics:
self.add_fabrics(fabrics)
def __init__(self, fabrics=None):
self.eswitches = {}
self.pci_utils = pci_utils.pciUtils()
self.rm = ResourceManager()
self.devices = set()
if fabrics:
self.add_fabrics(fabrics)
def __init__(self, fabrics=None):
self.acl_handler = EthtoolAclHandler()
self.eswitches = {}
self.pci_utils = pci_utils.pciUtils()
self.rm = ResourceManager()
self.devices = {
constants.VIF_TYPE_DIRECT: set(),
constants.VIF_TYPE_HOSTDEV: set()
}
if cfg.CONF.OF.start_of_agent:
self.of_handler = OfHandler()
if fabrics:
self.add_fabrics(fabrics)
class eSwitchHandler(object):
def __init__(self, fabrics=None):
self.eswitches = {}
self.pci_utils = pci_utils.pciUtils()
self.rm = ResourceManager()
self.devices = set()
if fabrics:
self.add_fabrics(fabrics)
def add_fabrics(self, fabrics):
res_fabrics = []
for fabric, pf in fabrics:
fabric_type = None
if pf in ("autoib", "autoeth"):
fabric_type = pf.strip("auto")
pf = self.pci_utils.get_auto_pf(fabric_type)
else:
fabric_type = self.pci_utils.get_interface_type(pf)
verify_vendor_pf = self.pci_utils.verify_vendor_pf(pf, constants.VENDOR)
if (
not verify_vendor_pf
or not self.pci_utils.is_sriov_pf(pf)
or not self.pci_utils.is_ifc_module(pf, fabric_type)
):
LOG.error(
"PF %s must have Mellanox Vendor ID" ",SR-IOV and driver module " "enabled. Terminating!" % pf
)
sys.exit(1)
if fabric_type:
self.eswitches[fabric] = eswitch_db.eSwitchDB()
self._add_fabric(fabric, pf, fabric_type)
res_fabrics.append((fabric, pf, fabric_type))
else:
LOG.info("No fabric type for PF:%s.Terminating!" % pf)
sys.exit(1)
self.sync_devices()
def sync_devices(self):
devices, vm_ids = self.rm.scan_attached_devices()
added_devs = {}
removed_devs = {}
added_devs = set(devices) - self.devices
removed_devs = self.devices - set(devices)
self._treat_added_devices(added_devs, vm_ids)
self._treat_removed_devices(removed_devs)
self.devices = set(devices)
def _add_fabric(self, fabric, pf, fabric_type):
self.rm.add_fabric(fabric, pf, fabric_type)
self._config_port_up(pf)
fabric_details = self.rm.get_fabric_details(fabric)
for vf in fabric_details["vfs"]:
self.eswitches[fabric].create_port(vf, constants.VIF_TYPE_HOSTDEV)
def _treat_added_devices(self, devices, vm_ids):
for device in devices:
dev, mac, fabric = device
if fabric:
self.eswitches[fabric].attach_vnic(port_name=dev, device_id=vm_ids[dev], vnic_mac=mac)
if self.eswitches[fabric].vnic_exists(mac):
self.eswitches[fabric].plug_nic(port_name=dev)
else:
LOG.info("No Fabric defined for device %s", dev)
def _treat_removed_devices(self, devices):
for dev, mac in devices:
fabric = self.rm.get_fabric_for_dev(dev)
if fabric:
self.eswitches[fabric].detach_vnic(vnic_mac=mac)
else:
LOG.info("No Fabric defined for device %s", dev)
def set_fabric_mapping(self, fabric, interface):
dev = self.rm.get_fabric_for_dev(interface)
if not dev:
fabrics = [(fabric, interface)]
self.add_fabrics(fabrics)
dev = interface
return (fabric, interface)
def get_vnics(self, fabrics):
vnics = {}
for fabric in fabrics:
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
vnics_for_eswitch = eswitch.get_attached_vnics()
vnics.update(vnics_for_eswitch)
else:
LOG.error("No eSwitch found for Fabric %s", fabric)
continue
LOG.info("vnics are %s", vnics)
return vnics
def create_port(self, fabric, vnic_type, device_id, vnic_mac, pci_slot):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
try:
if eswitch.attach_vnic(pci_slot, device_id, vnic_mac, pci_slot):
self._config_vf_mac_address(fabric, pci_slot, vnic_mac)
else:
raise exceptions.MlxException("Failed to attach vnic")
except (RuntimeError, exceptions.MlxException):
LOG.error("Create port operation failed ")
pci_slot = None
else:
LOG.error("No eSwitch found for Fabric %s", fabric)
return pci_slot
def plug_nic(self, fabric, device_id, vnic_mac, pci_slot):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
eswitch.port_table[pci_slot]["vnic"] = vnic_mac
eswitch.port_policy.update({vnic_mac: {"vlan": None, "dev": pci_slot, "device_id": device_id}})
self._config_vf_mac_address(fabric, pci_slot, vnic_mac)
eswitch.plug_nic(pci_slot)
else:
LOG.error("No eSwitch found for Fabric %s", fabric)
return pci_slot
def delete_port(self, fabric, vnic_mac):
dev = None
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev = eswitch.detach_vnic(vnic_mac)
if dev:
self._config_vf_mac_address(fabric, dev)
else:
LOG.warning("No eSwitch found for Fabric %s", fabric)
return dev
def port_release(self, fabric, vnic_mac):
ret = None
eswitch = self._get_vswitch_for_fabric(fabric)
dev = eswitch.get_dev_for_vnic(vnic_mac)
if dev:
if eswitch.get_port_state(dev) == constants.VPORT_STATE_UNPLUGGED:
ret = self.set_vlan(fabric, vnic_mac, constants.UNTAGGED_VLAN_ID)
self.port_down(fabric, vnic_mac)
eswitch = self._get_vswitch_for_fabric(fabric)
eswitch.port_release(vnic_mac)
return ret
def port_up(self, fabric, vnic_mac):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev = eswitch.get_dev_for_vnic(vnic_mac)
if not dev:
LOG.info("No device for MAC %s", vnic_mac)
def port_down(self, fabric, vnic_mac):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev = eswitch.get_dev_for_vnic(vnic_mac)
if dev:
LOG.info("IB port for MAC %s doen't support " "port down", vnic_mac)
else:
LOG.info("No device for MAC %s", vnic_mac)
def set_vlan(self, fabric, vnic_mac, vlan):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
eswitch.set_vlan(vnic_mac, vlan)
dev = eswitch.get_dev_for_vnic(vnic_mac)
state = eswitch.get_port_state(dev)
if dev:
if state in (constants.VPORT_STATE_ATTACHED, constants.VPORT_STATE_UNPLUGGED):
if eswitch.get_port_table()[dev]["alias"]:
dev = eswitch.get_port_table()[dev]["alias"]
try:
self._config_vlan_ib(fabric, dev, vlan)
return True
except RuntimeError:
LOG.error("Set VLAN operation failed")
return False
def get_eswitch_tables(self, fabrics):
tables = {}
for fabric in fabrics:
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
tables[fabric] = {
"port_table": eswitch.get_port_table_matrix(),
"port_policy": eswitch.get_port_policy_matrix(),
}
else:
LOG.info("Get eswitch tables: No eswitch %s" % fabric)
return tables
def _get_vswitch_for_fabric(self, fabric):
if fabric in self.eswitches:
return self.eswitches[fabric]
else:
return
def _config_vf_pkey(self, ppkey_idx, pkey_idx, pf_mlx_dev, vf_pci_id, hca_port):
path = constants.PKEY_INDEX_PATH % (pf_mlx_dev, vf_pci_id, hca_port, pkey_idx)
cmd = ["ebrctl", "write-sys", path, ppkey_idx]
execute(cmd, root_helper=None)
def _get_guid_idx(self, pf_mlx_dev, dev, hca_port):
path = constants.GUID_INDEX_PATH % (pf_mlx_dev, dev, hca_port)
with open(path) as fd:
idx = fd.readline().strip()
return idx
def _get_guid_from_mac(self, mac, device_type):
guid = None
if device_type == constants.CX3_VF_DEVICE_TYPE:
if mac is None:
guid = constants.INVALID_GUID_CX3
else:
mac = mac.replace(":", "")
prefix = mac[:6]
suffix = mac[6:]
guid = prefix + "0000" + suffix
elif device_type == constants.CX4_VF_DEVICE_TYPE:
if mac is None:
guid = constants.INVALID_GUID_CX4
else:
prefix = mac[:9]
suffix = mac[9:]
guid = prefix + "00:00:" + suffix
return guid
def _config_vf_mac_address(self, fabric, dev, vnic_mac=None):
fabric_details = self.rm.get_fabric_details(fabric)
vf_device_type = fabric_details["vfs"][dev]["vf_device_type"]
vguid = self._get_guid_from_mac(vnic_mac, vf_device_type)
if vf_device_type == constants.CX3_VF_DEVICE_TYPE:
self._config_vf_mac_address_cx3(vguid, dev, fabric_details)
elif vf_device_type == constants.CX4_VF_DEVICE_TYPE:
self._config_vf_mac_address_cx4(vguid, dev, fabric_details)
else:
LOG.error("Unsupported vf device type: %s ", vf_device_type)
def _config_vf_mac_address_cx3(self, vguid, dev, fabric_details):
hca_port = fabric_details["hca_port"]
pf_mlx_dev = fabric_details["pf_mlx_dev"]
self._config_vf_pkey(INVALID_PKEY, DEFAULT_PKEY_IDX, pf_mlx_dev, dev, hca_port)
guid_idx = self._get_guid_idx(pf_mlx_dev, dev, hca_port)
path = constants.ADMIN_GUID_PATH % (pf_mlx_dev, hca_port, guid_idx)
cmd = ["ebrctl", "write-sys", path, vguid]
execute(cmd, root_helper=None)
ppkey_idx = self._get_pkey_idx(int(DEFAULT_PKEY, 16), pf_mlx_dev, hca_port)
if ppkey_idx >= 0:
self._config_vf_pkey(ppkey_idx, PARTIAL_PKEY_IDX, pf_mlx_dev, dev, hca_port)
else:
LOG.error("Can't find partial management pkey" "for %s:%s" % (pf_mlx_dev, dev))
def _config_vf_mac_address_cx4(self, vguid, dev, fabric_details):
vf_num = fabric_details["vfs"][dev]["vf_num"]
pf_mlx_dev = fabric_details["pf_mlx_dev"]
guid_node = constants.CX4_GUID_NODE_PATH % {"module": pf_mlx_dev, "vf_num": vf_num}
guid_port = constants.CX4_GUID_PORT_PATH % {"module": pf_mlx_dev, "vf_num": vf_num}
guid_poliy = constants.CX4_GUID_POLICY_PATH % {"module": pf_mlx_dev, "vf_num": vf_num}
for path in (guid_node, guid_port):
cmd = ["ebrctl", "write-sys", path, vguid]
execute(cmd, root_helper=None)
cmd = ["ebrctl", "write-sys", guid_poliy, "Up\n"]
execute(cmd, root_helper=None)
def _config_vlan_ib(self, fabric, dev, vlan):
fabric_details = self.rm.get_fabric_details(fabric)
hca_port = fabric_details["hca_port"]
pf_mlx_dev = fabric_details["pf_mlx_dev"]
vf_device_type = fabric_details["vfs"][dev]["vf_device_type"]
if vf_device_type == constants.CX3_VF_DEVICE_TYPE:
self._config_vlan_ib_cx3(vlan, pf_mlx_dev, dev, hca_port)
elif vf_device_type == constants.CX4_VF_DEVICE_TYPE:
pass
else:
LOG.error("Unsupported vf device type: %s ", vf_device_type)
def _config_vlan_ib_cx3(self, vlan, pf_mlx_dev, dev, hca_port):
ppkey_idx = self._get_pkey_idx(str(vlan), pf_mlx_dev, hca_port)
if ppkey_idx:
self._config_vf_pkey(ppkey_idx, DEFAULT_PKEY_IDX, pf_mlx_dev, dev, hca_port)
def _get_pkey_idx(self, vlan, pf_mlx_dev, hca_port):
PKEYS_PATH = "/sys/class/infiniband/%s/ports/%s/pkeys/*"
paths = PKEYS_PATH % (pf_mlx_dev, hca_port)
for path in glob.glob(paths):
fd = open(path)
pkey = fd.readline()
fd.close()
# the MSB in pkey is the membership bit ( 0 - partial, 1 - full)
# the other 15 bit are the number of the pkey
# so we want to remove the 16th bit when compare pkey file
# to the vlan (pkey) we are looking for
is_match = int(pkey, 16) & DEFAULT_MASK == int(vlan) & DEFAULT_MASK
if is_match:
return path.split("/")[-1]
return None
def _config_port_up(self, dev):
cmd = ["ip", "link", "set", dev, "up"]
execute(cmd, root_helper=None)
class eSwitchHandler(object):
def __init__(self, fabrics=None):
self.acl_handler = EthtoolAclHandler()
self.eswitches = {}
self.pci_utils = pci_utils.pciUtils()
self.rm = ResourceManager()
self.devices = {
constants.VIF_TYPE_DIRECT: set(),
constants.VIF_TYPE_HOSTDEV: set()
}
if cfg.CONF.OF.start_of_agent:
self.of_handler = OfHandler()
if fabrics:
self.add_fabrics(fabrics)
def add_fabrics(self, fabrics):
res_fabrics = []
for fabric, pf in fabrics:
fabric_type = None
if pf in ('autoib', 'autoeth'):
fabric_type = pf.strip('auto')
pf = self.pci_utils.get_auto_pf(fabric_type)
else:
fabric_type = self.pci_utils.get_interface_type(pf)
if (not self.pci_utils.verify_vendor_pf(pf, constants.VENDOR)
or not self.pci_utils.is_sriov_pf(pf)
or not self.pci_utils.is_ifc_module(pf, fabric_type)):
LOG.error(
"PF %s must have Mellanox Vendor ID"
",SR-IOV and driver module enabled.Terminating!" % pf)
sys.exit(1)
if fabric_type:
self.eswitches[fabric] = eswitch_db.eSwitchDB()
self._add_fabric(fabric, pf, fabric_type)
res_fabrics.append((fabric, pf, fabric_type))
else:
LOG.debug("No fabric type for PF:%s.Terminating!" % pf)
sys.exit(1)
self.sync_devices()
if hasattr(self, 'of_handler'):
self.of_handler.add_fabrics(res_fabrics)
def sync_devices(self):
devices, vm_ids = self.rm.scan_attached_devices()
added_devs = {}
removed_devs = {}
for type, devs in devices.items():
added_devs[type] = set(devs) - self.devices[type]
removed_devs[type] = self.devices[type] - set(devs)
self._treat_added_devices(added_devs[type], type, vm_ids)
self._treat_removed_devices(removed_devs[type], type)
self.devices[type] = set(devices[type])
def _add_fabric(self, fabric, pf, fabric_type):
self.rm.add_fabric(fabric, pf, fabric_type)
self._config_port_up(pf)
vfs = self.rm.get_free_vfs(fabric)
eths = self.rm.get_free_eths(fabric)
for vf in vfs:
self.eswitches[fabric].create_port(vf, constants.VIF_TYPE_HOSTDEV)
for eth in eths:
self.eswitches[fabric].create_port(eth, constants.VIF_TYPE_DIRECT)
def _treat_added_devices(self, devices, dev_type, vm_ids):
for dev, mac, fabric in devices:
if fabric:
self.rm.allocate_device(fabric, dev_type=dev_type, dev=dev)
self.eswitches[fabric].attach_vnic(port_name=dev,
device_id=vm_ids[dev],
vnic_mac=mac)
if self.eswitches[fabric].vnic_exists(mac):
self.eswitches[fabric].plug_nic(port_name=dev)
else:
LOG.debug("No Fabric defined for device %s", dev)
def _treat_removed_devices(self, devices, dev_type):
for dev, mac in devices:
fabric = self.rm.get_fabric_for_dev(dev)
if fabric:
self.rm.deallocate_device(fabric, dev_type=dev_type, dev=dev)
self.eswitches[fabric].detach_vnic(vnic_mac=mac)
else:
LOG.debug("No Fabric defined for device %s", dev)
#-------------------------------------------------
# requests handling
#-------------------------------------------------
def set_fabric_mapping(self, fabric, interface):
dev = self.rm.get_fabric_for_dev(interface)
if not dev:
fabrics = [(fabric, interface)]
self.add_fabrics(fabrics)
dev = interface
return (fabric, interface)
def get_vnics(self, fabrics):
vnics = {}
for fabric in fabrics:
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
vnics_for_eswitch = eswitch.get_attached_vnics()
vnics.update(vnics_for_eswitch)
else:
LOG.error("No eSwitch found for Fabric %s", fabric)
continue
LOG.debug("vnics are %s", vnics)
return vnics
def create_port(self, fabric, vnic_type, device_id, vnic_mac, dev_name):
dev = None
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev = eswitch.get_dev_for_vnic(vnic_mac)
if not dev:
try:
dev = self.rm.allocate_device(fabric, vnic_type)
if eswitch.attach_vnic(dev, device_id, vnic_mac, dev_name):
pf, vf_index = self._get_device_pf_vf(
fabric, vnic_type, dev)
if vnic_type == constants.VIF_TYPE_HOSTDEV:
self._config_vf_mac_address(
fabric, dev, vf_index, vnic_mac)
else:
self._set_devname(dev_name, dev)
acl_rules = eswitch.get_acls_for_vnic(vnic_mac)
self.acl_handler.set_acl_rules(pf, acl_rules)
else:
raise MlxException('Failed to attach vnic')
except (RuntimeError, MlxException):
LOG.error('Create port operation failed ')
self.rm.deallocate_device(fabric, vnic_type, dev)
dev = None
else:
LOG.error("No eSwitch found for Fabric %s", fabric)
return dev
def plug_nic(self, fabric, device_id, vnic_mac):
dev = None
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev = eswitch.get_dev_for_vnic(vnic_mac)
if not dev:
#check for port_table entry with device_id and INVALID_MAC
for key, data in eswitch.port_table.items():
if data['device_id'] == device_id and data[
'vnic'] == constants.INVALID_MAC:
dev = key
break
else:
LOG.error(
"Plug NIC: Didn't find dev for MAC:%s and device_id:%s"
% (data['vnic'], device_id))
if dev:
eswitch.port_table[dev]['vnic'] = vnic_mac
eswitch.port_policy.update({
vnic_mac: {
'vlan': None,
'dev': dev,
'device_id': device_id,
'flow_ids': set([]),
'priority': 0,
}
})
pf, vf_index = self._get_device_pf_vf(
fabric, constants.VIF_TYPE_HOSTDEV, dev)
self._config_vf_mac_address(fabric, dev, vf_index,
vnic_mac)
else:
return None
eswitch.plug_nic(dev)
else:
LOG.error("No eSwitch found for Fabric %s", fabric)
return dev
def delete_port(self, fabric, vnic_mac):
"""
@note: Free Virtual function associated with vNIC MAc
"""
dev = None
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev_name = eswitch.get_dev_alias_for_vnic(vnic_mac)
dev = eswitch.detach_vnic(vnic_mac)
if dev:
dev_type = eswitch.get_dev_type(dev)
if dev_type == constants.VIF_TYPE_HOSTDEV:
pf, vf_index = self._get_device_pf_vf(
fabric, dev_type, dev)
#unset MAC to default value
self._config_vf_mac_address(fabric, dev, vf_index,
DEFAULT_MAC_ADDRESS)
if dev_name:
self._set_devname(dev, dev_name)
self.rm.deallocate_device(fabric, dev_type, dev)
else:
LOG.warning("No eSwitch found for Fabric %s", fabric)
return dev
def port_release(self, fabric, vnic_mac):
"""
@todo: handle failures
"""
ret = None
eswitch = self._get_vswitch_for_fabric(fabric)
dev = eswitch.get_dev_for_vnic(vnic_mac)
if dev:
if eswitch.get_port_state(dev) == constants.VPORT_STATE_UNPLUGGED:
ret = self.set_vlan(fabric, vnic_mac,
constants.UNTAGGED_VLAN_ID)
self.port_down(fabric, vnic_mac)
eswitch = self._get_vswitch_for_fabric(fabric)
eswitch.port_release(vnic_mac)
return ret
def port_down(self, fabric, vnic_mac):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev = eswitch.get_dev_for_vnic(vnic_mac)
if dev:
vnic_type = eswitch.get_port_type(dev)
self._config_port_down(dev, vnic_type)
else:
LOG.debug("No device for MAC %s", vnic_mac)
def set_vlan(self, fabric, vnic_mac, vlan):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
eswitch.set_vlan(vnic_mac, vlan)
dev = eswitch.get_dev_for_vnic(vnic_mac)
state = eswitch.get_port_state(dev)
if dev:
if state in (constants.VPORT_STATE_ATTACHED,
constants.VPORT_STATE_UNPLUGGED):
priority = eswitch.get_priority(vnic_mac)
vnic_type = eswitch.get_port_type(dev)
if eswitch.get_port_table()[dev]['alias']:
dev = eswitch.get_port_table()[dev]['alias']
pf, vf_index = self._get_device_pf_vf(
fabric, vnic_type, dev)
if pf and vf_index is not None:
try:
if vnic_type == constants.VIF_TYPE_DIRECT:
self._config_vlan_priority_direct(
pf, vf_index, dev, vlan, priority)
else:
self._config_vlan_priority_hostdev(
fabric, pf, vf_index, dev, vlan, priority)
return True
except RuntimeError:
LOG.error('Set VLAN operation failed')
else:
LOG.error(
'Invalid VF/PF index for device %s,PF-%s,VF Index - %s',
dev, pf, vf_index)
return False
def set_priority(self, fabric, vnic_mac, priority):
ret = False
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
vlan = eswitch.set_priority(vnic_mac, priority)
if vlan:
ret = self.set_vlan(fabric, vnic_mac, vlan)
return ret
def set_acl_rule(self, fabric, vnic_mac, params):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
pf = self.rm.get_fabric_pf(fabric)
flow_id, acl_rule = self.acl_handler.build_acl_rule(params)
if acl_rule:
eswitch.set_acl_rule(vnic_mac, acl_rule, flow_id)
vnic_state = eswitch.get_vnic_state(vnic_mac)
if vnic_state in (constants.VPORT_STATE_ATTACHED,
constants.VPORT_STATE_PENDING):
try:
acl_ref = self.acl_handler.set_acl_rule(pf, acl_rule)
eswitch.update_acl_rule_ref(flow_id, acl_ref)
return True
except RuntimeError:
LOG.error('Failed to set ACL rule flow_id-%s', flow_id)
return False
def delete_acl_rule(self, fabric, flow_id):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
pf = self.rm.get_fabric_pf(fabric)
(ref, vnic_mac) = eswitch.del_acl_rule(flow_id)
if ref:
try:
self.acl_handler.del_acl_rule(pf, ref)
return True
except RuntimeError:
LOG.error('Failed to delete ACL flow_id-%s', flow_id)
return False
def update_flow_id(self, fabric, old_flow_id, new_flow_id):
ret = False
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
ret = eswitch.update_flow_id(old_flow_id, new_flow_id)
return ret
def get_eswitch_tables(self, fabrics):
tables = {}
for fabric in fabrics:
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
tables[fabric] = {
'port_table': eswitch.get_port_table_matrix(),
'port_policy': eswitch.get_port_policy_matrix()
}
else:
LOG.debug("Get eswitch tables: No eswitch %s" % fabric)
return tables
def _get_vswitch_for_fabric(self, fabric):
if fabric in self.eswitches:
return self.eswitches[fabric]
else:
return
def _get_device_pf_vf(self, fabric, vnic_type, dev):
pf = self.rm.get_fabric_pf(fabric)
vf_index = self.pci_utils.get_vf_index(dev, vnic_type)
return pf, vf_index
def _config_vf_pkey(self, ppkey_idx, pkey_idx, pf_mlx_dev, vf_pci_id,
hca_port):
path = "/sys/class/infiniband/%s/iov/%s/ports/%s/pkey_idx/%s" % (
pf_mlx_dev, vf_pci_id, hca_port, pkey_idx)
cmd = ['ebrctl', 'write-sys', path, ppkey_idx]
execute(cmd, root_helper=None)
def _get_guid_idx(self, pf_mlx_dev, dev, hca_port):
path = "/sys/class/infiniband/%s/iov/%s/ports/%s/gid_idx/0" % (
pf_mlx_dev, dev, hca_port)
with open(path) as fd:
idx = fd.readline().strip()
return idx
def _get_guid_from_mac(self, mac):
if mac == DEFAULT_MAC_ADDRESS:
return constants.INVALID_GUID
mac = mac.replace(':', '')
prefix = mac[:6]
suffix = mac[6:]
guid = prefix + PADDING + suffix
return guid
def _config_vf_mac_address(self, fabric, dev, vf_index, vnic_mac):
vguid = self._get_guid_from_mac(vnic_mac)
fabric_details = self.rm.get_fabric_details(fabric)
pf = fabric_details['pf']
fabric_type = fabric_details['fabric_type']
if fabric_type == 'ib':
hca_port = fabric_details['hca_port']
pf_mlx_dev = fabric_details['pf_mlx_dev']
self._config_vf_pkey(INVALID_PKEY, DEFAULT_PKEY_IDX, pf_mlx_dev,
dev, hca_port)
guid_idx = self._get_guid_idx(pf_mlx_dev, dev, hca_port)
path = "/sys/class/infiniband/%s/iov/ports/%s/admin_guids/%s" % (
pf_mlx_dev, hca_port, guid_idx)
cmd = ['ebrctl', 'write-sys', path, vguid]
execute(cmd, root_helper=None)
ppkey_idx = self._get_pkey_idx(int(DEFAULT_PKEY, 16), pf_mlx_dev,
hca_port)
if ppkey_idx >= 0:
self._config_vf_pkey(ppkey_idx, PARTIAL_PKEY_IDX, pf_mlx_dev,
dev, hca_port)
else:
LOG.error("Can't find partial management pkey for %s:%s" %
(pf_mlx_dev, vf_index))
else:
cmd = ['ip', 'link', 'set', pf, 'vf', vf_index, 'mac', vnic_mac]
execute(cmd, root_helper=None)
def _config_vlan_priority_direct(self,
pf,
vf_index,
dev,
vlan,
priority='0'):
self._config_port_down(dev)
cmd = [
'ip', 'link', 'set', pf, 'vf', vf_index, 'vlan', vlan, 'qos',
priority
]
execute(cmd, root_helper=None)
self._config_port_up(dev)
def _config_vlan_priority_hostdev(self,
fabric,
pf,
vf_index,
dev,
vlan,
priority='0'):
fabric_details = self.rm.get_fabric_details(fabric)
fabric_type = fabric_details['fabric_type']
if fabric_type == 'ib':
self._config_vlan_ib(vlan, fabric_details, dev, vf_index)
else:
cmd = [
'ip', 'link', 'set', pf, 'vf', vf_index, 'vlan', vlan, 'qos',
priority
]
execute(cmd, root_helper=None)
def _config_vlan_ib(self, vlan, fabric_details, dev, vf_index):
hca_port = fabric_details['hca_port']
pf_mlx_dev = fabric_details['pf_mlx_dev']
ppkey_idx = self._get_pkey_idx(str(vlan), pf_mlx_dev, hca_port)
if ppkey_idx:
self._config_vf_pkey(ppkey_idx, DEFAULT_PKEY_IDX, pf_mlx_dev, dev,
hca_port)
def _get_pkey_idx(self, vlan, pf_mlx_dev, hca_port):
PKEYS_PATH = "/sys/class/infiniband/%s/ports/%s/pkeys/*"
paths = PKEYS_PATH % (pf_mlx_dev, hca_port)
for path in glob.glob(paths):
fd = open(path)
pkey = fd.readline()
fd.close()
# the MSB in pkey is the membership bit ( 0 - partial, 1 - full)
# the other 15 bit are the number of the pkey
# so we want to remove the 16th bit when compare pkey file
# to the vlan (pkey) we are looking for
is_match = int(pkey, 16) & DEFAULT_MASK == int(vlan) & DEFAULT_MASK
if is_match:
return path.split('/')[-1]
return None
def _config_port_down(self, dev, vnic_type=constants.VIF_TYPE_DIRECT):
if vnic_type == constants.VIF_TYPE_DIRECT:
cmd = ['ip', 'link', 'set', dev, 'down']
execute(cmd, root_helper=None)
def _config_port_up(self, dev):
cmd = ['ip', 'link', 'set', dev, 'up']
execute(cmd, root_helper=None)
def _set_devname(self, device_name, dev):
self._config_port_down(dev)
cmd = ['ip', 'link', 'set', dev, 'name', device_name]
execute(cmd, root_helper=None)
class eSwitchHandler(object):
def __init__(self, fabrics=None):
self.acl_handler = EthtoolAclHandler()
self.eswitches = {}
self.pci_utils = pci_utils.pciUtils()
self.rm = ResourceManager()
self.devices = {
constants.VIF_TYPE_DIRECT: set(),
constants.VIF_TYPE_HOSTDEV: set()
}
if cfg.CONF.OF.start_of_agent:
self.of_handler = OfHandler()
if fabrics:
self.add_fabrics(fabrics)
def add_fabrics(self, fabrics):
res_fabrics = []
for fabric, pf in fabrics:
fabric_type = None
if pf in ('autoib', 'autoeth'):
fabric_type = pf.strip('auto')
pf = self.pci_utils.get_auto_pf(fabric_type)
else:
fabric_type = self.pci_utils.get_interface_type(pf)
if (not self.pci_utils.verify_vendor_pf(pf, constants.VENDOR) or
not self.pci_utils.is_sriov_pf(pf) or
not self.pci_utils.is_ifc_module(pf, fabric_type)):
LOG.error("PF %s must have Mellanox Vendor ID"
",SR-IOV and driver module enabled.Terminating!" % pf)
sys.exit(1)
if fabric_type:
self.eswitches[fabric] = eswitch_db.eSwitchDB()
self._add_fabric(fabric, pf, fabric_type)
res_fabrics.append((fabric, pf, fabric_type))
else:
LOG.debug("No fabric type for PF:%s.Terminating!" % pf)
sys.exit(1)
self.sync_devices()
if hasattr(self, 'of_handler'):
self.of_handler.add_fabrics(res_fabrics)
def sync_devices(self):
devices, vm_ids = self.rm.scan_attached_devices()
added_devs = {}
removed_devs = {}
for type, devs in devices.items():
added_devs[type] = set(devs) - self.devices[type]
removed_devs[type] = self.devices[type] - set(devs)
self._treat_added_devices(added_devs[type], type, vm_ids)
self._treat_removed_devices(removed_devs[type], type)
self.devices[type] = set(devices[type])
def _add_fabric(self, fabric, pf, fabric_type):
self.rm.add_fabric(fabric, pf, fabric_type)
self._config_port_up(pf)
vfs = self.rm.get_free_vfs(fabric)
eths = self.rm.get_free_eths(fabric)
for vf in vfs:
self.eswitches[fabric].create_port(vf, constants.VIF_TYPE_HOSTDEV)
for eth in eths:
self.eswitches[fabric].create_port(eth, constants.VIF_TYPE_DIRECT)
def _treat_added_devices(self, devices, dev_type, vm_ids):
for dev, mac, fabric in devices:
if fabric:
self.rm.allocate_device(fabric, dev_type=dev_type, dev=dev)
self.eswitches[fabric].attach_vnic(port_name=dev, device_id=vm_ids[dev], vnic_mac=mac)
if self.eswitches[fabric].vnic_exists(mac):
self.eswitches[fabric].plug_nic(port_name=dev)
else:
LOG.debug("No Fabric defined for device %s", dev)
def _treat_removed_devices(self, devices, dev_type):
for dev, mac in devices:
fabric = self.rm.get_fabric_for_dev(dev)
if fabric:
self.rm.deallocate_device(fabric, dev_type=dev_type, dev=dev)
self.eswitches[fabric].detach_vnic(vnic_mac=mac)
else:
LOG.debug("No Fabric defined for device %s", dev)
#-------------------------------------------------
# requests handling
#-------------------------------------------------
def set_fabric_mapping(self, fabric, interface):
dev = self.rm.get_fabric_for_dev(interface)
if not dev:
fabrics = [(fabric, interface)]
self.add_fabrics(fabrics)
dev = interface
return (fabric, interface)
def get_vnics(self, fabrics):
vnics = {}
for fabric in fabrics:
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
vnics_for_eswitch = eswitch.get_attached_vnics()
vnics.update(vnics_for_eswitch)
else:
LOG.error("No eSwitch found for Fabric %s", fabric)
continue
LOG.debug("vnics are %s", vnics)
return vnics
def create_port(self, fabric, vnic_type, device_id, vnic_mac, dev_name):
dev = None
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev = eswitch.get_dev_for_vnic(vnic_mac)
if not dev:
try:
dev = self.rm.allocate_device(fabric, vnic_type)
if eswitch.attach_vnic(dev, device_id, vnic_mac, dev_name):
pf, vf_index = self._get_device_pf_vf(fabric, vnic_type, dev)
if vnic_type == constants.VIF_TYPE_HOSTDEV:
self._config_vf_mac_address(fabric, dev, vf_index, vnic_mac)
else:
self._set_devname(dev_name, dev)
acl_rules = eswitch.get_acls_for_vnic(vnic_mac)
self.acl_handler.set_acl_rules(pf, acl_rules)
else:
raise MlxException('Failed to attach vnic')
except (RuntimeError, MlxException):
LOG.error('Create port operation failed ')
self.rm.deallocate_device(fabric, vnic_type, dev)
dev = None
else:
LOG.error("No eSwitch found for Fabric %s", fabric)
return dev
def plug_nic(self, fabric, device_id, vnic_mac):
dev = None
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev = eswitch.get_dev_for_vnic(vnic_mac)
if not dev:
#check for port_table entry with device_id and INVALID_MAC
for key, data in eswitch.port_table.items():
if data['device_id'] == device_id and data['vnic'] == constants.INVALID_MAC:
dev = key
break
else:
LOG.error("Plug NIC: Didn't find dev for MAC:%s and device_id:%s" % (data['vnic'], device_id))
if dev:
eswitch.port_table[dev]['vnic'] = vnic_mac
eswitch.port_policy.update({vnic_mac: {'vlan': None,
'dev': dev,
'device_id': device_id,
'flow_ids': set([]),
'priority': 0,
}})
pf, vf_index = self._get_device_pf_vf(fabric, constants.VIF_TYPE_HOSTDEV, dev)
self._config_vf_mac_address(fabric, dev, vf_index, vnic_mac)
else:
return None
eswitch.plug_nic(dev)
else:
LOG.error("No eSwitch found for Fabric %s", fabric)
return dev
def delete_port(self, fabric, vnic_mac):
"""
@note: Free Virtual function associated with vNIC MAc
"""
dev = None
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev_name = eswitch.get_dev_alias_for_vnic(vnic_mac)
dev = eswitch.detach_vnic(vnic_mac)
if dev:
dev_type = eswitch.get_dev_type(dev)
if dev_type == constants.VIF_TYPE_HOSTDEV:
pf, vf_index = self._get_device_pf_vf(fabric, dev_type, dev)
#unset MAC to default value
self._config_vf_mac_address(fabric, dev, vf_index, DEFAULT_MAC_ADDRESS)
if dev_name:
self._set_devname(dev, dev_name)
self.rm.deallocate_device(fabric, dev_type, dev)
else:
LOG.warning("No eSwitch found for Fabric %s", fabric)
return dev
def port_release(self, fabric, vnic_mac):
"""
@todo: handle failures
"""
ret = None
eswitch = self._get_vswitch_for_fabric(fabric)
dev = eswitch.get_dev_for_vnic(vnic_mac)
if dev:
if eswitch.get_port_state(dev) == constants.VPORT_STATE_UNPLUGGED:
ret = self.set_vlan(fabric, vnic_mac, constants.UNTAGGED_VLAN_ID)
self.port_down(fabric, vnic_mac)
eswitch = self._get_vswitch_for_fabric(fabric)
eswitch.port_release(vnic_mac)
return ret
def port_down(self, fabric, vnic_mac):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
dev = eswitch.get_dev_for_vnic(vnic_mac)
if dev:
vnic_type = eswitch.get_port_type(dev)
self._config_port_down(dev, vnic_type)
else:
LOG.debug("No device for MAC %s", vnic_mac)
def set_vlan(self, fabric, vnic_mac, vlan):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
eswitch.set_vlan(vnic_mac, vlan)
dev = eswitch.get_dev_for_vnic(vnic_mac)
state = eswitch.get_port_state(dev)
if dev:
if state in (constants.VPORT_STATE_ATTACHED, constants.VPORT_STATE_UNPLUGGED):
priority = eswitch.get_priority(vnic_mac)
vnic_type = eswitch.get_port_type(dev)
if eswitch.get_port_table()[dev]['alias']:
dev = eswitch.get_port_table()[dev]['alias']
pf, vf_index = self._get_device_pf_vf(fabric, vnic_type, dev)
if pf and vf_index is not None:
try:
if vnic_type == constants.VIF_TYPE_DIRECT:
self._config_vlan_priority_direct(pf, vf_index, dev, vlan, priority)
else:
self._config_vlan_priority_hostdev(fabric, pf, vf_index, dev, vlan, priority)
return True
except RuntimeError:
LOG.error('Set VLAN operation failed')
else:
LOG.error('Invalid VF/PF index for device %s,PF-%s,VF Index - %s', dev, pf, vf_index)
return False
def set_priority(self, fabric, vnic_mac, priority):
ret = False
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
vlan = eswitch.set_priority(vnic_mac, priority)
if vlan:
ret = self.set_vlan(fabric, vnic_mac, vlan)
return ret
def set_acl_rule(self, fabric, vnic_mac, params):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
pf = self.rm.get_fabric_pf(fabric)
flow_id, acl_rule = self.acl_handler.build_acl_rule(params)
if acl_rule:
eswitch.set_acl_rule(vnic_mac, acl_rule, flow_id)
vnic_state = eswitch.get_vnic_state(vnic_mac)
if vnic_state in (constants.VPORT_STATE_ATTACHED, constants.VPORT_STATE_PENDING):
try:
acl_ref = self.acl_handler.set_acl_rule(pf, acl_rule)
eswitch.update_acl_rule_ref(flow_id, acl_ref)
return True
except RuntimeError:
LOG.error('Failed to set ACL rule flow_id-%s', flow_id)
return False
def delete_acl_rule(self, fabric, flow_id):
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
pf = self.rm.get_fabric_pf(fabric)
(ref, vnic_mac) = eswitch.del_acl_rule(flow_id)
if ref:
try:
self.acl_handler.del_acl_rule(pf, ref)
return True
except RuntimeError:
LOG.error('Failed to delete ACL flow_id-%s', flow_id)
return False
def update_flow_id(self, fabric, old_flow_id, new_flow_id):
ret = False
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
ret = eswitch.update_flow_id(old_flow_id, new_flow_id)
return ret
def get_eswitch_tables(self, fabrics):
tables = {}
for fabric in fabrics:
eswitch = self._get_vswitch_for_fabric(fabric)
if eswitch:
tables[fabric] = {'port_table': eswitch.get_port_table_matrix(),
'port_policy': eswitch.get_port_policy_matrix()}
else:
LOG.debug("Get eswitch tables: No eswitch %s" % fabric)
return tables
def _get_vswitch_for_fabric(self, fabric):
if fabric in self.eswitches:
return self.eswitches[fabric]
else:
return
def _get_device_pf_vf(self, fabric, vnic_type, dev):
pf = self.rm.get_fabric_pf(fabric)
vf_index = self.pci_utils.get_vf_index(dev, vnic_type)
return pf, vf_index
def _config_vf_pkey(self, ppkey_idx, pkey_idx, pf_mlx_dev, vf_pci_id, hca_port):
path = "/sys/class/infiniband/%s/iov/%s/ports/%s/pkey_idx/%s" % (pf_mlx_dev, vf_pci_id, hca_port, pkey_idx)
cmd = ['ebrctl', 'write-sys', path, ppkey_idx]
execute(cmd, root_helper=None)
def _get_guid_idx(self, pf_mlx_dev, dev, hca_port):
path = "/sys/class/infiniband/%s/iov/%s/ports/%s/gid_idx/0" % (pf_mlx_dev, dev, hca_port)
with open(path) as fd:
idx = fd.readline().strip()
return idx
def _get_guid_from_mac(self, mac):
if mac == DEFAULT_MAC_ADDRESS:
return constants.INVALID_GUID
mac = mac.replace(':', '')
prefix = mac[:6]
suffix = mac[6:]
guid = prefix + PADDING + suffix
return guid
def _config_vf_mac_address(self, fabric, dev, vf_index, vnic_mac):
vguid = self._get_guid_from_mac(vnic_mac)
fabric_details = self.rm.get_fabric_details(fabric)
pf = fabric_details['pf']
fabric_type = fabric_details['fabric_type']
if fabric_type == 'ib':
hca_port = fabric_details['hca_port']
pf_mlx_dev = fabric_details['pf_mlx_dev']
self._config_vf_pkey(INVALID_PKEY, DEFAULT_PKEY_IDX, pf_mlx_dev, dev, hca_port)
guid_idx = self._get_guid_idx(pf_mlx_dev, dev, hca_port)
path = "/sys/class/infiniband/%s/iov/ports/%s/admin_guids/%s" % (pf_mlx_dev, hca_port, guid_idx)
cmd = ['ebrctl', 'write-sys', path, vguid]
execute(cmd, root_helper=None)
ppkey_idx = self._get_pkey_idx(int(DEFAULT_PKEY, 16), pf_mlx_dev, hca_port)
if ppkey_idx >= 0:
self._config_vf_pkey(ppkey_idx, PARTIAL_PKEY_IDX, pf_mlx_dev, dev, hca_port)
else:
LOG.error("Can't find partial management pkey for %s:%s" % (pf_mlx_dev, vf_index))
else:
cmd = ['ip', 'link', 'set', pf, 'vf', vf_index, 'mac', vnic_mac]
execute(cmd, root_helper=None)
def _config_vlan_priority_direct(self, pf, vf_index, dev, vlan, priority='0'):
self._config_port_down(dev)
cmd = ['ip', 'link', 'set', pf, 'vf', vf_index, 'vlan', vlan, 'qos', priority]
execute(cmd, root_helper=None)
self._config_port_up(dev)
def _config_vlan_priority_hostdev(self, fabric, pf, vf_index, dev, vlan, priority='0'):
fabric_details = self.rm.get_fabric_details(fabric)
fabric_type = fabric_details['fabric_type']
if fabric_type == 'ib':
self._config_vlan_ib(vlan, fabric_details, dev, vf_index)
else:
cmd = ['ip', 'link', 'set', pf, 'vf', vf_index, 'vlan', vlan, 'qos', priority]
execute(cmd, root_helper=None)
def _config_vlan_ib(self, vlan, fabric_details, dev, vf_index):
hca_port = fabric_details['hca_port']
pf_mlx_dev = fabric_details['pf_mlx_dev']
ppkey_idx = self._get_pkey_idx(str(vlan), pf_mlx_dev, hca_port)
if ppkey_idx:
self._config_vf_pkey(ppkey_idx, DEFAULT_PKEY_IDX, pf_mlx_dev, dev, hca_port)
def _get_pkey_idx(self, vlan, pf_mlx_dev, hca_port):
PKEYS_PATH = "/sys/class/infiniband/%s/ports/%s/pkeys/*"
paths = PKEYS_PATH % (pf_mlx_dev, hca_port)
for path in glob.glob(paths):
fd = open(path)
pkey = fd.readline()
fd.close()
# the MSB in pkey is the membership bit ( 0 - partial, 1 - full)
# the other 15 bit are the number of the pkey
# so we want to remove the 16th bit when compare pkey file
# to the vlan (pkey) we are looking for
is_match = int(pkey, 16) & DEFAULT_MASK == int(vlan) & DEFAULT_MASK
if is_match:
return path.split('/')[-1]
return None
def _config_port_down(self, dev, vnic_type=constants.VIF_TYPE_DIRECT):
if vnic_type == constants.VIF_TYPE_DIRECT:
cmd = ['ip', 'link', 'set', dev, 'down']
execute(cmd, root_helper=None)
def _config_port_up(self, dev):
cmd = ['ip', 'link', 'set', dev, 'up']
execute(cmd, root_helper=None)
def _set_devname(self, device_name, dev):
self._config_port_down(dev)
cmd = ['ip', 'link', 'set', dev, 'name', device_name]
execute(cmd, root_helper=None)