def connect(self, vif, ifname, netns):
with b_base.get_ipdb() as h_ipdb:
# NOTE(vikasc): Ideally 'ifname' should be used here but instead a
# temporary name is being used while creating the device for
# container in host network namespace. This is because cni expects
# only 'eth0' as interface name and if host already has an
# interface named 'eth0', device creation will fail with 'already
# exists' error.
temp_name = vif.vif_name
# TODO(vikasc): evaluate whether we should have stevedore
# driver for getting the link device.
vm_iface_name = config.CONF.binding.link_iface
args = self._get_iface_create_args(vif)
with h_ipdb.create(ifname=temp_name,
link=h_ipdb.interfaces[vm_iface_name],
**args) as iface:
iface.net_ns_fd = utils.convert_netns(netns)
with b_base.get_ipdb(netns) as c_ipdb:
with c_ipdb.interfaces[temp_name] as iface:
iface.ifname = ifname
iface.mtu = vif.network.mtu
iface.address = str(vif.address)
iface.up()
def connect(self, vif, ifname, netns):
h_ipdb = b_base.get_ipdb()
c_ipdb = b_base.get_ipdb(netns)
# NOTE(vikasc): Ideally 'ifname' should be used here but instead a
# temporary name is being used while creating the device for container
# in host network namespace. This is because cni expects only 'eth0'
# as interface name and if host already has an interface named 'eth0',
# device creation will fail with 'already exists' error.
temp_name = vif.vif_name
# TODO(vikasc): evaluate whether we should have stevedore
# driver for getting the link device.
vm_iface_name = config.CONF.binding.link_iface
vlan_id = vif.vlan_id
with h_ipdb.create(ifname=temp_name,
link=h_ipdb.interfaces[vm_iface_name],
kind='vlan',
vlan_id=vlan_id) as iface:
iface.net_ns_fd = netns
with c_ipdb.interfaces[temp_name] as iface:
iface.ifname = ifname
iface.mtu = vif.network.mtu
iface.address = str(vif.address)
iface.up()
def connect(self, vif, ifname, netns):
physnet = vif.physnet
h_ipdb = b_base.get_ipdb()
c_ipdb = b_base.get_ipdb(netns)
pf_names = self._get_host_pf_names(physnet)
vf_name, vf_index, pf = self._get_available_vf_info(pf_names)
if not vf_name:
error_msg = "No free interfaces for pfysnet {} available".format(
physnet)
LOG.error(error_msg)
raise exceptions.CNIError(error_msg)
if vif.network.should_provide_vlan:
vlan_id = vif.network.vlan
self._set_vf_vlan(pf, vf_index, vlan_id)
with h_ipdb.interfaces[vf_name] as host_iface:
host_iface.net_ns_fd = utils.convert_netns(netns)
with c_ipdb.interfaces[vf_name] as iface:
iface.ifname = ifname
iface.address = vif.address
iface.mtu = vif.network.mtu
iface.up()
def connect(self, vif, ifname, netns):
host_ifname = vif.vif_name
with b_base.get_ipdb() as h_ipdb:
if host_ifname in h_ipdb.interfaces:
# NOTE(dulek): This most likely means that we already run
# connect for this iface and there's a leftover
# host-side vif. Let's remove it, its peer should
# get deleted automatically by the kernel.
LOG.debug(
'Found leftover host vif %s. Removing it before'
'connecting.', host_ifname)
with h_ipdb.interfaces[host_ifname] as h_iface:
h_iface.remove()
with b_base.get_ipdb(netns) as c_ipdb:
with c_ipdb.create(ifname=ifname, peer=host_ifname,
kind='veth') as c_iface:
c_iface.mtu = vif.network.mtu
c_iface.address = str(vif.address)
c_iface.up()
if netns:
with c_ipdb.interfaces[host_ifname] as h_iface:
h_iface.net_ns_pid = os.getpid()
with b_base.get_ipdb() as h_ipdb:
with h_ipdb.interfaces[host_ifname] as h_iface:
h_iface.mtu = vif.network.mtu
h_iface.up()
def connect(self, vif, ifname, netns, container_id):
host_ifname = vif.vif_name
# NOTE(dulek): Check if we already run connect for this iface and if
# there's a leftover host-side vif. If so we need to
# remove it, its peer should get deleted automatically by
# the kernel.
with b_base.get_ipdb() as h_ipdb:
self._remove_ifaces(h_ipdb, (host_ifname, ))
if vif.network.mtu:
interface_mtu = vif.network.mtu
else:
LOG.info(
"Default mtu %(mtu)s is used for interface, "
"for mtu of network if configured with 0",
{"mtu": CONF.neutron_defaults.network_device_mtu})
interface_mtu = CONF.neutron_defaults.network_device_mtu
with b_base.get_ipdb(netns) as c_ipdb:
with c_ipdb.create(ifname=ifname, peer=host_ifname,
kind='veth') as c_iface:
c_iface.mtu = interface_mtu
c_iface.address = str(vif.address)
c_iface.up()
if netns:
with c_ipdb.interfaces[host_ifname] as h_iface:
h_iface.net_ns_pid = os.getpid()
with b_base.get_ipdb() as h_ipdb:
with h_ipdb.interfaces[host_ifname] as h_iface:
h_iface.mtu = interface_mtu
h_iface.up()
def connect(self, vif, ifname, netns, container_id):
physnet = vif.physnet
pf_names = self._get_host_pf_names(physnet)
vf_name, vf_index, pf, pci_info = self._get_available_vf_info(pf_names)
if not vf_name:
raise exceptions.CNIError(
"No free interfaces for physnet {} available".format(physnet))
LOG.debug("Connect {} as {} (port_id={}) in container_id={}".format(
vf_name, ifname, vif.id, container_id))
if vif.network.should_provide_vlan:
vlan_id = vif.network.vlan
self._set_vf_vlan(pf, vf_index, vlan_id)
self._set_vf_mac(pf, vf_index, vif.address)
with b_base.get_ipdb() as h_ipdb, b_base.get_ipdb(netns) as c_ipdb:
with h_ipdb.interfaces[vf_name] as host_iface:
host_iface.net_ns_fd = utils.convert_netns(netns)
with c_ipdb.interfaces[vf_name] as iface:
iface.ifname = ifname
iface.mtu = vif.network.mtu
iface.up()
self._save_pci_info(vif.id, pci_info)
def connect(self, vif, ifname, netns, container_id):
pci = self._choose_pci(vif, ifname, netns)
vf_name, vf_index, pf, pci_info = self._get_vf_info(pci)
LOG.debug("Connect {} as {} (port_id={}) in container_id={}".format(
vf_name, ifname, vif.id, container_id))
if vif.network.should_provide_vlan:
vlan_id = vif.network.vlan
self._set_vf_vlan(pf, vf_index, vlan_id)
self._set_vf_mac(pf, vf_index, vif.address)
with b_base.get_ipdb() as h_ipdb, b_base.get_ipdb(netns) as c_ipdb:
with h_ipdb.interfaces[vf_name] as host_iface:
host_iface.net_ns_fd = utils.convert_netns(netns)
with c_ipdb.interfaces[vf_name] as iface:
iface.ifname = ifname
iface.mtu = vif.network.mtu
iface.up()
pod_link = vif.pod_link
self._annotate_device(pod_link, pci)
self._save_pci_info(vif.id, pci_info)
def connect(self, vif, ifname, netns, container_id):
# NOTE(vikasc): Ideally 'ifname' should be used here but instead a
# temporary name is being used while creating the device for
# container in host network namespace. This is because cni expects
# only 'eth0' as interface name and if host already has an
# interface named 'eth0', device creation will fail with 'already
# exists' error.
temp_name = vif.vif_name
# First let's take a peek into the pod namespace and try to remove any
# leftover interface in case we got restarted before CNI returned to
# kubelet.
with b_base.get_ipdb(netns) as c_ipdb:
self._remove_ifaces(c_ipdb, (temp_name, ifname), netns)
# We might also have leftover interface in the host netns, let's try to
# remove it too. This is outside of the main host's IPDB context
# manager to make sure removal is commited before starting next
# transaction.
with b_base.get_ipdb() as h_ipdb:
self._remove_ifaces(h_ipdb, (temp_name, ))
with b_base.get_ipdb() as h_ipdb:
# TODO(vikasc): evaluate whether we should have stevedore
# driver for getting the link device.
vm_iface_name = self._detect_iface_name(h_ipdb)
mtu = h_ipdb.interfaces[vm_iface_name].mtu
if mtu < vif.network.mtu:
# NOTE(dulek): This might happen if Neutron and DHCP agent
# have different MTU settings. See
# https://bugs.launchpad.net/kuryr-kubernetes/+bug/1863212
raise exceptions.CNIBindingFailure(
f'MTU of interface {vm_iface_name} ({mtu}) is smaller '
f'than MTU of pod network {vif.network.id} '
f'({vif.network.mtu}). Please make sure pod network '
f'has the same or smaller MTU as node (VM) network.')
args = self._get_iface_create_args(vif)
with h_ipdb.create(ifname=temp_name,
link=h_ipdb.interfaces[vm_iface_name],
**args) as iface:
iface.net_ns_fd = utils.convert_netns(netns)
with b_base.get_ipdb(netns) as c_ipdb:
with c_ipdb.interfaces[temp_name] as iface:
iface.ifname = ifname
iface.mtu = vif.network.mtu
iface.address = str(vif.address)
iface.up()
def _move_to_netns(self, ifname, netns, vif, vf_name, vf_index, pf):
if vf_index and pf:
if vif.network.should_provide_vlan:
vlan_id = vif.network.vlan
self._set_vf_vlan(pf, vf_index, vlan_id)
self._set_vf_mac(pf, vf_index, vif.address)
with b_base.get_ipdb() as h_ipdb, b_base.get_ipdb(netns) as c_ipdb:
with h_ipdb.interfaces[vf_name] as host_iface:
host_iface.net_ns_fd = utils.convert_netns(netns)
with c_ipdb.interfaces[vf_name] as iface:
iface.ifname = ifname
iface.mtu = vif.network.mtu
iface.up()
def connect(self, vif, ifname, netns, container_id):
super(BridgeDriver, self).connect(vif, ifname, netns, container_id)
host_ifname = vif.vif_name
bridge_name = vif.bridge_name
with b_base.get_ipdb() as h_ipdb:
with h_ipdb.interfaces[bridge_name] as h_br:
h_br.add_port(host_ifname)
def disconnect(self, vif, ifname, netns, container_id):
# NOTE(dulek): Interfaces should get deleted with the netns, but it may
# happen that kubelet or crio will call new CNI ADD before
# the old netns is deleted. This might result in VLAN ID
# conflict. In oder to protect from that let's remove the
# netns ifaces here anyway.
with b_base.get_ipdb(netns) as c_ipdb:
self._remove_ifaces(c_ipdb, (vif.vif_name, ifname), netns)
def is_alive(self):
bridge_name = CONF.neutron_defaults.ovs_bridge
try:
with b_base.get_ipdb() as h_ipdb:
h_ipdb.interfaces[bridge_name]
return True
except Exception:
LOG.debug("Reporting Driver not healthy.")
return False
def connect(self, vif, ifname, netns):
host_ifname = vif.vif_name
with b_base.get_ipdb(netns) as c_ipdb:
with c_ipdb.create(ifname=ifname, peer=host_ifname,
kind='veth') as c_iface:
c_iface.mtu = vif.network.mtu
c_iface.address = str(vif.address)
c_iface.up()
if netns:
with c_ipdb.interfaces[host_ifname] as h_iface:
h_iface.net_ns_pid = os.getpid()
with b_base.get_ipdb() as h_ipdb:
with h_ipdb.interfaces[host_ifname] as h_iface:
h_iface.mtu = vif.network.mtu
h_iface.up()
def connect(self, vif, ifname, netns):
super(BridgeDriver, self).connect(vif, ifname, netns)
host_ifname = vif.vif_name
bridge_name = vif.bridge_name
h_ipdb = b_base.get_ipdb()
with h_ipdb.interfaces[bridge_name] as h_br:
h_br.add_port(host_ifname)
def is_healthy(self):
bridge_name = CONF.neutron_defaults.ovs_bridge
try:
with b_base.get_ipdb() as h_ipdb:
h_ipdb.interfaces[bridge_name]
return True
except Exception:
LOG.warning("Default OVS bridge %s does not exist on "
"the host.", bridge_name)
return False
def is_alive(self):
bridge_name = CONF.neutron_defaults.ovs_bridge
try:
with b_base.get_ipdb() as h_ipdb:
h_ipdb.interfaces[bridge_name]
return True
except Exception:
LOG.error(
"The configured ovs_bridge=%s integration interface "
"does not exists. Reporting that driver is not healthy.",
bridge_name)
return False
def connect(self, vif, ifname, netns, container_id):
host_ifname = vif.vif_name
with b_base.get_ipdb() as h_ipdb:
if host_ifname in h_ipdb.interfaces:
# NOTE(dulek): This most likely means that we already run
# connect for this iface and there's a leftover
# host-side vif. Let's remove it, its peer should
# get deleted automatically by the kernel.
LOG.debug(
'Found leftover host vif %s. Removing it before'
'connecting.', host_ifname)
with h_ipdb.interfaces[host_ifname] as h_iface:
h_iface.remove()
if vif.network.mtu:
interface_mtu = vif.network.mtu
else:
LOG.info(
"Default mtu %(mtu)s is used for interface, "
"for mtu of network if configured with 0",
{"mtu": CONF.neutron_defaults.network_device_mtu})
interface_mtu = CONF.neutron_defaults.network_device_mtu
with b_base.get_ipdb(netns) as c_ipdb:
with c_ipdb.create(ifname=ifname, peer=host_ifname,
kind='veth') as c_iface:
c_iface.mtu = interface_mtu
c_iface.address = str(vif.address)
c_iface.up()
if netns:
with c_ipdb.interfaces[host_ifname] as h_iface:
h_iface.net_ns_pid = os.getpid()
with b_base.get_ipdb() as h_ipdb:
with h_ipdb.interfaces[host_ifname] as h_iface:
h_iface.mtu = interface_mtu
h_iface.up()
def connect(self, vif, ifname, netns, container_id):
# NOTE(vikasc): Ideally 'ifname' should be used here but instead a
# temporary name is being used while creating the device for
# container in host network namespace. This is because cni expects
# only 'eth0' as interface name and if host already has an
# interface named 'eth0', device creation will fail with 'already
# exists' error.
temp_name = vif.vif_name
# First let's take a peek into the pod namespace and try to remove any
# leftover interface in case we got restarted before CNI returned to
# kubelet.
with b_base.get_ipdb(netns) as c_ipdb:
self._remove_ifaces(c_ipdb, (temp_name, ifname), netns)
# We might also have leftover interface in the host netns, let's try to
# remove it too. This is outside of the main host's IPDB context
# manager to make sure removal is commited before starting next
# transaction.
with b_base.get_ipdb() as h_ipdb:
self._remove_ifaces(h_ipdb, (temp_name,))
try:
with b_base.get_ipdb() as h_ipdb:
# TODO(vikasc): evaluate whether we should have stevedore
# driver for getting the link device.
vm_iface_name = config.CONF.binding.link_iface
args = self._get_iface_create_args(vif)
with h_ipdb.create(ifname=temp_name,
link=h_ipdb.interfaces[vm_iface_name],
**args) as iface:
iface.net_ns_fd = utils.convert_netns(netns)
except pyroute2.NetlinkError as e:
if e.code == errno.EEXIST:
# NOTE(dulek): This is related to bug 1854928. It's super-rare,
# so aim of this piece is to gater any info useful
# for determining when it happens.
LOG.exception('Creation of pod interface failed, most likely '
'due to duplicated VLAN id. This will probably '
'cause kuryr-daemon to crashloop. Trying to '
'gather debugging information.')
with b_base.get_ipdb() as h_ipdb:
LOG.error('List of host interfaces: %s', h_ipdb.interfaces)
with b_base.get_ipdb(netns) as c_ipdb:
LOG.error('List of pod namespace interfaces: %s',
c_ipdb.interfaces)
raise
with b_base.get_ipdb(netns) as c_ipdb:
with c_ipdb.interfaces[temp_name] as iface:
iface.ifname = ifname
iface.mtu = vif.network.mtu
iface.address = str(vif.address)
iface.up()
def connect(self, vif, ifname, netns, container_id):
# NOTE(vikasc): Ideally 'ifname' should be used here but instead a
# temporary name is being used while creating the device for
# container in host network namespace. This is because cni expects
# only 'eth0' as interface name and if host already has an
# interface named 'eth0', device creation will fail with 'already
# exists' error.
temp_name = vif.vif_name
# First let's take a peek into the pod namespace and try to remove any
# leftover interface in case we got restarted before CNI returned to
# kubelet.
with b_base.get_ipdb(netns) as c_ipdb:
self._remove_ifaces(c_ipdb, (temp_name, ifname), netns)
with b_base.get_ipdb() as h_ipdb:
# TODO(vikasc): evaluate whether we should have stevedore
# driver for getting the link device.
vm_iface_name = config.CONF.binding.link_iface
# We might also have leftover interface in the host netns, let's
# try to remove it too.
self._remove_ifaces(h_ipdb, (temp_name,))
args = self._get_iface_create_args(vif)
with h_ipdb.create(ifname=temp_name,
link=h_ipdb.interfaces[vm_iface_name],
**args) as iface:
iface.net_ns_fd = utils.convert_netns(netns)
with b_base.get_ipdb(netns) as c_ipdb:
with c_ipdb.interfaces[temp_name] as iface:
iface.ifname = ifname
iface.mtu = vif.network.mtu
iface.address = str(vif.address)
iface.up()
def _cleanup_conflicting_vlan(self, netns, vlan_id):
if vlan_id is None:
# Better to not attempt that, might remove way to much.
return
netns_dir = os.path.dirname(netns)
for ns in os.listdir(netns_dir):
ns = os.fsdecode(ns)
with b_base.get_ipdb(ns) as c_ipdb:
for ifname, iface in c_ipdb.interfaces.items():
if iface.vlan_id == vlan_id:
LOG.warning(
f'Found offending interface {ifname} with VLAN ID '
f'{vlan_id} in netns {ns}. Trying to remove it.')
with c_ipdb.interfaces[ifname] as iface:
iface.remove()
break
def connect(self, vif, ifname, netns, container_id):
# NOTE(vikasc): Ideally 'ifname' should be used here but instead a
# temporary name is being used while creating the device for
# container in host network namespace. This is because cni expects
# only 'eth0' as interface name and if host already has an
# interface named 'eth0', device creation will fail with 'already
# exists' error.
temp_name = vif.vif_name
# First let's take a peek into the pod namespace and try to remove any
# leftover interface in case we got restarted before CNI returned to
# kubelet.
with b_base.get_ipdb(netns) as c_ipdb:
self._remove_ifaces(c_ipdb, (temp_name, ifname), netns)
# We might also have leftover interface in the host netns, let's try to
# remove it too. This is outside of the main host's IPDB context
# manager to make sure removal is commited before starting next
# transaction.
with b_base.get_ipdb() as h_ipdb:
self._remove_ifaces(h_ipdb, (temp_name, ))
try:
with b_base.get_ipdb() as h_ipdb:
# TODO(vikasc): evaluate whether we should have stevedore
# driver for getting the link device.
vm_iface_name = self._detect_iface_name(h_ipdb)
mtu = h_ipdb.interfaces[vm_iface_name].mtu
if mtu != vif.network.mtu:
# NOTE(dulek): This might happen if Neutron and DHCP agent
# have different MTU settings. See
# https://bugs.launchpad.net/kuryr-kubernetes/+bug/1863212
raise exceptions.CNIBindingFailure(
f'MTU of interface {vm_iface_name} ({mtu}) does not '
f'match MTU of pod network {vif.network.id} '
f'({vif.network.mtu}). Please make sure pod network '
f'has the same MTU as node (VM) network.')
args = self._get_iface_create_args(vif)
with h_ipdb.create(ifname=temp_name,
link=h_ipdb.interfaces[vm_iface_name],
**args) as iface:
iface.net_ns_fd = utils.convert_netns(netns)
except pyroute2.NetlinkError as e:
if e.code == errno.EEXIST:
# NOTE(dulek): This is related to bug 1854928. It's super-rare,
# so aim of this piece is to gater any info useful
# for determining when it happens.
LOG.exception(f'Creation of pod interface failed due to VLAN '
f'ID (vlan_info={args}) conflict. Probably the '
f'CRI had not cleaned up the network namespace '
f'of deleted pods. This should not be a '
f'permanent issue but may cause restart of '
f'kuryr-cni pod.')
raise
with b_base.get_ipdb(netns) as c_ipdb:
with c_ipdb.interfaces[temp_name] as iface:
iface.ifname = ifname
iface.mtu = vif.network.mtu
iface.address = str(vif.address)
iface.up()
def _cleanup_conflicting_vlan(self, netns, vlan_id):
if vlan_id is None:
# Better to not attempt that, might remove way to much.
return
netns_paths = []
handled_netns = set()
with b_base.get_ipdb() as h_ipdb:
vm_iface_name = self._detect_iface_name(h_ipdb)
vm_iface_index = h_ipdb.interfaces[vm_iface_name].index
if netns.startswith('/proc'):
# Paths have /proc/<pid>/ns/net pattern, we need to iterate
# over /proc.
netns_dir = utils.convert_netns('/proc')
for pid in os.listdir(netns_dir):
if not pid.isdigit():
# Ignore all the non-pid stuff in /proc
continue
netns_paths.append(os.path.join(netns_dir, pid, 'ns/net'))
else:
# cri-o manages netns, they're in /var/run/netns/* or similar.
netns_dir = os.path.dirname(netns)
netns_paths = os.listdir(netns_dir)
netns_paths = [
os.path.join(netns_dir, netns_path)
for netns_path in netns_paths
]
for netns_path in netns_paths:
netns_path = os.fsdecode(netns_path)
try:
# NOTE(dulek): inode can be used to clearly distinguish the
# netns' as `man namespaces` says:
#
# Since Linux 3.8, they appear as symbolic links. If two
# processes are in the same namespace, then the device IDs and
# inode numbers of their /proc/[pid]/ns/xxx symbolic links will
# be the same; an application can check this using the
# stat.st_dev and stat.st_ino fields returned by stat(2).
netns_stat = os.stat(netns_path)
netns_id = netns_stat.st_dev, netns_stat.st_ino
except OSError:
continue
if netns_id in handled_netns:
continue
handled_netns.add(netns_id)
try:
with b_base.get_ipdb(netns_path) as c_ipdb:
for ifname, iface in c_ipdb.interfaces.items():
if (iface.vlan_id == vlan_id
and iface.link == vm_iface_index):
LOG.warning(
f'Found offending interface {ifname} with '
f'VLAN ID {vlan_id} in netns {netns_path}. '
f'Trying to remove it.')
with c_ipdb.interfaces[ifname] as found_iface:
found_iface.remove()
break
except OSError:
continue
def _get_iface_name_by_mac(self, mac_address):
with b_base.get_ipdb() as h_ipdb:
for name, data in h_ipdb.interfaces.items():
if data['address'] == mac_address:
return data['ifname']
