def sync(self):
rt = netif.RoutingTable()
new_routes = [
self._netif_route(route)
for route in self.middleware.call_sync('staticroute.query')
]
default_route_ipv4 = rt.default_route_ipv4
default_route_ipv6 = rt.default_route_ipv6
for route in rt.routes:
if route in new_routes:
new_routes.remove(route)
continue
if route not in [default_route_ipv4, default_route_ipv6
] and route.gateway is not None:
self.logger.debug('Removing route %r', route.__getstate__())
try:
rt.delete(route)
except Exception as e:
self.logger.warning('Failed to remove route: %r', e)
for route in new_routes:
self.logger.debug('Adding route %r', route.__getstate__())
try:
rt.add(route)
except Exception as e:
self.logger.warning('Failed to add route: %r', e)
def cleanup_cni(self):
# We want to remove all CNI related configuration when k8s stops
# We will clean configuration done by kube-router now
# Below command will cleanup iptables rules and other ipvs bits changed by kube-router
cp = subprocess.Popen(['kube-router', '--cleanup-config'],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE)
stderr = cp.communicate()[1]
if cp.returncode:
raise CallError(
f'Failed to cleanup kube-router configuration: {stderr.decode()}'
)
tables = netif.RoutingTable().routing_tables
for t_name in filter(lambda t: t in tables,
('kube-router', 'kube-router-dsr')):
table = tables[t_name]
table.flush_routes()
table.flush_rules()
interfaces = netif.list_interfaces()
for iface in map(
lambda n: interfaces[n],
filter(lambda n: n in interfaces,
('kube-bridge', 'kube-dummy-if'))):
self.middleware.call_sync('interface.unconfigure', iface, [], [])
def cleanup_cni(self):
# We want to remove all CNI related configuration when k8s stops
# We will clean configuration done by kube-router now
# Below command will cleanup iptables rules and other ipvs bits changed by kube-router
cp = subprocess.Popen(['kube-router', '--cleanup-config'],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE)
stderr = cp.communicate()[1]
if cp.returncode:
# Let's log the error as to why kube-router was not able to clean up ipvs bits
# TODO: We should raise an exception but right now this is broken upstream and raising an exception
# here means we won't be cleaning/flushing the locally configured routes adding to the issue
self.logger.error(
'Failed to cleanup kube-router configuration: %r',
stderr.decode())
rule_table = netif.RuleTable()
if rule_table.rule_exists(KUBEROUTER_RULE_PRIORITY):
rule_table.delete_rule(KUBEROUTER_RULE_PRIORITY)
tables = netif.RoutingTable().routing_tables
for t_name in filter(lambda t: t in tables,
('kube-router', 'kube-router-dsr')):
table = tables[t_name]
table.flush_routes()
table.flush_rules()
interfaces = netif.list_interfaces()
for iface in map(
lambda n: interfaces[n],
filter(lambda n: n in interfaces,
('kube-bridge', 'kube-dummy-if'))):
self.middleware.call_sync('interface.unconfigure', iface, [], [])
def add_routes_to_kube_router_table(self):
rt = netif.RoutingTable()
kube_router_table = rt.routing_tables['kube-router']
for route in filter(lambda r: (r.interface or '') == 'kube-bridge',
rt.routes_internal(table_filter=254)):
route.table_id = kube_router_table.table_id
if route in kube_router_table.routes:
continue
rt.add(route)
def do_delete(self, id):
"""
Delete Static Route of `id`.
"""
staticroute = self.middleware.call_sync('staticroute.get_instance', id)
rv = self.middleware.call_sync('datastore.delete',
self._config.datastore, id)
try:
rt = netif.RoutingTable()
rt.delete(self._netif_route(staticroute))
except Exception as e:
self.logger.warn(
'Failed to delete static route %s: %s',
staticroute['destination'],
e,
)
return rv
def add_routes_to_kube_router_table(self):
rt = netif.RoutingTable()
kube_router_table = rt.routing_tables['kube-router']
cluster_cidr = ipaddress.ip_network(
self.middleware.call_sync('kubernetes.config')['cluster_cidr'],
False)
while not any(r.interface == 'kube-bridge'
and str(r.network) == str(cluster_cidr.network_address)
for r in rt.routes_internal(table_filter=254)):
time.sleep(5)
for route in filter(lambda r: (r.interface or '') == 'kube-bridge',
rt.routes_internal(table_filter=254)):
route.table_id = kube_router_table.table_id
if route in kube_router_table.routes:
continue
rt.add(route)
def setup_nic_attach(self):
nic_attach = self.data['attributes'].get('nic_attach')
interfaces = netif.list_interfaces()
if nic_attach and nic_attach not in interfaces:
raise CallError(f'{nic_attach} not found.')
else:
if not nic_attach:
try:
nic_attach = netif.RoutingTable().default_route_ipv4.interface
nic = netif.get_interface(nic_attach)
except Exception as e:
raise CallError(f'Unable to retrieve default interface: {e}')
else:
nic = netif.get_interface(nic_attach)
if netif.InterfaceFlags.UP not in nic.flags:
nic.up()
self.nic_attach = nic.name
def add_user_route_to_kube_router_table(self):
# User route is a default route for kube router table which is going to be
# used for traffic going outside k8s cluster via pods
config = self.middleware.call_sync('kubernetes.config')
if all(not config[k] for k in config if k.startswith('route_v')):
return
rt = netif.RoutingTable()
kube_router_table = rt.routing_tables['kube-router']
for k in filter(
lambda k: config[f'{k}_gateway'] and config[f'{k}_interface'],
('route_v4', 'route_v6')):
factory = ipaddress.IPv4Address if k.endswith(
'v4') else ipaddress.IPv6Address
rt.add(
netif.Route(
factory(0),
factory(0),
ipaddress.ip_address(config[f'{k}_gateway']),
config[f'{k}_interface'],
table_id=kube_router_table.table_id,
))
async def setup_cni(self):
kube_config = await self.middleware.call('datastore.query',
'services.kubernetes', [],
{'get': True})
config = await self.config()
async with api_client() as (api, context):
cni_config = kube_config['cni_config']
for cni in config:
if not await self.validate_cni_integrity(cni, kube_config):
cni_config[
cni] = await service_accounts.get_service_account_details(
context['core_api'],
config[cni]['service_account'])
await self.middleware.call('datastore.update', 'services.kubernetes',
kube_config['id'],
{'cni_config': cni_config})
await self.middleware.call('etc.generate', 'cni')
await self.middleware.call('service.start', 'kuberouter')
rt = netif.RoutingTable()
timeout = 60
while timeout > 0:
if 'kube-router' not in rt.routing_tables:
await asyncio.sleep(2)
timeout -= 2
else:
break
if 'kube-router' not in rt.routing_tables:
raise CallError(
'Unable to locate kube-router routing table. Please refer to kuberouter logs.'
)
await self.middleware.call(
'k8s.cni.add_user_route_to_kube_router_table')
def identity(self):
nic_attach = self.data['attributes'].get('nic_attach')
if not nic_attach:
nic_attach = netif.RoutingTable().default_route_ipv4.interface
return nic_attach