def _update_listener_acls(self, loadbalancer, listener_id, allowed_cidrs):
admin_state_up = True
if allowed_cidrs is None:
# World accessible, no restriction on the listeners
pass
elif len(allowed_cidrs) == 0:
# Prevent any traffic as no CIDR is allowed
admin_state_up = False
request = {
'allowed_cidrs': allowed_cidrs,
'admin_state_up': admin_state_up,
}
# Wait for the loadbalancer to be ACTIVE
self._wait_for_provisioning(loadbalancer, _ACTIVATION_TIMEOUT,
_LB_STS_POLL_FAST_INTERVAL)
lbaas = clients.get_loadbalancer_client()
try:
lbaas.update_listener(listener_id, **request)
except os_exc.SDKException:
LOG.exception('Error when updating listener %s' % listener_id)
raise k_exc.ResourceNotReady(listener_id)
def _update_crd_spec(self, loadbalancer_crd, service):
svc_ip = self._get_service_ip(service)
ports = service['spec'].get('ports')
for port in ports:
if type(port['targetPort']) == int:
port['targetPort'] = str(port['targetPort'])
project_id = self._drv_project.get_project(service)
sg_ids = self._drv_sg.get_security_groups(service, project_id)
subnet_id = self._get_subnet_id(service, project_id, svc_ip)
spec_type = service['spec'].get('type')
kubernetes = clients.get_kubernetes_client()
patch = {
'spec': {
'ip': svc_ip,
'ports': ports,
'project_id': project_id,
'security_groups_ids': sg_ids,
'subnet_id': subnet_id,
'type': spec_type
}
}
LOG.debug('Patching KuryrLoadBalancer CRD %s', loadbalancer_crd)
try:
kubernetes.patch_crd('spec',
loadbalancer_crd['metadata']['selfLink'],
patch['spec'])
except k_exc.K8sResourceNotFound:
LOG.debug('KuryrLoadBalancer CRD not found %s', loadbalancer_crd)
except k_exc.K8sConflict:
raise k_exc.ResourceNotReady(loadbalancer_crd)
except k_exc.K8sClientException:
LOG.exception('Error updating kuryrnet CRD %s', loadbalancer_crd)
raise
return loadbalancer_crd
def add(self, params):
vifs = self._do_work(params, b_base.connect)
pod_name = self._get_pod_name(params)
# NOTE(dulek): Saving containerid to be able to distinguish old DEL
# requests that we should ignore. We need a lock to
# prevent race conditions and replace whole object in the
# dict for multiprocessing.Manager to notice that.
with lockutils.lock(pod_name, external=True):
d = self.registry[pod_name]
d['containerid'] = params.CNI_CONTAINERID
self.registry[pod_name] = d
LOG.debug('Saved containerid = %s for pod %s',
params.CNI_CONTAINERID, pod_name)
# Wait for VIFs to become active.
timeout = CONF.cni_daemon.vif_annotation_timeout
# Wait for timeout sec, 1 sec between tries, retry when even one
# vif is not active.
@retrying.retry(stop_max_delay=timeout * 1000, wait_fixed=RETRY_DELAY,
retry_on_result=lambda x: any(
map(lambda y: not y[1].active, x.items())))
def wait_for_active(pod_name):
return {
ifname: base.VersionedObject.obj_from_primitive(vif_obj) for
ifname, vif_obj in self.registry[pod_name]['vifs'].items()
}
vifs = wait_for_active(pod_name)
for vif in vifs.values():
if not vif.active:
raise exceptions.ResourceNotReady(pod_name)
return vifs[k_const.DEFAULT_IFNAME]
def delete_network_pools(self, net_id):
if not hasattr(self, '_available_ports_pools'):
LOG.info("Kuryr-controller not yet ready to delete network "
"pools.")
raise exceptions.ResourceNotReady(net_id)
neutron = clients.get_neutron_client()
# NOTE(ltomasbo): Note the pods should already be deleted, but their
# associated ports may not have been recycled yet, therefore not being
# on the available_ports_pools dict. The next call forces it to be on
# that dict before cleaning it up
self._trigger_return_to_pool()
for pool_key, ports_ids in self._available_ports_pools.items():
if self._get_pool_key_net(pool_key) != net_id:
continue
self._available_ports_pools[pool_key] = []
trunk_id = self._get_trunk_id(neutron, pool_key)
try:
self._drv_vif._remove_subports(neutron, trunk_id, ports_ids)
except n_exc.NeutronClientException:
LOG.exception('Error removing subports from trunk: %s',
trunk_id)
continue
for port_id in ports_ids:
try:
self._drv_vif._release_vlan_id(
self._existing_vifs[port_id].vlan_id)
del self._existing_vifs[port_id]
except KeyError:
LOG.debug('Port %s is not in the ports list.', port_id)
try:
neutron.delete_port(port_id)
except n_exc.PortNotFoundClient:
LOG.debug(
'Unable to delete subport %s as it no longer '
'exists.', port_id)
def _set_lbaas_spec(self, service, lbaas_spec):
# TODO(ivc): extract annotation interactions
if lbaas_spec is None:
LOG.debug("Removing LBaaSServiceSpec annotation: %r", lbaas_spec)
annotation = None
else:
lbaas_spec.obj_reset_changes(recursive=True)
LOG.debug("Setting LBaaSServiceSpec annotation: %r", lbaas_spec)
annotation = jsonutils.dumps(lbaas_spec.obj_to_primitive(),
sort_keys=True)
svc_link = service['metadata']['selfLink']
ep_link = self._get_endpoints_link(service)
k8s = clients.get_kubernetes_client()
try:
k8s.annotate(ep_link,
{k_const.K8S_ANNOTATION_LBAAS_SPEC: annotation})
except k_exc.K8sClientException:
# REVISIT(ivc): only raise ResourceNotReady for NotFound
raise k_exc.ResourceNotReady(ep_link)
k8s.annotate(svc_link,
{k_const.K8S_ANNOTATION_LBAAS_SPEC: annotation},
resource_version=service['metadata']['resourceVersion'])
def create_subnet(self, ns_name, project_id, net_id):
os_net = clients.get_network_client()
subnet_name = "ns/" + ns_name + "-subnet"
tags = oslo_cfg.CONF.neutron_defaults.resource_tags
if tags:
subnets = os_net.subnets(name=subnet_name, tags=tags)
else:
subnets = os_net.subnets(name=subnet_name)
try:
# NOTE(ltomasbo): only one subnet must exists
subnet = next(subnets)
return subnet.id, subnet.cidr
except StopIteration:
LOG.debug('Subnet does not exist. Creating.')
# create subnet with namespace as name
subnet_pool_id = oslo_cfg.CONF.namespace_subnet.pod_subnet_pool
ip_version = utils.get_subnetpool_version(subnet_pool_id)
try:
neutron_subnet = (os_net.create_subnet(
network_id=net_id,
ip_version=ip_version,
name=subnet_name,
enable_dhcp=False,
subnetpool_id=subnet_pool_id,
project_id=project_id))
except os_exc.ConflictException:
LOG.debug(
"Max number of retries on neutron side achieved, "
"raising ResourceNotReady to retry subnet creation "
"for %s", subnet_name)
raise exceptions.ResourceNotReady(subnet_name)
c_utils.tag_neutron_resources([neutron_subnet])
return neutron_subnet.id, neutron_subnet.cidr
def _delete_namespace_network_resources(self, subnet_id, net_id):
os_net = clients.get_network_client()
if subnet_id:
router_id = oslo_cfg.CONF.namespace_subnet.pod_router
try:
clients.handle_neutron_errors(
os_net.remove_interface_from_router,
router_id,
subnet_id=subnet_id)
except os_exc.NotFoundException as e:
# Nothing to worry about, either router or subnet is no more,
# or subnet is already detached.
LOG.debug(e.message)
pass
except os_exc.SDKException:
LOG.exception(
"Error deleting subnet %(subnet)s from router "
"%(router)s.", {
'subnet': subnet_id,
'router': router_id
})
raise
try:
os_net.delete_network(net_id)
except os_exc.ConflictException:
LOG.exception(
"One or more ports in use on the network %s. "
"Deleting leftovers ports before retrying", net_id)
leftover_ports = os_net.ports(network_id=net_id)
for leftover_port in leftover_ports:
# NOTE(dulek): '' is there because Neutron seems to unset
# device_owner on detach.
if leftover_port.device_owner not in [
'', 'trunk:subport', kl_const.DEVICE_OWNER
]:
continue
try:
# NOTE(gryf): there is unlikely, that we get an exception
# like PortNotFound or something, since openstacksdk
# doesn't raise an exception if port doesn't exists nor
# return any information.
os_net.delete_port(leftover_port.id)
except os_exc.SDKException as e:
if "currently a subport for trunk" in str(e):
if leftover_port.status == "DOWN":
LOG.warning(
"Port %s is in DOWN status but still "
"associated to a trunk. This should "
"not happen. Trying to delete it from "
"the trunk.", leftover_port.id)
# Get the trunk_id from the error message
trunk_id = (
str(e).split('trunk')[1].split('.')[0].strip())
try:
os_net.delete_trunk_subports(
trunk_id, [{
'port_id': leftover_port.id
}])
except os_exc.NotFoundException:
LOG.debug("Port %s already removed from trunk %s",
leftover_port['id'], trunk_id)
else:
LOG.exception(
"Unexpected error deleting leftover "
"port %s. Skiping it and continue with "
"the other rest.", leftover_port.id)
raise exceptions.ResourceNotReady(net_id)
except os_exc.SDKException:
LOG.exception("Error deleting network %s.", net_id)
raise
def on_present(self, kuryrport_crd):
if not kuryrport_crd['status']['vifs']:
# Get vifs
if not self.get_vifs(kuryrport_crd):
# Ignore this event, according to one of the cases logged in
# get_vifs method.
return
vifs = {ifname: {'default': data['default'],
'vif': objects.base.VersionedObject
.obj_from_primitive(data['vif'])}
for ifname, data in kuryrport_crd['status']['vifs'].items()}
if all([v['vif'].active for v in vifs.values()]):
return
changed = False
try:
for ifname, data in vifs.items():
if (data['vif'].plugin == constants.KURYR_VIF_TYPE_SRIOV and
oslo_cfg.CONF.sriov.enable_node_annotations):
pod_node = kuryrport_crd['spec']['podNodeName']
# TODO(gryf): This probably will need adoption, so it will
# add information to CRD instead of the pod.
driver_utils.update_port_pci_info(pod_node, data['vif'])
if not data['vif'].active:
try:
self._drv_vif_pool.activate_vif(data['vif'])
changed = True
except os_exc.ResourceNotFound:
LOG.debug("Port not found, possibly already deleted. "
"No need to activate it")
finally:
if changed:
try:
name = kuryrport_crd['metadata']['name']
namespace = kuryrport_crd['metadata']['namespace']
pod = self.k8s.get(f"{constants.K8S_API_NAMESPACES}"
f"/{namespace}/pods/{name}")
except k_exc.K8sResourceNotFound as ex:
LOG.exception("Failed to get pod: %s", ex)
raise
project_id = self._drv_project.get_project(pod)
try:
self._update_kuryrport_crd(kuryrport_crd, vifs)
except k_exc.K8sResourceNotFound as ex:
LOG.exception("Failed to update KuryrPort CRD: %s", ex)
security_groups = self._drv_sg.get_security_groups(
pod, project_id)
for ifname, data in vifs.items():
self._drv_vif_pool.release_vif(pod, data['vif'],
project_id,
security_groups)
except k_exc.K8sClientException:
raise k_exc.ResourceNotReady(pod['metadata']['name'])
try:
self._record_pod_creation_metric(pod)
except Exception:
LOG.debug("Failed to record metric for pod %s", name)
if self._is_network_policy_enabled():
crd_pod_selectors = self._drv_sg.create_sg_rules(pod)
if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules:
services = driver_utils.get_services()
self._update_services(services, crd_pod_selectors,
project_id)
def _move_annotations_to_crd(self, endpoints):
"""Support upgrade from annotations to KuryrLoadBalancer CRD."""
try:
spec = (endpoints['metadata']['annotations']
[k_const.K8S_ANNOTATION_LBAAS_SPEC])
except KeyError:
spec = None
try:
state = (endpoints['metadata']['annotations']
[k_const.K8S_ANNOTATION_LBAAS_STATE])
except KeyError:
state = None
if not state and not spec:
# No annotations, return
return False
if state or spec:
if state:
_dict = jsonutils.loads(state)
# This is strongly using the fact that annotation's o.vo
# and CRD has the same structure.
state = obj_lbaas.flatten_object(_dict)
# Endpoints should always have the spec in the annotation
spec_dict = jsonutils.loads(spec)
spec = obj_lbaas.flatten_object(spec_dict)
if state and state['service_pub_ip_info'] is None:
del state['service_pub_ip_info']
for spec_port in spec['ports']:
if not spec_port.get('name'):
del spec_port['name']
if not spec['lb_ip']:
del spec['lb_ip']
try:
self._create_crd_spec(endpoints, spec, state)
except k_exc.ResourceNotReady:
LOG.info('KuryrLoadBalancer CRD %s already exists.',
utils.get_res_unique_name(endpoints))
except k_exc.K8sClientException:
raise k_exc.ResourceNotReady(endpoints)
# In this step we only need to make sure all annotations are
# removed. It may happen that the Endpoints only had spec set,
# in which case we just remove it and let the normal flow handle
# creation of the LB.
k8s = clients.get_kubernetes_client()
service_link = utils.get_service_link(endpoints)
to_remove = [
(endpoints['metadata']['selfLink'],
k_const.K8S_ANNOTATION_LBAAS_SPEC),
(service_link,
k_const.K8S_ANNOTATION_LBAAS_SPEC),
]
if state:
to_remove.append((endpoints['metadata']['selfLink'],
k_const.K8S_ANNOTATION_LBAAS_STATE))
for path, name in to_remove:
try:
k8s.remove_annotations(path, name)
except k_exc.K8sClientException:
LOG.warning('Error removing %s annotation from %s', name,
path)
return True
def add(self, params):
kp_name = self._get_obj_name(params)
timeout = CONF.cni_daemon.vif_annotation_timeout
# Try to confirm if CRD in the registry is not stale cache. If it is,
# remove it.
with lockutils.lock(kp_name, external=True):
if kp_name in self.registry:
cached_kp = self.registry[kp_name]['kp']
try:
kp = self.k8s.get(k_utils.get_res_link(cached_kp))
except Exception:
LOG.exception('Error when getting KuryrPort %s', kp_name)
raise exceptions.ResourceNotReady(kp_name)
if kp['metadata']['uid'] != cached_kp['metadata']['uid']:
LOG.warning(
'Stale KuryrPort %s detected in cache. (API '
'uid=%s, cached uid=%s). Removing it from '
'cache.', kp_name, kp['metadata']['uid'],
cached_kp['metadata']['uid'])
del self.registry[kp_name]
vifs = self._do_work(params, b_base.connect, timeout)
# NOTE(dulek): Saving containerid to be able to distinguish old DEL
# requests that we should ignore. We need a lock to
# prevent race conditions and replace whole object in the
# dict for multiprocessing.Manager to notice that.
with lockutils.lock(kp_name, external=True):
d = self.registry[kp_name]
d['containerid'] = params.CNI_CONTAINERID
self.registry[kp_name] = d
LOG.debug('Saved containerid = %s for CRD %s',
params.CNI_CONTAINERID, kp_name)
# Wait for timeout sec, 1 sec between tries, retry when even one
# vif is not active.
@retrying.retry(stop_max_delay=timeout * 1000,
wait_fixed=RETRY_DELAY,
retry_on_result=utils.any_vif_inactive)
def wait_for_active(kp_name):
return self.registry[kp_name]['vifs']
data = {
'metadata': {
'name': params.args.K8S_POD_NAME,
'namespace': params.args.K8S_POD_NAMESPACE
}
}
pod = k_utils.get_referenced_object(data, 'Pod')
try:
self.k8s.add_event(
pod, 'CNIWaitingForVIFs',
f'Waiting for Neutron ports of {kp_name} to '
f'become ACTIVE after binding.')
vifs = wait_for_active(kp_name)
except retrying.RetryError:
self.k8s.add_event(
pod, 'CNITimedOutWaitingForVIFs',
f'Timed out waiting for Neutron ports of '
f'{kp_name} to become ACTIVE after binding.', 'Warning')
raise exceptions.CNINeutronPortActivationTimeout(
kp_name, self.registry[kp_name]['vifs'])
return vifs[k_const.DEFAULT_IFNAME]
def on_present(self, namespace):
ns_name = namespace['metadata']['name']
current_namespace_labels = namespace['metadata'].get('labels')
previous_namespace_labels = drivers_utils.get_annotated_labels(
namespace, constants.K8S_ANNOTATION_NAMESPACE_LABEL)
LOG.debug("Got previous namespace labels from annotation: %r",
previous_namespace_labels)
if current_namespace_labels != previous_namespace_labels:
self._drv_sg.update_namespace_sg_rules(namespace)
self._set_namespace_labels(namespace, current_namespace_labels)
project_id = self._drv_project.get_project(namespace)
net_crd_id = self._get_net_crd_id(namespace)
if net_crd_id:
LOG.debug("CRD existing at the new namespace")
return
net_crd_name = 'ns-' + ns_name
net_crd = self._get_net_crd(net_crd_name)
if net_crd:
LOG.debug("Previous CRD existing at the new namespace. "
"Deleting namespace resources and retying its creation.")
self.on_deleted(namespace, net_crd)
raise exceptions.ResourceNotReady(namespace)
# NOTE(ltomasbo): Ensure there is no previously created networks
# leftovers due to a kuryr-controller crash/restart
LOG.debug("Deleting leftovers network resources for namespace: %s",
ns_name)
self._drv_subnets.cleanup_namespace_networks(ns_name)
LOG.debug("Creating network resources for namespace: %s", ns_name)
net_crd_spec = self._drv_subnets.create_namespace_network(
ns_name, project_id)
try:
net_crd_sg = self._drv_sg.create_namespace_sg(
ns_name, project_id, net_crd_spec)
except n_exc.NeutronClientException:
LOG.exception("Error creating security group for the namespace. "
"Rolling back created network resources.")
self._drv_subnets.rollback_network_resources(net_crd_spec, ns_name)
raise
if net_crd_sg:
net_crd_spec.update(net_crd_sg)
else:
LOG.debug("No SG created for the namespace. Namespace isolation "
"will not be enforced.")
# create CRD resource for the network
try:
net_crd = self._add_kuryrnet_crd(ns_name, net_crd_spec)
self._set_net_crd(namespace, net_crd)
self._drv_sg.create_namespace_sg_rules(namespace)
self._set_namespace_labels(namespace, current_namespace_labels)
except exceptions.K8sClientException:
LOG.exception("Kubernetes client exception. Rolling back "
"resources created for the namespace.")
self._drv_subnets.rollback_network_resources(net_crd_spec, ns_name)
if net_crd_sg.get('sgId'):
self._drv_sg.delete_sg(net_crd_sg['sgId'])
self._del_kuryrnet_crd(net_crd_name)
def on_present(self, kuryrport_crd, *args, **kwargs):
if not kuryrport_crd['status']['vifs']:
# Get vifs
if not self.get_vifs(kuryrport_crd):
# Ignore this event, according to one of the cases logged in
# get_vifs method.
return
retry_info = kwargs.get('retry_info')
vifs = {
ifname: {
'default': data['default'],
'vif':
objects.base.VersionedObject.obj_from_primitive(data['vif'])
}
for ifname, data in kuryrport_crd['status']['vifs'].items()
}
if all([v['vif'].active for v in vifs.values()]):
return
changed = False
pod = self._get_pod(kuryrport_crd)
try:
for ifname, data in vifs.items():
if (data['vif'].plugin == constants.KURYR_VIF_TYPE_SRIOV
and oslo_cfg.CONF.sriov.enable_node_annotations):
pod_node = kuryrport_crd['spec']['podNodeName']
# TODO(gryf): This probably will need adoption, so it will
# add information to CRD instead of the pod.
driver_utils.update_port_pci_info(pod_node, data['vif'])
if not data['vif'].active:
try:
self._drv_vif_pool.activate_vif(data['vif'],
pod=pod,
retry_info=retry_info)
changed = True
except k_exc.ResourceNotReady:
if retry_info and retry_info.get('elapsed',
0) > ACTIVE_TIMEOUT:
self.k8s.add_event(
pod, 'ActivatePortFailed',
'Activating Neutron port has '
'timed out', 'Warning')
raise
except os_exc.ResourceNotFound:
self.k8s.add_event(
pod, 'ActivatePortFailed',
'Activating Neutron port has '
'failed, possibly deleted', 'Warning')
LOG.debug("Port not found, possibly already deleted. "
"No need to activate it")
finally:
if changed:
project_id = self._drv_project.get_project(pod)
try:
self._update_kuryrport_crd(kuryrport_crd, vifs)
except k_exc.K8sResourceNotFound as ex:
LOG.exception("Failed to update KuryrPort CRD: %s", ex)
security_groups = self._drv_sg.get_security_groups(
pod, project_id)
for ifname, data in vifs.items():
self._drv_vif_pool.release_vif(pod, data['vif'],
project_id,
security_groups)
self.k8s.add_event(
pod, 'UpdateKuryrPortCRDFailed',
f'Marking ports as ACTIVE in the '
f'KuryrPort failed: {ex}', 'Warning')
except k_exc.K8sClientException:
raise k_exc.ResourceNotReady(pod['metadata']['name'])
try:
self._record_pod_creation_metric(pod)
except Exception:
LOG.debug("Failed to record metric for pod %s",
pod['metadata']['name'])
if driver_utils.is_network_policy_enabled():
crd_pod_selectors = self._drv_sg.create_sg_rules(pod)
if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules:
services = driver_utils.get_services()
self._update_services(services, crd_pod_selectors,
project_id)
def on_finalize(self, knp):
LOG.debug("Finalizing KuryrNetworkPolicy %s")
project_id = self._drv_project.get_project(knp)
pods_to_update = self._drv_policy.affected_pods(knp)
crd_sg = knp['status'].get('securityGroupId')
try:
policy = self._get_networkpolicy(
knp['metadata']['annotations']['networkPolicyLink'])
except exceptions.K8sResourceNotFound:
# NP is already gone, let's just try to clean up.
policy = None
if crd_sg:
for pod in pods_to_update:
if driver_utils.is_host_network(pod):
continue
pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id)
if crd_sg in pod_sgs:
pod_sgs.remove(crd_sg)
if not pod_sgs:
pod_sgs = CONF.neutron_defaults.pod_security_groups
if not pod_sgs:
raise cfg.RequiredOptError(
'pod_security_groups',
cfg.OptGroup('neutron_defaults'))
try:
self._drv_vif_pool.update_vif_sgs(pod, pod_sgs)
except os_exc.NotFoundException:
LOG.debug("Fail to update pod sgs."
" Retrying policy deletion.")
raise exceptions.ResourceNotReady(knp)
# ensure ports at the pool don't have the NP sg associated
try:
net_id = self._get_policy_net_id(knp)
self._drv_vif_pool.remove_sg_from_pools(crd_sg, net_id)
except exceptions.K8sResourceNotFound:
# Probably the network got removed already, we can ignore it.
pass
if (CONF.octavia_defaults.enforce_sg_rules and policy
and not self._is_egress_only_policy(policy)):
services = driver_utils.get_services(
knp['metadata']['namespace'])
for svc in services.get('items'):
if (not svc['spec'].get('selector')
or not self._is_service_affected(
svc, pods_to_update)):
continue
sgs = self._drv_svc_sg.get_security_groups(svc, project_id)
try:
self._drv_lbaas.update_lbaas_sg(svc, sgs)
except exceptions.ResourceNotReady:
# We can ignore LB that's being created - its SGs will
# get handled when members will be getting created.
pass
self._drv_policy.delete_np_sg(crd_sg)
LOG.debug("Removing finalizers from KuryrNetworkPolicy and "
"NetworkPolicy.")
if policy:
self.k8s.remove_finalizer(policy,
constants.NETWORKPOLICY_FINALIZER)
self.k8s.remove_finalizer(knp, constants.NETWORKPOLICY_FINALIZER)
def _apply_members_security_groups(self, loadbalancer, port, target_port,
protocol, sg_rule_name, listener_id,
new_sgs=None):
LOG.debug("Applying members security groups.")
os_net = clients.get_network_client()
lb_sg = None
if CONF.octavia_defaults.enforce_sg_rules:
vip_port = self._get_vip_port(loadbalancer)
if vip_port:
try:
lb_sg = vip_port.security_group_ids[0]
except IndexError:
LOG.warning("We still waiting for SG to be created for "
"VIP %s", vip_port)
raise k_exc.ResourceNotReady(listener_id)
else:
LOG.debug("Skipping sg update for lb %s", loadbalancer['name'])
return
# NOTE (maysams) It might happen that the update of LBaaS SG
# has been triggered and the LBaaS SG was not created yet.
# This update is skiped, until the LBaaS members are created.
if not lb_sg:
return
if self._octavia_acls:
self._create_listeners_acls(loadbalancer, port, target_port,
protocol, lb_sg, new_sgs, listener_id)
return
lbaas_sg_rules = os_net.security_group_rules(
security_group_id=lb_sg, project_id=loadbalancer['project_id'])
all_pod_rules = []
add_default_rules = False
if new_sgs:
sgs = new_sgs
else:
sgs = loadbalancer['security_groups']
sg_rule_ethertype = k_const.IPv4
if utils.get_service_subnet_version() == k_const.IP_VERSION_6:
sg_rule_ethertype = k_const.IPv6
# Check if Network Policy allows listener on the pods
for sg in sgs:
if sg != lb_sg:
if sg in config.CONF.neutron_defaults.pod_security_groups:
# If default sg is set, this means there is no NP
# associated to the service, thus falling back to the
# default listener rules
add_default_rules = True
break
rules = os_net.security_group_rules(security_group_id=sg)
for rule in rules:
# copying ingress rules with same protocol onto the
# loadbalancer sg rules
# NOTE(ltomasbo): NP sg can only have rules with
# or without remote_ip_prefix. Rules with remote_group_id
# are not possible, therefore only applying the ones
# with or without remote_ip_prefix.
if (rule.protocol == protocol.lower() and
rule.direction == 'ingress'):
# If listener port not in allowed range, skip
min_port = rule.port_range_min
max_port = rule.port_range_max
if (min_port and target_port not in range(min_port,
max_port+1)):
continue
all_pod_rules.append(rule)
try:
LOG.debug("Creating LBaaS sg rule for sg: %r",
lb_sg)
os_net.create_security_group_rule(
direction='ingress',
ether_type=sg_rule_ethertype,
port_range_min=port,
port_range_max=port,
protocol=protocol,
remote_ip_prefix=rule.remote_ip_prefix,
security_group_id=lb_sg,
description=sg_rule_name)
except os_exc.ConflictException:
pass
except os_exc.SDKException:
LOG.exception('Failed when creating security '
'group rule for listener %s.',
sg_rule_name)
# Delete LBaaS sg rules that do not match NP
for rule in lbaas_sg_rules:
if (rule.protocol != protocol.lower() or
rule.port_range_min != port or
rule.direction != 'ingress'):
if all_pod_rules and self._is_default_rule(rule):
LOG.debug("Removing default LBaaS sg rule for sg: %r",
lb_sg)
os_net.delete_security_group_rule(rule.id)
continue
self._delete_rule_if_no_match(rule, all_pod_rules)
if add_default_rules:
try:
LOG.debug("Restoring default LBaaS sg rule for sg: %r", lb_sg)
os_net.create_security_group_rule(direction='ingress',
ether_type=sg_rule_ethertype,
port_range_min=port,
port_range_max=port,
protocol=protocol,
security_group_id=lb_sg,
description=sg_rule_name)
except os_exc.ConflictException:
pass
except os_exc.SDKException:
LOG.exception('Failed when creating security group rule for '
'listener %s.', sg_rule_name)
def on_present(self, pod, *args, **kwargs):
if utils.is_host_network(pod):
return
pod_name = pod['metadata']['name']
if utils.is_pod_completed(pod):
LOG.debug("Pod %s has completed execution, "
"removing the vifs", pod_name)
self.on_finalize(pod)
return
if not self._is_pod_scheduled(pod):
# REVISIT(ivc): consider an additional configurable check that
# would allow skipping pods to enable heterogeneous environments
# where certain pods/namespaces/nodes can be managed by other
# networking solutions/CNI drivers.
return
namespace = pod['metadata']['namespace']
kuryrnetwork_path = '{}/{}/kuryrnetworks/{}'.format(
constants.K8S_API_CRD_NAMESPACES, namespace,
namespace)
kuryrnetwork = driver_utils.get_k8s_resource(kuryrnetwork_path)
kuryrnetwork_status = kuryrnetwork.get('status', {})
if (CONF.kubernetes.pod_subnets_driver == 'namespace' and
(not kuryrnetwork or not kuryrnetwork_status.get('routerId'))):
namespace_path = '{}/{}'.format(
constants.K8S_API_NAMESPACES, namespace)
LOG.debug("Triggering Namespace Handling %s", namespace_path)
try:
self.k8s.annotate(namespace_path,
{'KuryrTrigger': str(uuid.uuid4())})
except k_exc.K8sResourceNotFound:
LOG.warning('Ignoring Pod handling, no Namespace %s.',
namespace)
return
raise k_exc.ResourceNotReady(pod)
# NOTE(gryf): Set the finalizer as soon, as we have pod created. On
# subsequent updates of the pod, add_finalizer will ignore this if
# finalizer exist.
try:
if not self.k8s.add_finalizer(pod, constants.POD_FINALIZER):
# NOTE(gryf) It might happen that pod will be deleted even
# before we got here.
return
except k_exc.K8sClientException as ex:
self.k8s.add_event(pod, 'FailedToAddFinalizerToPod',
f'Adding finalizer to pod has failed: {ex}',
'Warning')
LOG.exception("Failed to add finalizer to pod object: %s", ex)
raise
kp = driver_utils.get_kuryrport(pod)
LOG.debug("Got KuryrPort: %r", kp)
if not kp:
try:
self._add_kuryrport_crd(pod)
except k_exc.K8sNamespaceTerminating:
# The underlying namespace is being terminated, we can
# ignore this and let `on_finalize` handle this now.
LOG.warning('Namespace %s is being terminated, ignoring Pod '
'%s in that namespace.',
pod['metadata']['namespace'],
pod_name)
return
except k_exc.K8sClientException as ex:
self.k8s.add_event(pod, 'FailedToCreateKuryrPortCRD',
f'Creating corresponding KuryrPort CRD has '
f'failed: {ex}', 'Warning')
LOG.exception("Kubernetes Client Exception creating "
"KuryrPort CRD: %s", ex)
raise k_exc.ResourceNotReady(pod)
def update_lbaas_sg(self, service, sgs):
LOG.debug('Setting SG for LBaaS VIP port')
svc_namespace = service['metadata']['namespace']
svc_name = service['metadata']['name']
svc_ports = service['spec'].get('ports', [])
lbaas_name = c_utils.get_resource_name(svc_name,
prefix=svc_namespace + "/")
endpoints_link = utils.get_endpoints_link(service)
k8s = clients.get_kubernetes_client()
try:
k8s.get(endpoints_link)
except k_exc.K8sResourceNotFound:
LOG.debug("Endpoint not Found. Skipping LB SG update for "
"%s as the LB resources are not present", lbaas_name)
return
try:
klb = k8s.get(f'{k_const.K8S_API_CRD_NAMESPACES}/{svc_namespace}/'
f'kuryrloadbalancers/{svc_name}')
except k_exc.K8sResourceNotFound:
LOG.debug('No KuryrLoadBalancer for service %s created yet.',
lbaas_name)
raise k_exc.ResourceNotReady(svc_name)
if (not klb.get('status', {}).get('loadbalancer') or
klb.get('status', {}).get('listeners') is None):
LOG.debug('KuryrLoadBalancer for service %s not populated yet.',
lbaas_name)
raise k_exc.ResourceNotReady(svc_name)
klb['status']['loadbalancer']['security_groups'] = sgs
lb = klb['status']['loadbalancer']
try:
k8s.patch_crd('status/loadbalancer', utils.get_res_link(klb),
{'security_groups': sgs})
except k_exc.K8sResourceNotFound:
LOG.debug('KuryrLoadBalancer CRD not found %s', lbaas_name)
return
except k_exc.K8sClientException:
LOG.exception('Error updating KuryLoadBalancer CRD %s', lbaas_name)
raise
lsnr_ids = {(listener['protocol'], listener['port']): listener['id']
for listener in klb['status']['listeners']}
for port in svc_ports:
port_protocol = port['protocol']
lbaas_port = port['port']
target_port = port['targetPort']
suffix = f"{port_protocol}:{lbaas_port}"
sg_rule_name = c_utils.get_resource_name(lbaas_name,
suffix=':' + suffix)
listener_id = lsnr_ids.get((port_protocol, lbaas_port))
if listener_id is None:
LOG.warning("There is no listener associated to the protocol "
"%s and port %s. Skipping", port_protocol,
lbaas_port)
continue
self._apply_members_security_groups(lb, lbaas_port,
target_port, port_protocol,
sg_rule_name, listener_id, sgs)
def get_vifs(self, kuryrport_crd):
try:
pod = self.k8s.get(f"{constants.K8S_API_NAMESPACES}"
f"/{kuryrport_crd['metadata']['namespace']}"
f"/pods"
f"/{kuryrport_crd['metadata']['name']}")
except k_exc.K8sResourceNotFound as ex:
LOG.exception("Failed to get pod: %s", ex)
# TODO(gryf): Release resources
self.k8s.remove_finalizer(kuryrport_crd,
constants.KURYRPORT_FINALIZER)
raise
project_id = self._drv_project.get_project(pod)
security_groups = self._drv_sg.get_security_groups(pod, project_id)
try:
subnets = self._drv_subnets.get_subnets(pod, project_id)
except (os_exc.ResourceNotFound, k_exc.K8sResourceNotFound):
LOG.warning("Subnet does not exists. If namespace driver is "
"used, probably the namespace for the pod is "
"already deleted. So this pod does not need to "
"get a port as it will be deleted too. If the "
"default subnet driver is used, then you must "
"select an existing subnet to be used by Kuryr.")
return False
# Request the default interface of pod
try:
main_vif = self._drv_vif_pool.request_vif(pod, project_id,
subnets,
security_groups)
except os_exc.ResourceNotFound:
# NOTE(gryf): It might happen, that between getting security
# groups above and requesting VIF, network policy is deleted,
# hence we will get 404 from OpenStackSDK. Let's retry, to refresh
# information regarding SG.
LOG.warning("SG not found during VIF requesting. Retrying.")
raise k_exc.ResourceNotReady(pod['metadata']['name'])
if not main_vif:
pod_name = pod['metadata']['name']
LOG.warning("Ignoring event due to pod %s not being "
"scheduled yet.", pod_name)
return False
vifs = {constants.DEFAULT_IFNAME: {'default': True, 'vif': main_vif}}
# Request the additional interfaces from multiple drivers
index = 0
for driver in self._drv_multi_vif:
additional_vifs = driver.request_additional_vifs(pod, project_id,
security_groups)
for index, vif in enumerate(additional_vifs, start=index+1):
ifname = (oslo_cfg.CONF.kubernetes.additional_ifname_prefix +
str(index))
vifs[ifname] = {'default': False, 'vif': vif}
try:
self._update_kuryrport_crd(kuryrport_crd, vifs)
except k_exc.K8sClientException as ex:
LOG.exception("Kubernetes Client Exception creating "
"KuryrPort CRD: %s", ex)
for ifname, data in vifs.items():
self._drv_vif_pool.release_vif(pod, data['vif'],
project_id,
security_groups)
return True
def on_present(self, pod):
if (driver_utils.is_host_network(pod)
or not self._is_pod_scheduled(pod)):
# REVISIT(ivc): consider an additional configurable check that
# would allow skipping pods to enable heterogeneous environments
# where certain pods/namespaces/nodes can be managed by other
# networking solutions/CNI drivers.
return
state = driver_utils.get_pod_state(pod)
LOG.debug("Got VIFs from annotation: %r", state)
project_id = self._drv_project.get_project(pod)
security_groups = self._drv_sg.get_security_groups(pod, project_id)
if not state:
try:
subnets = self._drv_subnets.get_subnets(pod, project_id)
except (os_exc.ResourceNotFound, k_exc.K8sResourceNotFound):
LOG.warning("Subnet does not exists. If namespace driver is "
"used, probably the namespace for the pod is "
"already deleted. So this pod does not need to "
"get a port as it will be deleted too. If the "
"default subnet driver is used, then you must "
"select an existing subnet to be used by Kuryr.")
return
# Request the default interface of pod
main_vif = self._drv_vif_pool.request_vif(pod, project_id, subnets,
security_groups)
if not main_vif:
pod_name = pod['metadata']['name']
LOG.warning(
"Ignoring event due to pod %s not being "
"scheduled yet.", pod_name)
return
state = objects.vif.PodState(default_vif=main_vif)
# Request the additional interfaces from multiple dirvers
additional_vifs = []
for driver in self._drv_multi_vif:
additional_vifs.extend(
driver.request_additional_vifs(pod, project_id,
security_groups))
if additional_vifs:
state.additional_vifs = {}
for i, vif in enumerate(additional_vifs, start=1):
k = (oslo_cfg.CONF.kubernetes.additional_ifname_prefix +
str(i))
state.additional_vifs[k] = vif
try:
self._set_pod_state(pod, state)
except k_exc.K8sClientException as ex:
LOG.debug("Failed to set annotation: %s", ex)
# FIXME(ivc): improve granularity of K8sClient exceptions:
# only resourceVersion conflict should be ignored
for ifname, vif in state.vifs.items():
self._drv_vif_pool.release_vif(pod, vif, project_id,
security_groups)
else:
changed = False
try:
for ifname, vif in state.vifs.items():
if vif.plugin == constants.KURYR_VIF_TYPE_SRIOV:
driver_utils.update_port_pci_info(pod, vif)
if not vif.active:
try:
self._drv_vif_pool.activate_vif(pod, vif)
changed = True
except n_exc.PortNotFoundClient:
LOG.debug("Port not found, possibly already "
"deleted. No need to activate it")
finally:
if changed:
try:
self._set_pod_state(pod, state)
except k_exc.K8sResourceNotFound as ex:
LOG.exception("Failed to set annotation: %s", ex)
for ifname, vif in state.vifs.items():
self._drv_vif_pool.release_vif(
pod, vif, project_id, security_groups)
except k_exc.K8sClientException:
pod_name = pod['metadata']['name']
raise k_exc.ResourceNotReady(pod_name)
if self._is_network_policy_enabled():
crd_pod_selectors = self._drv_sg.create_sg_rules(pod)
if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules:
services = driver_utils.get_services()
self._update_services(services, crd_pod_selectors,
project_id)
def on_present(self, pod):
if driver_utils.is_host_network(pod) or not self._is_pending_node(pod):
# REVISIT(ivc): consider an additional configurable check that
# would allow skipping pods to enable heterogeneous environments
# where certain pods/namespaces/nodes can be managed by other
# networking solutions/CNI drivers.
return
state = driver_utils.get_pod_state(pod)
LOG.debug("Got VIFs from annotation: %r", state)
project_id = self._drv_project.get_project(pod)
security_groups = self._drv_sg.get_security_groups(pod, project_id)
if not state:
subnets = self._drv_subnets.get_subnets(pod, project_id)
# Request the default interface of pod
main_vif = self._drv_vif_pool.request_vif(pod, project_id, subnets,
security_groups)
if not main_vif:
pod_name = pod['metadata']['name']
LOG.warning(
"Ignoring event due to pod %s not being "
"scheduled yet.", pod_name)
return
state = objects.vif.PodState(default_vif=main_vif)
# Request the additional interfaces from multiple dirvers
additional_vifs = []
for driver in self._drv_multi_vif:
additional_vifs.extend(
driver.request_additional_vifs(pod, project_id,
security_groups))
if additional_vifs:
state.additional_vifs = {}
for i, vif in enumerate(additional_vifs, start=1):
k = constants.ADDITIONAL_IFNAME_PREFIX + str(i)
state.additional_vifs[k] = vif
try:
self._set_pod_state(pod, state)
except k_exc.K8sClientException as ex:
LOG.debug("Failed to set annotation: %s", ex)
# FIXME(ivc): improve granularity of K8sClient exceptions:
# only resourceVersion conflict should be ignored
for ifname, vif in state.vifs.items():
self._drv_vif_pool.release_vif(pod, vif, project_id,
security_groups)
else:
changed = False
try:
for ifname, vif in state.vifs.items():
if vif.plugin == constants.KURYR_VIF_TYPE_SRIOV:
driver_utils.update_port_pci_info(pod, vif)
if not vif.active:
self._drv_vif_pool.activate_vif(pod, vif)
changed = True
finally:
if changed:
try:
self._set_pod_state(pod, state)
except k_exc.K8sResourceNotFound as ex:
LOG.exception("Failed to set annotation: %s", ex)
for ifname, vif in state.vifs.items():
self._drv_vif_pool.release_vif(
pod, vif, project_id, security_groups)
except k_exc.K8sClientException:
pod_name = pod['metadata']['name']
raise k_exc.ResourceNotReady(pod_name)
if self._is_network_policy_enabled():
crd_pod_selectors = self._drv_sg.create_sg_rules(pod)
if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules:
services = driver_utils.get_services()
self._update_services(services, crd_pod_selectors,
project_id)
