def on_present(self, policy): LOG.debug("Created or updated: %s", policy) project_id = self._drv_project.get_project(policy) pods_to_update = [] modified_pods = self._drv_policy.ensure_network_policy(policy, project_id) if modified_pods: pods_to_update.extend(modified_pods) matched_pods = self._drv_policy.affected_pods(policy) pods_to_update.extend(matched_pods) for pod in pods_to_update: if driver_utils.is_host_network(pod): continue pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) if (pods_to_update and oslo_cfg.CONF.octavia_defaults.enforce_sg_rules and not self._is_egress_only_policy(policy)): # NOTE(ltomasbo): only need to change services if the pods that # they point to are updated services = driver_utils.get_services( policy['metadata']['namespace']) for service in services.get('items'): # TODO(ltomasbo): Skip other services that are not affected # by the policy if (not service['spec'].get('selector') or not self._is_service_affected(service, pods_to_update)): continue sgs = self._drv_svc_sg.get_security_groups(service, project_id) self._drv_lbaas.update_lbaas_sg(service, sgs)
def on_present(self, pod): if driver_utils.is_host_network(pod) or not self._has_pod_state(pod): # NOTE(ltomasbo): The event will be retried once the vif handler # annotates the pod with the pod state. return current_pod_labels = pod['metadata'].get('labels') previous_pod_labels = self._get_pod_labels(pod) LOG.debug("Got previous pod labels from annotation: %r", previous_pod_labels) if current_pod_labels == previous_pod_labels: return crd_pod_selectors = self._drv_sg.update_sg_rules(pod) project_id = self._drv_project.get_project(pod) security_groups = self._drv_sg.get_security_groups(pod, project_id) self._drv_vif_pool.update_vif_sgs(pod, security_groups) try: self._set_pod_labels(pod, current_pod_labels) except k_exc.K8sResourceNotFound: LOG.debug("Pod already deleted, no need to retry.") return if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)
def on_present(self, pod, *args, **kwargs): if driver_utils.is_host_network(pod) or not self._has_vifs(pod): # NOTE(ltomasbo): The event will be retried once the vif handler # annotates the pod with the pod state. return current_pod_info = (pod['metadata'].get('labels'), pod['status'].get('podIP')) previous_pod_info = self._get_pod_info(pod) LOG.debug("Got previous pod info from annotation: %r", previous_pod_info) if current_pod_info == previous_pod_info: return # FIXME(dulek): We should be able to just do create if only podIP # changed, right? crd_pod_selectors = self._drv_sg.update_sg_rules(pod) project_id = self._drv_project.get_project(pod) security_groups = self._drv_sg.get_security_groups(pod, project_id) self._drv_vif_pool.update_vif_sgs(pod, security_groups) try: self._set_pod_info(pod, current_pod_info) except k_exc.K8sResourceNotFound: LOG.debug("Pod already deleted, no need to retry.") return if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)
def on_deleted(self, policy): LOG.debug("Deleted network policy: %s", policy) project_id = self._drv_project.get_project(policy) pods_to_update = self._drv_policy.affected_pods(policy) netpolicy_crd = self._drv_policy.get_kuryrnetpolicy_crd(policy) if netpolicy_crd: crd_sg = netpolicy_crd['spec'].get('securityGroupId') for pod in pods_to_update: if driver_utils.is_host_network(pod): continue pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) if crd_sg in pod_sgs: pod_sgs.remove(crd_sg) if not pod_sgs: pod_sgs = ( oslo_cfg.CONF.neutron_defaults.pod_security_groups) if not pod_sgs: raise oslo_cfg.RequiredOptError( 'pod_security_groups', oslo_cfg.OptGroup('neutron_defaults')) self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) self._drv_policy.release_network_policy(netpolicy_crd) services = driver_utils.get_services( policy['metadata']['namespace']) for service in services.get('items'): if (service['metadata']['name'] == 'kubernetes' or not self._is_service_affected( service, pods_to_update)): continue sgs = self._drv_svc_sg.get_security_groups(service, project_id) self._drv_lbaas.update_lbaas_sg(service, sgs)
def on_finalize(self, kuryrnet_crd, *args, **kwargs): LOG.debug("Deleting kuryrnetwork CRD resources: %s", kuryrnet_crd) net_id = kuryrnet_crd.get('status', {}).get('netId') if net_id: self._drv_vif_pool.delete_network_pools(net_id) try: self._drv_subnets.delete_namespace_subnet(kuryrnet_crd) except k_exc.ResourceNotReady: LOG.debug("Subnet is not ready to be removed.") # TODO(ltomasbo): Once KuryrPort CRDs is supported, we should # execute a delete network ports method here to remove the # ports associated to the namespace/subnet, ensuring next # retry will be successful raise namespace = { 'metadata': {'name': kuryrnet_crd['spec']['nsName']}} crd_selectors = self._drv_sg.delete_namespace_sg_rules(namespace) if (driver_utils.is_network_policy_enabled() and crd_selectors and oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): project_id = kuryrnet_crd['spec']['projectId'] services = driver_utils.get_services() self._update_services(services, crd_selectors, project_id) LOG.debug('Removing finalizer for KuryrNet CRD %s', kuryrnet_crd) try: self.k8s.remove_finalizer(kuryrnet_crd, constants.KURYRNETWORK_FINALIZER) except k_exc.K8sClientException: LOG.exception('Error removing kuryrnetwork CRD finalizer for %s', kuryrnet_crd) raise
def on_present(self, policy): LOG.debug("Created or updated: %s", policy) project_id = self._drv_project.get_project(policy) pods_to_update = [] modified_pods = self._drv_policy.ensure_network_policy(policy, project_id) if modified_pods: pods_to_update.extend(modified_pods) matched_pods = self._drv_policy.affected_pods(policy) pods_to_update.extend(matched_pods) for pod in pods_to_update: if driver_utils.is_host_network(pod): continue pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) if pods_to_update: # NOTE(ltomasbo): only need to change services if the pods that # they point to are updated services = driver_utils.get_services( policy['metadata']['namespace']) for service in services.get('items'): # TODO(ltomasbo): Skip other services that are not affected # by the policy if service['metadata']['name'] == 'kubernetes': continue sgs = self._drv_svc_sg.get_security_groups(service, project_id) self._drv_lbaas.update_lbaas_sg(service, sgs)
def on_present(self, kuryrport_crd, *args, **kwargs): if not kuryrport_crd['status']['vifs']: # Get vifs if not self.get_vifs(kuryrport_crd): # Ignore this event, according to one of the cases logged in # get_vifs method. return retry_info = kwargs.get('retry_info') vifs = {ifname: {'default': data['default'], 'vif': objects.base.VersionedObject .obj_from_primitive(data['vif'])} for ifname, data in kuryrport_crd['status']['vifs'].items()} if all([v['vif'].active for v in vifs.values()]): return changed = False pod = self._get_pod(kuryrport_crd) try: for ifname, data in vifs.items(): if (data['vif'].plugin == constants.KURYR_VIF_TYPE_SRIOV and oslo_cfg.CONF.sriov.enable_node_annotations): pod_node = kuryrport_crd['spec']['podNodeName'] # TODO(gryf): This probably will need adoption, so it will # add information to CRD instead of the pod. driver_utils.update_port_pci_info(pod_node, data['vif']) if not data['vif'].active: try: self._drv_vif_pool.activate_vif(data['vif'], pod=pod, retry_info=retry_info) changed = True except os_exc.ResourceNotFound: LOG.debug("Port not found, possibly already deleted. " "No need to activate it") finally: if changed: project_id = self._drv_project.get_project(pod) try: self._update_kuryrport_crd(kuryrport_crd, vifs) except k_exc.K8sResourceNotFound as ex: LOG.exception("Failed to update KuryrPort CRD: %s", ex) security_groups = self._drv_sg.get_security_groups( pod, project_id) for ifname, data in vifs.items(): self._drv_vif_pool.release_vif(pod, data['vif'], project_id, security_groups) except k_exc.K8sClientException: raise k_exc.ResourceNotReady(pod['metadata']['name']) if driver_utils.is_network_policy_enabled(): crd_pod_selectors = self._drv_sg.create_sg_rules(pod) if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)
def on_deleted(self, pod): if driver_utils.is_host_network(pod): return project_id = self._drv_project.get_project(pod) crd_pod_selectors = self._drv_sg.delete_sg_rules(pod) try: security_groups = self._drv_sg.get_security_groups(pod, project_id) except k_exc.ResourceNotReady: # NOTE(ltomasbo): If the namespace object gets deleted first the # namespace security group driver will raise a ResourceNotReady # exception as it cannot access anymore the kuryrnet CRD annotated # on the namespace object. In such case we set security groups to # empty list so that if pools are enabled they will be properly # released. security_groups = [] state = driver_utils.get_pod_state(pod) LOG.debug("Got VIFs from annotation: %r", state) if state: for ifname, vif in state.vifs.items(): self._drv_vif_pool.release_vif(pod, vif, project_id, security_groups) if (self._is_network_policy_enabled() and oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)
def on_present(self, pod): if driver_utils.is_host_network(pod) or not self._is_pending_node(pod): # REVISIT(ivc): consider an additional configurable check that # would allow skipping pods to enable heterogeneous environments # where certain pods/namespaces/nodes can be managed by other # networking solutions/CNI drivers. return state = driver_utils.get_pod_state(pod) LOG.debug("Got VIFs from annotation: %r", state) project_id = self._drv_project.get_project(pod) if not state: security_groups = self._drv_sg.get_security_groups(pod, project_id) subnets = self._drv_subnets.get_subnets(pod, project_id) # Request the default interface of pod main_vif = self._drv_vif_pool.request_vif(pod, project_id, subnets, security_groups) state = objects.vif.PodState(default_vif=main_vif) # Request the additional interfaces from multiple dirvers additional_vifs = [] for driver in self._drv_multi_vif: additional_vifs.extend( driver.request_additional_vifs(pod, project_id, security_groups)) if additional_vifs: state.additional_vifs = {} for i, vif in enumerate(additional_vifs, start=1): k = constants.ADDITIONAL_IFNAME_PREFIX + str(i) state.additional_vifs[k] = vif try: self._set_pod_state(pod, state) except k_exc.K8sClientException as ex: LOG.debug("Failed to set annotation: %s", ex) # FIXME(ivc): improve granularity of K8sClient exceptions: # only resourceVersion conflict should be ignored for ifname, vif in state.vifs.items(): self._drv_vif_pool.release_vif(pod, vif, project_id, security_groups) else: changed = False try: for ifname, vif in state.vifs.items(): if vif.plugin == constants.KURYR_VIF_TYPE_SRIOV: driver_utils.update_port_pci_info(pod, vif) if not vif.active: self._drv_vif_pool.activate_vif(pod, vif) changed = True finally: if changed: self._set_pod_state(pod, state) if self._is_network_policy_enabled(): crd_pod_selectors = self._drv_sg.create_sg_rules(pod) if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)
def on_finalize(self, kuryrport_crd): name = kuryrport_crd['metadata']['name'] namespace = kuryrport_crd['metadata']['namespace'] try: pod = self.k8s.get(f"{constants.K8S_API_NAMESPACES}" f"/{namespace}/pods/{name}") except k_exc.K8sResourceNotFound as ex: LOG.exception("Failed to get pod: %s", ex) # TODO(gryf): Free resources self.k8s.remove_finalizer(kuryrport_crd, constants.POD_FINALIZER) raise if (driver_utils.is_host_network(pod) or not pod['spec'].get('nodeName')): return project_id = self._drv_project.get_project(pod) try: crd_pod_selectors = self._drv_sg.delete_sg_rules(pod) except k_exc.ResourceNotReady: # NOTE(ltomasbo): If the pod is being deleted before # kuryr-controller annotated any information about the port # associated, there is no need for deleting sg rules associated to # it. So this exception could be safetly ignored for the current # sg drivers. Only the NP driver associates rules to the pods ips, # and that waits for annotations to start. # # NOTE(gryf): perhaps we don't need to handle this case, since # during CRD creation all the things, including security groups # rules would be created too. LOG.debug("Skipping SG rules deletion associated to the pod %s", pod) crd_pod_selectors = [] try: security_groups = self._drv_sg.get_security_groups(pod, project_id) except k_exc.ResourceNotReady: # NOTE(ltomasbo): If the namespace object gets deleted first the # namespace security group driver will raise a ResourceNotReady # exception as it cannot access anymore the kuryrnetwork CRD # annotated on the namespace object. In such case we set security # groups to empty list so that if pools are enabled they will be # properly released. security_groups = [] for data in kuryrport_crd['spec']['vifs'].values(): vif = objects.base.VersionedObject.obj_from_primitive(data['vif']) self._drv_vif_pool.release_vif(pod, vif, project_id, security_groups) if (self._is_network_policy_enabled() and crd_pod_selectors and oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id) # Remove finalizer out of pod. self.k8s.remove_finalizer(pod, constants.POD_FINALIZER) # Finally, remove finalizer from KuryrPort CRD self.k8s.remove_finalizer(kuryrport_crd, constants.KURYRPORT_FINALIZER)
def on_present(self, namespace): ns_name = namespace['metadata']['name'] current_namespace_labels = namespace['metadata'].get('labels') previous_namespace_labels = driver_utils.get_annotated_labels( namespace, constants.K8S_ANNOTATION_NAMESPACE_LABEL) LOG.debug("Got previous namespace labels from annotation: %r", previous_namespace_labels) project_id = self._drv_project.get_project(namespace) if current_namespace_labels != previous_namespace_labels: crd_selectors = self._drv_sg.update_namespace_sg_rules(namespace) self._set_namespace_labels(namespace, current_namespace_labels) if (self._is_network_policy_enabled() and crd_selectors and oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): services = driver_utils.get_services() self._update_services(services, crd_selectors, project_id) net_crd_id = self._get_net_crd_id(namespace) if net_crd_id: LOG.debug("CRD existing at the new namespace") return net_crd_name = 'ns-' + ns_name net_crd = self._get_net_crd(net_crd_name) if net_crd: LOG.debug("Previous CRD existing at the new namespace. " "Deleting namespace resources and retying its creation.") self.on_deleted(namespace, net_crd) raise exceptions.ResourceNotReady(namespace) # NOTE(ltomasbo): Ensure there is no previously created networks # leftovers due to a kuryr-controller crash/restart LOG.debug("Deleting leftovers network resources for namespace: %s", ns_name) self._drv_subnets.cleanup_namespace_networks(ns_name) LOG.debug("Creating network resources for namespace: %s", ns_name) net_crd_spec = self._drv_subnets.create_namespace_network( ns_name, project_id) # create CRD resource for the network try: net_crd = self._add_kuryrnet_crd(ns_name, net_crd_spec) self._drv_sg.create_namespace_sg_rules(namespace) self._set_net_crd(namespace, net_crd) except (exceptions.K8sClientException, exceptions.K8sResourceNotFound): LOG.exception("Kuryrnet CRD creation failed. Rolling back " "resources created for the namespace.") self._drv_subnets.rollback_network_resources(net_crd_spec, ns_name) try: self._del_kuryrnet_crd(net_crd_name) except exceptions.K8sClientException: LOG.exception( "Error when trying to rollback the KuryrNet CRD " "object %s", net_crd_name) raise exceptions.ResourceNotReady(namespace)
def _create_svc_egress_sg_rule(self, policy_namespace, sg_rule_body_list, resource=None, port=None, protocol=None): services = driver_utils.get_services() if not resource: svc_subnet = utils.get_subnet_cidr( CONF.neutron_defaults.service_subnet) rule = driver_utils.create_security_group_rule_body( 'egress', port, protocol=protocol, cidr=svc_subnet) if rule not in sg_rule_body_list: sg_rule_body_list.append(rule) return for service in services.get('items'): if self._is_pod(resource): pod_labels = resource['metadata'].get('labels') svc_selector = service['spec'].get('selector') if not svc_selector or not pod_labels: continue else: if not driver_utils.match_labels(svc_selector, pod_labels): continue elif resource.get('cidr'): # NOTE(maysams) Accounts for traffic to pods under # a service matching an IPBlock rule. svc_namespace = service['metadata']['namespace'] if svc_namespace != policy_namespace: continue svc_selector = service['spec'].get('selector') pods = driver_utils.get_pods({ 'selector': svc_selector }, svc_namespace).get('items') if not self._pods_in_ip_block(pods, resource): continue else: ns_name = service['metadata']['namespace'] if ns_name != resource['metadata']['name']: continue cluster_ip = service['spec'].get('clusterIP') if not cluster_ip: continue rule = driver_utils.create_security_group_rule_body( 'egress', port, protocol=protocol, cidr=cluster_ip) if rule not in sg_rule_body_list: sg_rule_body_list.append(rule)
def on_deleted(self, policy): LOG.debug("Deleted network policy: %s", policy) project_id = self._drv_project.get_project(policy) pods_to_update = self._drv_policy.affected_pods(policy) netpolicy_crd = self._drv_policy.get_kuryrnetpolicy_crd(policy) if netpolicy_crd: crd_sg = netpolicy_crd['spec'].get('securityGroupId') for pod in pods_to_update: if driver_utils.is_host_network(pod): continue pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) if crd_sg in pod_sgs: pod_sgs.remove(crd_sg) if not pod_sgs: pod_sgs = ( oslo_cfg.CONF.neutron_defaults.pod_security_groups) if not pod_sgs: raise oslo_cfg.RequiredOptError( 'pod_security_groups', oslo_cfg.OptGroup('neutron_defaults')) try: self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) except os_exc.NotFoundException: LOG.debug("Fail to update pod sgs." " Retrying policy deletion.") raise exceptions.ResourceNotReady(policy) # ensure ports at the pool don't have the NP sg associated net_id = self._get_policy_net_id(policy) self._drv_vif_pool.remove_sg_from_pools(crd_sg, net_id) self._drv_policy.release_network_policy(netpolicy_crd) if (oslo_cfg.CONF.octavia_defaults.enforce_sg_rules and not self._is_egress_only_policy(policy)): services = driver_utils.get_services( policy['metadata']['namespace']) for svc in services.get('items'): if (not svc['spec'].get('selector') or not self._is_service_affected(svc, pods_to_update)): continue sgs = self._drv_svc_sg.get_security_groups(svc, project_id) self._drv_lbaas.update_lbaas_sg(svc, sgs)
def on_deleted(self, namespace, net_crd=None): LOG.debug("Deleting namespace: %s", namespace) if not net_crd: net_crd_id = self._get_net_crd_id(namespace) if not net_crd_id: LOG.warning("There is no CRD annotated at the namespace %s", namespace) return net_crd = self._get_net_crd(net_crd_id) if not net_crd: LOG.warning("This should not happen. Probably this is event " "is processed twice due to a restart or etcd is " "not in sync") # NOTE(ltomasbo): We should rely on etcd properly behaving, so # we are returning here to prevent duplicated events processing # but not to prevent etcd failures. return net_crd_name = net_crd['metadata']['name'] self._drv_vif_pool.delete_network_pools(net_crd['spec']['netId']) try: self._drv_subnets.delete_namespace_subnet(net_crd) except exceptions.ResourceNotReady: LOG.debug("Subnet is not ready to be removed.") # TODO(ltomasbo): Once KuryrPort CRDs is supported, we should # execute a delete network ports method here to remove the ports # associated to the namespace/subnet, ensuring next retry will be # successful raise sg_id = net_crd['spec'].get('sgId') if sg_id: self._drv_sg.delete_sg(sg_id) else: LOG.debug("There is no security group associated with the " "namespace to be deleted") self._del_kuryrnet_crd(net_crd_name) crd_selectors = self._drv_sg.delete_namespace_sg_rules(namespace) if (self._is_network_policy_enabled() and crd_selectors and oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): project_id = self._drv_project.get_project(namespace) services = driver_utils.get_services() self._update_services(services, crd_selectors, project_id)
def _cleanup_leftover_lbaas(self): lbaas_client = clients.get_loadbalancer_client() services = [] try: services = driver_utils.get_services().get('items') except k_exc.K8sClientException: LOG.debug("Skipping cleanup of leftover lbaas. " "Error retriving Kubernetes services") return services_cluster_ip = set(service['spec']['clusterIP'] for service in services if service['spec'].get('clusterIP')) lbaas_spec = {} self._drv_lbaas.add_tags('loadbalancer', lbaas_spec) loadbalancers = lbaas_client.load_balancers(**lbaas_spec) for loadbalancer in loadbalancers: if loadbalancer.vip_address not in services_cluster_ip: lb_obj = obj_lbaas.LBaaSLoadBalancer(**loadbalancer) eventlet.spawn(self._ensure_release_lbaas, lb_obj)
def on_deleted(self, pod): if (driver_utils.is_host_network(pod) or not pod['spec'].get('nodeName')): return project_id = self._drv_project.get_project(pod) try: crd_pod_selectors = self._drv_sg.delete_sg_rules(pod) except k_exc.ResourceNotReady: # NOTE(ltomasbo): If the pod is being deleted before # kuryr-controller annotated any information about the port # associated, there is no need for deleting sg rules associated to # it. So this exception could be safetly ignored for the current # sg drivers. Only the NP driver associates rules to the pods ips, # and that waits for annotations to start. LOG.debug( "Pod was not yet annotated by Kuryr-controller. " "Skipping SG rules deletion associated to the pod %s", pod) crd_pod_selectors = [] try: security_groups = self._drv_sg.get_security_groups(pod, project_id) except k_exc.ResourceNotReady: # NOTE(ltomasbo): If the namespace object gets deleted first the # namespace security group driver will raise a ResourceNotReady # exception as it cannot access anymore the kuryrnet CRD annotated # on the namespace object. In such case we set security groups to # empty list so that if pools are enabled they will be properly # released. security_groups = [] state = driver_utils.get_pod_state(pod) LOG.debug("Got VIFs from annotation: %r", state) if state: for ifname, vif in state.vifs.items(): self._drv_vif_pool.release_vif(pod, vif, project_id, security_groups) if (self._is_network_policy_enabled() and crd_pod_selectors and oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)
def on_present(self, kuryrnet_crd): ns_name = kuryrnet_crd['spec']['nsName'] project_id = kuryrnet_crd['spec']['projectId'] kns_status = kuryrnet_crd.get('status', {}) crd_creation = False net_id = kns_status.get('netId') if not net_id: net_id = self._drv_subnets.create_network(ns_name, project_id) status = {'netId': net_id} self._patch_kuryrnetwork_crd(kuryrnet_crd, status) crd_creation = True subnet_id = kns_status.get('subnetId') if not subnet_id or crd_creation: subnet_id, subnet_cidr = self._drv_subnets.create_subnet( ns_name, project_id, net_id) status = {'subnetId': subnet_id, 'subnetCIDR': subnet_cidr} self._patch_kuryrnetwork_crd(kuryrnet_crd, status) crd_creation = True if not kns_status.get('routerId') or crd_creation: router_id = self._drv_subnets.add_subnet_to_router(subnet_id) status = {'routerId': router_id, 'populated': False} self._patch_kuryrnetwork_crd(kuryrnet_crd, status) crd_creation = True # check labels to create sg rules ns_labels = kns_status.get('nsLabels', {}) if (crd_creation or ns_labels != kuryrnet_crd['spec']['nsLabels']): # update SG and svc SGs namespace = driver_utils.get_namespace(ns_name) crd_selectors = self._drv_sg.update_namespace_sg_rules(namespace) if (self._is_network_policy_enabled() and crd_selectors and oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): services = driver_utils.get_services() self._update_services(services, crd_selectors, project_id) # update status status = {'nsLabels': kuryrnet_crd['spec']['nsLabels']} self._patch_kuryrnetwork_crd(kuryrnet_crd, status, labels=True)
def _cleanup_leftover_lbaas(self): lbaas_client = clients.get_loadbalancer_client() services = [] try: services = driver_utils.get_services().get('items') except k_exc.K8sClientException: LOG.debug("Skipping cleanup of leftover lbaas. " "Error retriving Kubernetes services") return services_cluster_ip = { service['spec']['clusterIP']: service for service in services if service['spec'].get('clusterIP') } services_without_selector = set( service['spec']['clusterIP'] for service in services if (service['spec'].get('clusterIP') and not service['spec'].get('selector'))) lbaas_spec = {} self._drv_lbaas.add_tags('loadbalancer', lbaas_spec) loadbalancers = lbaas_client.load_balancers(**lbaas_spec) for loadbalancer in loadbalancers: if loadbalancer.vip_address not in services_cluster_ip.keys(): lb_obj = obj_lbaas.LBaaSLoadBalancer(**loadbalancer) eventlet.spawn(self._ensure_release_lbaas, lb_obj) else: # check if the provider is the right one if (loadbalancer.vip_address not in services_without_selector and self._lb_provider and self._lb_provider != loadbalancer.provider): LOG.debug("Removing loadbalancer with old provider: %s", loadbalancer) lb_obj = obj_lbaas.LBaaSLoadBalancer(**loadbalancer) eventlet.spawn( self._ensure_release_lbaas, lb_obj, services_cluster_ip[loadbalancer.vip_address]) # NOTE(ltomasbo): give some extra time in between lbs # recreation actions time.sleep(1)
def on_present(self, pod): if driver_utils.is_host_network(pod) or not self._has_pod_state(pod): # NOTE(ltomasbo): The event will be retried once the vif handler # annotates the pod with the pod state. return current_pod_labels = pod['metadata'].get('labels') previous_pod_labels = self._get_pod_labels(pod) LOG.debug("Got previous pod labels from annotation: %r", previous_pod_labels) if current_pod_labels == previous_pod_labels: return crd_pod_selectors = self._drv_sg.update_sg_rules(pod) project_id = self._drv_project.get_project(pod) security_groups = self._drv_sg.get_security_groups(pod, project_id) self._drv_vif_pool.update_vif_sgs(pod, security_groups) self._set_pod_labels(pod, current_pod_labels) services = driver_utils.get_services(pod['metadata']['namespace']) self._update_services(services, crd_pod_selectors, project_id)
def on_present(self, pod): if (driver_utils.is_host_network(pod) or not self._is_pod_scheduled(pod)): # REVISIT(ivc): consider an additional configurable check that # would allow skipping pods to enable heterogeneous environments # where certain pods/namespaces/nodes can be managed by other # networking solutions/CNI drivers. return state = driver_utils.get_pod_state(pod) LOG.debug("Got VIFs from annotation: %r", state) project_id = self._drv_project.get_project(pod) security_groups = self._drv_sg.get_security_groups(pod, project_id) if not state: try: subnets = self._drv_subnets.get_subnets(pod, project_id) except (os_exc.ResourceNotFound, k_exc.K8sResourceNotFound): LOG.warning("Subnet does not exists. If namespace driver is " "used, probably the namespace for the pod is " "already deleted. So this pod does not need to " "get a port as it will be deleted too. If the " "default subnet driver is used, then you must " "select an existing subnet to be used by Kuryr.") return # Request the default interface of pod main_vif = self._drv_vif_pool.request_vif(pod, project_id, subnets, security_groups) if not main_vif: pod_name = pod['metadata']['name'] LOG.warning( "Ignoring event due to pod %s not being " "scheduled yet.", pod_name) return state = objects.vif.PodState(default_vif=main_vif) # Request the additional interfaces from multiple dirvers additional_vifs = [] for driver in self._drv_multi_vif: additional_vifs.extend( driver.request_additional_vifs(pod, project_id, security_groups)) if additional_vifs: state.additional_vifs = {} for i, vif in enumerate(additional_vifs, start=1): k = (oslo_cfg.CONF.kubernetes.additional_ifname_prefix + str(i)) state.additional_vifs[k] = vif try: self._set_pod_state(pod, state) except k_exc.K8sClientException as ex: LOG.debug("Failed to set annotation: %s", ex) # FIXME(ivc): improve granularity of K8sClient exceptions: # only resourceVersion conflict should be ignored for ifname, vif in state.vifs.items(): self._drv_vif_pool.release_vif(pod, vif, project_id, security_groups) else: changed = False try: for ifname, vif in state.vifs.items(): if vif.plugin == constants.KURYR_VIF_TYPE_SRIOV: driver_utils.update_port_pci_info(pod, vif) if not vif.active: try: self._drv_vif_pool.activate_vif(pod, vif) changed = True except n_exc.PortNotFoundClient: LOG.debug("Port not found, possibly already " "deleted. No need to activate it") finally: if changed: try: self._set_pod_state(pod, state) except k_exc.K8sResourceNotFound as ex: LOG.exception("Failed to set annotation: %s", ex) for ifname, vif in state.vifs.items(): self._drv_vif_pool.release_vif( pod, vif, project_id, security_groups) except k_exc.K8sClientException: pod_name = pod['metadata']['name'] raise k_exc.ResourceNotReady(pod_name) if self._is_network_policy_enabled(): crd_pod_selectors = self._drv_sg.create_sg_rules(pod) if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)
def on_present(self, kuryrnet_crd, *args, **kwargs): ns_name = kuryrnet_crd['spec']['nsName'] project_id = kuryrnet_crd['spec']['projectId'] kns_status = kuryrnet_crd.get('status', {}) namespace = driver_utils.get_namespace(ns_name) crd_creation = False net_id = kns_status.get('netId') if not net_id: try: net_id = self._drv_subnets.create_network(namespace, project_id) except os_exc.SDKException as ex: self.k8s.add_event(kuryrnet_crd, 'CreateNetworkFailed', f'Error during creating Neutron network: ' f'{ex.details}', 'Warning') raise status = {'netId': net_id} self._patch_kuryrnetwork_crd(kuryrnet_crd, status) self.k8s.add_event(kuryrnet_crd, 'CreateNetworkSucceed', f'Neutron network {net_id} for namespace') crd_creation = True subnet_id = kns_status.get('subnetId') if not subnet_id or crd_creation: try: subnet_id, subnet_cidr = self._drv_subnets.create_subnet( namespace, project_id, net_id) except os_exc.ConflictException as ex: self.k8s.add_event(kuryrnet_crd, 'CreateSubnetFailed', f'Error during creating Neutron subnet ' f'for network {net_id}: {ex.details}', 'Warning') raise status = {'subnetId': subnet_id, 'subnetCIDR': subnet_cidr} self._patch_kuryrnetwork_crd(kuryrnet_crd, status) self.k8s.add_event(kuryrnet_crd, 'CreateSubnetSucceed', f'Neutron subnet {subnet_id} for network ' f'{net_id}') crd_creation = True if not kns_status.get('routerId') or crd_creation: try: router_id = self._drv_subnets.add_subnet_to_router(subnet_id) except os_exc.SDKException as ex: self.k8s.add_event(kuryrnet_crd, 'AddingSubnetToRouterFailed', f'Error adding Neutron subnet {subnet_id} ' f'to router {router_id}: {ex.details}', 'Warning') raise status = {'routerId': router_id, 'populated': False} self._patch_kuryrnetwork_crd(kuryrnet_crd, status) self.k8s.add_event(kuryrnet_crd, 'AddingSubnetToRouterSucceed', f'Neutron subnet {subnet_id} added to router ' f'{router_id}') crd_creation = True # check labels to create sg rules ns_labels = kns_status.get('nsLabels', {}) if (crd_creation or ns_labels != kuryrnet_crd['spec']['nsLabels']): # update SG and svc SGs crd_selectors = self._drv_sg.update_namespace_sg_rules(namespace) if (driver_utils.is_network_policy_enabled() and crd_selectors and oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): services = driver_utils.get_services() self._update_services(services, crd_selectors, project_id) # update status status = {'nsLabels': kuryrnet_crd['spec']['nsLabels']} self._patch_kuryrnetwork_crd(kuryrnet_crd, status, labels=True) self.k8s.add_event(kuryrnet_crd, 'SGUpdateTriggered', 'Neutron security groups update has been ' 'triggered')
def _create_svc_egress_sg_rule(self, policy_namespace, sg_rule_body_list, resource=None, port=None, protocol=None): # FIXME(dulek): We could probably filter by namespace here for pods # and namespace resources? services = driver_utils.get_services() if not resource: svc_subnet = utils.get_subnet_cidr( CONF.neutron_defaults.service_subnet) rule = driver_utils.create_security_group_rule_body( 'egress', port, protocol=protocol, cidr=svc_subnet) if rule not in sg_rule_body_list: sg_rule_body_list.append(rule) return for service in services.get('items'): if service['metadata'].get('deletionTimestamp'): # Ignore services being deleted continue cluster_ip = service['spec'].get('clusterIP') if not cluster_ip or cluster_ip == 'None': # Headless services has 'None' as clusterIP, ignore. continue svc_name = service['metadata']['name'] svc_namespace = service['metadata']['namespace'] if self._is_pod(resource): pod_labels = resource['metadata'].get('labels') svc_selector = service['spec'].get('selector') if not svc_selector: targets = driver_utils.get_endpoints_targets( svc_name, svc_namespace) pod_ip = resource['status'].get('podIP') if pod_ip and pod_ip not in targets: continue elif pod_labels: if not driver_utils.match_labels(svc_selector, pod_labels): continue elif resource.get('cidr'): # NOTE(maysams) Accounts for traffic to pods under # a service matching an IPBlock rule. svc_selector = service['spec'].get('selector') if not svc_selector: # Retrieving targets of services on any Namespace targets = driver_utils.get_endpoints_targets( svc_name, svc_namespace) if (not targets or not self._targets_in_ip_block(targets, resource)): continue else: if svc_namespace != policy_namespace: continue pods = driver_utils.get_pods({ 'selector': svc_selector }, svc_namespace).get('items') if not self._pods_in_ip_block(pods, resource): continue else: ns_name = service['metadata']['namespace'] if ns_name != resource['metadata']['name']: continue rule = driver_utils.create_security_group_rule_body( 'egress', port, protocol=protocol, cidr=cluster_ip) if rule not in sg_rule_body_list: sg_rule_body_list.append(rule)
def on_finalize(self, knp): LOG.debug("Finalizing KuryrNetworkPolicy %s", knp) project_id = self._drv_project.get_project(knp) pods_to_update = self._drv_policy.affected_pods(knp) crd_sg = knp['status'].get('securityGroupId') try: policy = self._get_networkpolicy( knp['metadata']['annotations']['networkPolicyLink']) except exceptions.K8sResourceNotFound: # NP is already gone, let's just try to clean up. policy = None if crd_sg: for pod in pods_to_update: if driver_utils.is_host_network(pod): continue pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) if crd_sg in pod_sgs: pod_sgs.remove(crd_sg) if not pod_sgs: pod_sgs = CONF.neutron_defaults.pod_security_groups if not pod_sgs: raise cfg.RequiredOptError( 'pod_security_groups', cfg.OptGroup('neutron_defaults')) try: self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) except os_exc.NotFoundException: # Pod got deleted in the meanwhile, safe to ignore. pass # ensure ports at the pool don't have the NP sg associated try: net_id = self._get_policy_net_id(knp) self._drv_vif_pool.remove_sg_from_pools(crd_sg, net_id) except exceptions.K8sResourceNotFound: # Probably the network got removed already, we can ignore it. pass if (CONF.octavia_defaults.enforce_sg_rules and policy and not self._is_egress_only_policy(policy)): services = driver_utils.get_services( knp['metadata']['namespace']) for svc in services.get('items'): if (not svc['spec'].get('selector') or not self._is_service_affected( svc, pods_to_update)): continue sgs = self._drv_svc_sg.get_security_groups(svc, project_id) try: self._drv_lbaas.update_lbaas_sg(svc, sgs) except exceptions.ResourceNotReady: # We can ignore LB that's being created - its SGs will # get handled when members will be getting created. pass self._drv_policy.delete_np_sg(crd_sg) LOG.debug("Removing finalizers from KuryrNetworkPolicy and " "NetworkPolicy.") if policy: self.k8s.remove_finalizer(policy, constants.NETWORKPOLICY_FINALIZER) self.k8s.remove_finalizer(knp, constants.NETWORKPOLICY_FINALIZER)
def on_present(self, knp): uniq_name = utils.get_res_unique_name(knp) LOG.debug('on_present() for NP %s', uniq_name) project_id = self._drv_project.get_project(knp) if not knp['status'].get('securityGroupId'): LOG.debug('Creating SG for NP %s', uniq_name) # TODO(dulek): Do this right, why do we have a project driver per # resource?! This one expects policy, not knp, but it # ignores it anyway! sg_id = self._drv_policy.create_security_group(knp, project_id) knp = self._patch_kuryrnetworkpolicy_crd( knp, 'status', {'securityGroupId': sg_id}) LOG.debug('Created SG %s for NP %s', sg_id, uniq_name) else: # TODO(dulek): Check if it really exists, recreate if not. sg_id = knp['status'].get('securityGroupId') # First update SG rules as we want to apply updated ones current = knp['status']['securityGroupRules'] required = knp['spec']['ingressSgRules'] + knp['spec']['egressSgRules'] required = [r['sgRule'] for r in required] # FIXME(dulek): This *might* be prone to race conditions if failure # happens between SG rule is created/deleted and status # is annotated. We don't however need to revert on failed # K8s operations - creation, deletion of SG rules and # attaching or detaching SG from ports are idempotent # so we can repeat them. What worries me is losing track # of an update due to restart. The only way to do it # would be to periodically check if what's in `status` # is the reality in OpenStack API. That should be just # two Neutron API calls + possible resync. to_add = [] to_remove = [] for r in required: if not self._find_sgs(r, current): to_add.append(r) for i, c in enumerate(current): if not self._find_sgs(c, required): to_remove.append((i, c['id'])) LOG.debug('SGs to add for NP %s: %s', uniq_name, to_add) for sg_rule in to_add: LOG.debug('Adding SG rule %s for NP %s', sg_rule, uniq_name) sg_rule['security_group_id'] = sg_id sgr_id = driver_utils.create_security_group_rule(sg_rule) sg_rule['id'] = sgr_id knp = self._patch_kuryrnetworkpolicy_crd( knp, 'status', {'securityGroupRules/-': sg_rule}, 'add') # We need to remove starting from the last one in order to maintain # indexes. Please note this will start to fail miserably if we start # to change status from multiple places. to_remove.reverse() LOG.debug('SGs to remove for NP %s: %s', uniq_name, [x[1] for x in to_remove]) for i, sg_rule_id in to_remove: LOG.debug('Removing SG rule %s as it is no longer part of NP %s', sg_rule_id, uniq_name) driver_utils.delete_security_group_rule(sg_rule_id) knp = self._patch_kuryrnetworkpolicy_crd( knp, 'status/securityGroupRules', i, 'remove') pods_to_update = [] previous_sel = knp['status'].get('podSelector', None) current_sel = knp['spec']['podSelector'] if previous_sel is None: # Fresh NetworkPolicy that was never applied. pods_to_update.extend(self._drv_policy.namespaced_pods(knp)) elif previous_sel != current_sel or previous_sel == {}: pods_to_update.extend( self._drv_policy.affected_pods(knp, previous_sel)) matched_pods = self._drv_policy.affected_pods(knp) pods_to_update.extend(matched_pods) for pod in pods_to_update: if driver_utils.is_host_network(pod): continue pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) try: self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) except os_exc.NotFoundException: # Pod got deleted in the meanwhile, should be safe to ignore. pass # FIXME(dulek): We should not need this one day. policy = self._get_networkpolicy( knp['metadata']['annotations']['networkPolicyLink']) if (pods_to_update and CONF.octavia_defaults.enforce_sg_rules and not self._is_egress_only_policy(policy)): # NOTE(ltomasbo): only need to change services if the pods that # they point to are updated services = driver_utils.get_services(knp['metadata']['namespace']) for service in services.get('items', []): # TODO(ltomasbo): Skip other services that are not affected # by the policy # FIXME(dulek): Make sure to include svcs without selector when # we start supporting them. if (not service['spec'].get('selector') or not self._is_service_affected( service, pods_to_update)): continue sgs = self._drv_svc_sg.get_security_groups(service, project_id) try: self._drv_lbaas.update_lbaas_sg(service, sgs) except exceptions.ResourceNotReady: # We can ignore LB that's being created - its SGs will get # handled when members will be getting created. pass self._patch_kuryrnetworkpolicy_crd(knp, 'status', {'podSelector': current_sel})
def on_finalize(self, kuryrport_crd): name = kuryrport_crd['metadata']['name'] namespace = kuryrport_crd['metadata']['namespace'] try: pod = self.k8s.get(f"{constants.K8S_API_NAMESPACES}" f"/{namespace}/pods/{name}") except k_exc.K8sResourceNotFound: LOG.error("Pod %s/%s doesn't exists, deleting orphaned KuryrPort", namespace, name) # TODO(gryf): Free resources try: self.k8s.remove_finalizer(kuryrport_crd, constants.KURYRPORT_FINALIZER) except k_exc.K8sClientException as ex: LOG.exception("Failed to remove finalizer from KuryrPort %s", ex) raise return # FIXME(dulek): hostNetwork condition can be removed once we know we # won't upgrade from version creating ports for host # networking pods. if ('deletionTimestamp' not in pod['metadata'] and not driver_utils.is_host_network(pod)): # NOTE(gryf): Ignore deleting KuryrPort, since most likely it was # removed manually, while we need vifs for corresponding pod # object which apperantely is still running. LOG.warning('Manually triggered KuryrPort %s removal. This ' 'action should be avoided, since KuryrPort CRDs are ' 'internal to Kuryr.', name) return project_id = self._drv_project.get_project(pod) try: crd_pod_selectors = self._drv_sg.delete_sg_rules(pod) except k_exc.ResourceNotReady: # NOTE(ltomasbo): If the pod is being deleted before # kuryr-controller annotated any information about the port # associated, there is no need for deleting sg rules associated to # it. So this exception could be safetly ignored for the current # sg drivers. Only the NP driver associates rules to the pods ips, # and that waits for annotations to start. # # NOTE(gryf): perhaps we don't need to handle this case, since # during CRD creation all the things, including security groups # rules would be created too. LOG.debug("Skipping SG rules deletion associated to the pod %s", pod) crd_pod_selectors = [] try: security_groups = self._drv_sg.get_security_groups(pod, project_id) except k_exc.ResourceNotReady: # NOTE(ltomasbo): If the namespace object gets deleted first the # namespace security group driver will raise a ResourceNotReady # exception as it cannot access anymore the kuryrnetwork CRD # annotated on the namespace object. In such case we set security # groups to empty list so that if pools are enabled they will be # properly released. security_groups = [] for data in kuryrport_crd['status']['vifs'].values(): vif = objects.base.VersionedObject.obj_from_primitive(data['vif']) self._drv_vif_pool.release_vif(pod, vif, project_id, security_groups) if (self._is_network_policy_enabled() and crd_pod_selectors and oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id) # Remove finalizer out of pod. self.k8s.remove_finalizer(pod, constants.POD_FINALIZER) # Finally, remove finalizer from KuryrPort CRD self.k8s.remove_finalizer(kuryrport_crd, constants.KURYRPORT_FINALIZER)