def _update_listener_acls(self, loadbalancer, listener_id, allowed_cidrs): admin_state_up = True if allowed_cidrs is None: # World accessible, no restriction on the listeners pass elif len(allowed_cidrs) == 0: # Prevent any traffic as no CIDR is allowed admin_state_up = False request = { 'allowed_cidrs': allowed_cidrs, 'admin_state_up': admin_state_up, } # Wait for the loadbalancer to be ACTIVE self._wait_for_provisioning(loadbalancer, _ACTIVATION_TIMEOUT, _LB_STS_POLL_FAST_INTERVAL) lbaas = clients.get_loadbalancer_client() try: lbaas.update_listener(listener_id, **request) except os_exc.SDKException: LOG.exception('Error when updating listener %s' % listener_id) raise k_exc.ResourceNotReady(listener_id)
def _update_crd_spec(self, loadbalancer_crd, service): svc_ip = self._get_service_ip(service) ports = service['spec'].get('ports') for port in ports: if type(port['targetPort']) == int: port['targetPort'] = str(port['targetPort']) project_id = self._drv_project.get_project(service) sg_ids = self._drv_sg.get_security_groups(service, project_id) subnet_id = self._get_subnet_id(service, project_id, svc_ip) spec_type = service['spec'].get('type') kubernetes = clients.get_kubernetes_client() patch = { 'spec': { 'ip': svc_ip, 'ports': ports, 'project_id': project_id, 'security_groups_ids': sg_ids, 'subnet_id': subnet_id, 'type': spec_type } } LOG.debug('Patching KuryrLoadBalancer CRD %s', loadbalancer_crd) try: kubernetes.patch_crd('spec', loadbalancer_crd['metadata']['selfLink'], patch['spec']) except k_exc.K8sResourceNotFound: LOG.debug('KuryrLoadBalancer CRD not found %s', loadbalancer_crd) except k_exc.K8sConflict: raise k_exc.ResourceNotReady(loadbalancer_crd) except k_exc.K8sClientException: LOG.exception('Error updating kuryrnet CRD %s', loadbalancer_crd) raise return loadbalancer_crd
def add(self, params): vifs = self._do_work(params, b_base.connect) pod_name = self._get_pod_name(params) # NOTE(dulek): Saving containerid to be able to distinguish old DEL # requests that we should ignore. We need a lock to # prevent race conditions and replace whole object in the # dict for multiprocessing.Manager to notice that. with lockutils.lock(pod_name, external=True): d = self.registry[pod_name] d['containerid'] = params.CNI_CONTAINERID self.registry[pod_name] = d LOG.debug('Saved containerid = %s for pod %s', params.CNI_CONTAINERID, pod_name) # Wait for VIFs to become active. timeout = CONF.cni_daemon.vif_annotation_timeout # Wait for timeout sec, 1 sec between tries, retry when even one # vif is not active. @retrying.retry(stop_max_delay=timeout * 1000, wait_fixed=RETRY_DELAY, retry_on_result=lambda x: any( map(lambda y: not y[1].active, x.items()))) def wait_for_active(pod_name): return { ifname: base.VersionedObject.obj_from_primitive(vif_obj) for ifname, vif_obj in self.registry[pod_name]['vifs'].items() } vifs = wait_for_active(pod_name) for vif in vifs.values(): if not vif.active: raise exceptions.ResourceNotReady(pod_name) return vifs[k_const.DEFAULT_IFNAME]
def delete_network_pools(self, net_id): if not hasattr(self, '_available_ports_pools'): LOG.info("Kuryr-controller not yet ready to delete network " "pools.") raise exceptions.ResourceNotReady(net_id) neutron = clients.get_neutron_client() # NOTE(ltomasbo): Note the pods should already be deleted, but their # associated ports may not have been recycled yet, therefore not being # on the available_ports_pools dict. The next call forces it to be on # that dict before cleaning it up self._trigger_return_to_pool() for pool_key, ports_ids in self._available_ports_pools.items(): if self._get_pool_key_net(pool_key) != net_id: continue self._available_ports_pools[pool_key] = [] trunk_id = self._get_trunk_id(neutron, pool_key) try: self._drv_vif._remove_subports(neutron, trunk_id, ports_ids) except n_exc.NeutronClientException: LOG.exception('Error removing subports from trunk: %s', trunk_id) continue for port_id in ports_ids: try: self._drv_vif._release_vlan_id( self._existing_vifs[port_id].vlan_id) del self._existing_vifs[port_id] except KeyError: LOG.debug('Port %s is not in the ports list.', port_id) try: neutron.delete_port(port_id) except n_exc.PortNotFoundClient: LOG.debug( 'Unable to delete subport %s as it no longer ' 'exists.', port_id)
def _set_lbaas_spec(self, service, lbaas_spec): # TODO(ivc): extract annotation interactions if lbaas_spec is None: LOG.debug("Removing LBaaSServiceSpec annotation: %r", lbaas_spec) annotation = None else: lbaas_spec.obj_reset_changes(recursive=True) LOG.debug("Setting LBaaSServiceSpec annotation: %r", lbaas_spec) annotation = jsonutils.dumps(lbaas_spec.obj_to_primitive(), sort_keys=True) svc_link = service['metadata']['selfLink'] ep_link = self._get_endpoints_link(service) k8s = clients.get_kubernetes_client() try: k8s.annotate(ep_link, {k_const.K8S_ANNOTATION_LBAAS_SPEC: annotation}) except k_exc.K8sClientException: # REVISIT(ivc): only raise ResourceNotReady for NotFound raise k_exc.ResourceNotReady(ep_link) k8s.annotate(svc_link, {k_const.K8S_ANNOTATION_LBAAS_SPEC: annotation}, resource_version=service['metadata']['resourceVersion'])
def create_subnet(self, ns_name, project_id, net_id): os_net = clients.get_network_client() subnet_name = "ns/" + ns_name + "-subnet" tags = oslo_cfg.CONF.neutron_defaults.resource_tags if tags: subnets = os_net.subnets(name=subnet_name, tags=tags) else: subnets = os_net.subnets(name=subnet_name) try: # NOTE(ltomasbo): only one subnet must exists subnet = next(subnets) return subnet.id, subnet.cidr except StopIteration: LOG.debug('Subnet does not exist. Creating.') # create subnet with namespace as name subnet_pool_id = oslo_cfg.CONF.namespace_subnet.pod_subnet_pool ip_version = utils.get_subnetpool_version(subnet_pool_id) try: neutron_subnet = (os_net.create_subnet( network_id=net_id, ip_version=ip_version, name=subnet_name, enable_dhcp=False, subnetpool_id=subnet_pool_id, project_id=project_id)) except os_exc.ConflictException: LOG.debug( "Max number of retries on neutron side achieved, " "raising ResourceNotReady to retry subnet creation " "for %s", subnet_name) raise exceptions.ResourceNotReady(subnet_name) c_utils.tag_neutron_resources([neutron_subnet]) return neutron_subnet.id, neutron_subnet.cidr
def _delete_namespace_network_resources(self, subnet_id, net_id): os_net = clients.get_network_client() if subnet_id: router_id = oslo_cfg.CONF.namespace_subnet.pod_router try: clients.handle_neutron_errors( os_net.remove_interface_from_router, router_id, subnet_id=subnet_id) except os_exc.NotFoundException as e: # Nothing to worry about, either router or subnet is no more, # or subnet is already detached. LOG.debug(e.message) pass except os_exc.SDKException: LOG.exception( "Error deleting subnet %(subnet)s from router " "%(router)s.", { 'subnet': subnet_id, 'router': router_id }) raise try: os_net.delete_network(net_id) except os_exc.ConflictException: LOG.exception( "One or more ports in use on the network %s. " "Deleting leftovers ports before retrying", net_id) leftover_ports = os_net.ports(network_id=net_id) for leftover_port in leftover_ports: # NOTE(dulek): '' is there because Neutron seems to unset # device_owner on detach. if leftover_port.device_owner not in [ '', 'trunk:subport', kl_const.DEVICE_OWNER ]: continue try: # NOTE(gryf): there is unlikely, that we get an exception # like PortNotFound or something, since openstacksdk # doesn't raise an exception if port doesn't exists nor # return any information. os_net.delete_port(leftover_port.id) except os_exc.SDKException as e: if "currently a subport for trunk" in str(e): if leftover_port.status == "DOWN": LOG.warning( "Port %s is in DOWN status but still " "associated to a trunk. This should " "not happen. Trying to delete it from " "the trunk.", leftover_port.id) # Get the trunk_id from the error message trunk_id = ( str(e).split('trunk')[1].split('.')[0].strip()) try: os_net.delete_trunk_subports( trunk_id, [{ 'port_id': leftover_port.id }]) except os_exc.NotFoundException: LOG.debug("Port %s already removed from trunk %s", leftover_port['id'], trunk_id) else: LOG.exception( "Unexpected error deleting leftover " "port %s. Skiping it and continue with " "the other rest.", leftover_port.id) raise exceptions.ResourceNotReady(net_id) except os_exc.SDKException: LOG.exception("Error deleting network %s.", net_id) raise
def on_present(self, kuryrport_crd): if not kuryrport_crd['status']['vifs']: # Get vifs if not self.get_vifs(kuryrport_crd): # Ignore this event, according to one of the cases logged in # get_vifs method. return vifs = {ifname: {'default': data['default'], 'vif': objects.base.VersionedObject .obj_from_primitive(data['vif'])} for ifname, data in kuryrport_crd['status']['vifs'].items()} if all([v['vif'].active for v in vifs.values()]): return changed = False try: for ifname, data in vifs.items(): if (data['vif'].plugin == constants.KURYR_VIF_TYPE_SRIOV and oslo_cfg.CONF.sriov.enable_node_annotations): pod_node = kuryrport_crd['spec']['podNodeName'] # TODO(gryf): This probably will need adoption, so it will # add information to CRD instead of the pod. driver_utils.update_port_pci_info(pod_node, data['vif']) if not data['vif'].active: try: self._drv_vif_pool.activate_vif(data['vif']) changed = True except os_exc.ResourceNotFound: LOG.debug("Port not found, possibly already deleted. " "No need to activate it") finally: if changed: try: name = kuryrport_crd['metadata']['name'] namespace = kuryrport_crd['metadata']['namespace'] pod = self.k8s.get(f"{constants.K8S_API_NAMESPACES}" f"/{namespace}/pods/{name}") except k_exc.K8sResourceNotFound as ex: LOG.exception("Failed to get pod: %s", ex) raise project_id = self._drv_project.get_project(pod) try: self._update_kuryrport_crd(kuryrport_crd, vifs) except k_exc.K8sResourceNotFound as ex: LOG.exception("Failed to update KuryrPort CRD: %s", ex) security_groups = self._drv_sg.get_security_groups( pod, project_id) for ifname, data in vifs.items(): self._drv_vif_pool.release_vif(pod, data['vif'], project_id, security_groups) except k_exc.K8sClientException: raise k_exc.ResourceNotReady(pod['metadata']['name']) try: self._record_pod_creation_metric(pod) except Exception: LOG.debug("Failed to record metric for pod %s", name) if self._is_network_policy_enabled(): crd_pod_selectors = self._drv_sg.create_sg_rules(pod) if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)
def _move_annotations_to_crd(self, endpoints): """Support upgrade from annotations to KuryrLoadBalancer CRD.""" try: spec = (endpoints['metadata']['annotations'] [k_const.K8S_ANNOTATION_LBAAS_SPEC]) except KeyError: spec = None try: state = (endpoints['metadata']['annotations'] [k_const.K8S_ANNOTATION_LBAAS_STATE]) except KeyError: state = None if not state and not spec: # No annotations, return return False if state or spec: if state: _dict = jsonutils.loads(state) # This is strongly using the fact that annotation's o.vo # and CRD has the same structure. state = obj_lbaas.flatten_object(_dict) # Endpoints should always have the spec in the annotation spec_dict = jsonutils.loads(spec) spec = obj_lbaas.flatten_object(spec_dict) if state and state['service_pub_ip_info'] is None: del state['service_pub_ip_info'] for spec_port in spec['ports']: if not spec_port.get('name'): del spec_port['name'] if not spec['lb_ip']: del spec['lb_ip'] try: self._create_crd_spec(endpoints, spec, state) except k_exc.ResourceNotReady: LOG.info('KuryrLoadBalancer CRD %s already exists.', utils.get_res_unique_name(endpoints)) except k_exc.K8sClientException: raise k_exc.ResourceNotReady(endpoints) # In this step we only need to make sure all annotations are # removed. It may happen that the Endpoints only had spec set, # in which case we just remove it and let the normal flow handle # creation of the LB. k8s = clients.get_kubernetes_client() service_link = utils.get_service_link(endpoints) to_remove = [ (endpoints['metadata']['selfLink'], k_const.K8S_ANNOTATION_LBAAS_SPEC), (service_link, k_const.K8S_ANNOTATION_LBAAS_SPEC), ] if state: to_remove.append((endpoints['metadata']['selfLink'], k_const.K8S_ANNOTATION_LBAAS_STATE)) for path, name in to_remove: try: k8s.remove_annotations(path, name) except k_exc.K8sClientException: LOG.warning('Error removing %s annotation from %s', name, path) return True
def add(self, params): kp_name = self._get_obj_name(params) timeout = CONF.cni_daemon.vif_annotation_timeout # Try to confirm if CRD in the registry is not stale cache. If it is, # remove it. with lockutils.lock(kp_name, external=True): if kp_name in self.registry: cached_kp = self.registry[kp_name]['kp'] try: kp = self.k8s.get(k_utils.get_res_link(cached_kp)) except Exception: LOG.exception('Error when getting KuryrPort %s', kp_name) raise exceptions.ResourceNotReady(kp_name) if kp['metadata']['uid'] != cached_kp['metadata']['uid']: LOG.warning( 'Stale KuryrPort %s detected in cache. (API ' 'uid=%s, cached uid=%s). Removing it from ' 'cache.', kp_name, kp['metadata']['uid'], cached_kp['metadata']['uid']) del self.registry[kp_name] vifs = self._do_work(params, b_base.connect, timeout) # NOTE(dulek): Saving containerid to be able to distinguish old DEL # requests that we should ignore. We need a lock to # prevent race conditions and replace whole object in the # dict for multiprocessing.Manager to notice that. with lockutils.lock(kp_name, external=True): d = self.registry[kp_name] d['containerid'] = params.CNI_CONTAINERID self.registry[kp_name] = d LOG.debug('Saved containerid = %s for CRD %s', params.CNI_CONTAINERID, kp_name) # Wait for timeout sec, 1 sec between tries, retry when even one # vif is not active. @retrying.retry(stop_max_delay=timeout * 1000, wait_fixed=RETRY_DELAY, retry_on_result=utils.any_vif_inactive) def wait_for_active(kp_name): return self.registry[kp_name]['vifs'] data = { 'metadata': { 'name': params.args.K8S_POD_NAME, 'namespace': params.args.K8S_POD_NAMESPACE } } pod = k_utils.get_referenced_object(data, 'Pod') try: self.k8s.add_event( pod, 'CNIWaitingForVIFs', f'Waiting for Neutron ports of {kp_name} to ' f'become ACTIVE after binding.') vifs = wait_for_active(kp_name) except retrying.RetryError: self.k8s.add_event( pod, 'CNITimedOutWaitingForVIFs', f'Timed out waiting for Neutron ports of ' f'{kp_name} to become ACTIVE after binding.', 'Warning') raise exceptions.CNINeutronPortActivationTimeout( kp_name, self.registry[kp_name]['vifs']) return vifs[k_const.DEFAULT_IFNAME]
def on_present(self, namespace): ns_name = namespace['metadata']['name'] current_namespace_labels = namespace['metadata'].get('labels') previous_namespace_labels = drivers_utils.get_annotated_labels( namespace, constants.K8S_ANNOTATION_NAMESPACE_LABEL) LOG.debug("Got previous namespace labels from annotation: %r", previous_namespace_labels) if current_namespace_labels != previous_namespace_labels: self._drv_sg.update_namespace_sg_rules(namespace) self._set_namespace_labels(namespace, current_namespace_labels) project_id = self._drv_project.get_project(namespace) net_crd_id = self._get_net_crd_id(namespace) if net_crd_id: LOG.debug("CRD existing at the new namespace") return net_crd_name = 'ns-' + ns_name net_crd = self._get_net_crd(net_crd_name) if net_crd: LOG.debug("Previous CRD existing at the new namespace. " "Deleting namespace resources and retying its creation.") self.on_deleted(namespace, net_crd) raise exceptions.ResourceNotReady(namespace) # NOTE(ltomasbo): Ensure there is no previously created networks # leftovers due to a kuryr-controller crash/restart LOG.debug("Deleting leftovers network resources for namespace: %s", ns_name) self._drv_subnets.cleanup_namespace_networks(ns_name) LOG.debug("Creating network resources for namespace: %s", ns_name) net_crd_spec = self._drv_subnets.create_namespace_network( ns_name, project_id) try: net_crd_sg = self._drv_sg.create_namespace_sg( ns_name, project_id, net_crd_spec) except n_exc.NeutronClientException: LOG.exception("Error creating security group for the namespace. " "Rolling back created network resources.") self._drv_subnets.rollback_network_resources(net_crd_spec, ns_name) raise if net_crd_sg: net_crd_spec.update(net_crd_sg) else: LOG.debug("No SG created for the namespace. Namespace isolation " "will not be enforced.") # create CRD resource for the network try: net_crd = self._add_kuryrnet_crd(ns_name, net_crd_spec) self._set_net_crd(namespace, net_crd) self._drv_sg.create_namespace_sg_rules(namespace) self._set_namespace_labels(namespace, current_namespace_labels) except exceptions.K8sClientException: LOG.exception("Kubernetes client exception. Rolling back " "resources created for the namespace.") self._drv_subnets.rollback_network_resources(net_crd_spec, ns_name) if net_crd_sg.get('sgId'): self._drv_sg.delete_sg(net_crd_sg['sgId']) self._del_kuryrnet_crd(net_crd_name)
def on_present(self, kuryrport_crd, *args, **kwargs): if not kuryrport_crd['status']['vifs']: # Get vifs if not self.get_vifs(kuryrport_crd): # Ignore this event, according to one of the cases logged in # get_vifs method. return retry_info = kwargs.get('retry_info') vifs = { ifname: { 'default': data['default'], 'vif': objects.base.VersionedObject.obj_from_primitive(data['vif']) } for ifname, data in kuryrport_crd['status']['vifs'].items() } if all([v['vif'].active for v in vifs.values()]): return changed = False pod = self._get_pod(kuryrport_crd) try: for ifname, data in vifs.items(): if (data['vif'].plugin == constants.KURYR_VIF_TYPE_SRIOV and oslo_cfg.CONF.sriov.enable_node_annotations): pod_node = kuryrport_crd['spec']['podNodeName'] # TODO(gryf): This probably will need adoption, so it will # add information to CRD instead of the pod. driver_utils.update_port_pci_info(pod_node, data['vif']) if not data['vif'].active: try: self._drv_vif_pool.activate_vif(data['vif'], pod=pod, retry_info=retry_info) changed = True except k_exc.ResourceNotReady: if retry_info and retry_info.get('elapsed', 0) > ACTIVE_TIMEOUT: self.k8s.add_event( pod, 'ActivatePortFailed', 'Activating Neutron port has ' 'timed out', 'Warning') raise except os_exc.ResourceNotFound: self.k8s.add_event( pod, 'ActivatePortFailed', 'Activating Neutron port has ' 'failed, possibly deleted', 'Warning') LOG.debug("Port not found, possibly already deleted. " "No need to activate it") finally: if changed: project_id = self._drv_project.get_project(pod) try: self._update_kuryrport_crd(kuryrport_crd, vifs) except k_exc.K8sResourceNotFound as ex: LOG.exception("Failed to update KuryrPort CRD: %s", ex) security_groups = self._drv_sg.get_security_groups( pod, project_id) for ifname, data in vifs.items(): self._drv_vif_pool.release_vif(pod, data['vif'], project_id, security_groups) self.k8s.add_event( pod, 'UpdateKuryrPortCRDFailed', f'Marking ports as ACTIVE in the ' f'KuryrPort failed: {ex}', 'Warning') except k_exc.K8sClientException: raise k_exc.ResourceNotReady(pod['metadata']['name']) try: self._record_pod_creation_metric(pod) except Exception: LOG.debug("Failed to record metric for pod %s", pod['metadata']['name']) if driver_utils.is_network_policy_enabled(): crd_pod_selectors = self._drv_sg.create_sg_rules(pod) if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)
def on_finalize(self, knp): LOG.debug("Finalizing KuryrNetworkPolicy %s") project_id = self._drv_project.get_project(knp) pods_to_update = self._drv_policy.affected_pods(knp) crd_sg = knp['status'].get('securityGroupId') try: policy = self._get_networkpolicy( knp['metadata']['annotations']['networkPolicyLink']) except exceptions.K8sResourceNotFound: # NP is already gone, let's just try to clean up. policy = None if crd_sg: for pod in pods_to_update: if driver_utils.is_host_network(pod): continue pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) if crd_sg in pod_sgs: pod_sgs.remove(crd_sg) if not pod_sgs: pod_sgs = CONF.neutron_defaults.pod_security_groups if not pod_sgs: raise cfg.RequiredOptError( 'pod_security_groups', cfg.OptGroup('neutron_defaults')) try: self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) except os_exc.NotFoundException: LOG.debug("Fail to update pod sgs." " Retrying policy deletion.") raise exceptions.ResourceNotReady(knp) # ensure ports at the pool don't have the NP sg associated try: net_id = self._get_policy_net_id(knp) self._drv_vif_pool.remove_sg_from_pools(crd_sg, net_id) except exceptions.K8sResourceNotFound: # Probably the network got removed already, we can ignore it. pass if (CONF.octavia_defaults.enforce_sg_rules and policy and not self._is_egress_only_policy(policy)): services = driver_utils.get_services( knp['metadata']['namespace']) for svc in services.get('items'): if (not svc['spec'].get('selector') or not self._is_service_affected( svc, pods_to_update)): continue sgs = self._drv_svc_sg.get_security_groups(svc, project_id) try: self._drv_lbaas.update_lbaas_sg(svc, sgs) except exceptions.ResourceNotReady: # We can ignore LB that's being created - its SGs will # get handled when members will be getting created. pass self._drv_policy.delete_np_sg(crd_sg) LOG.debug("Removing finalizers from KuryrNetworkPolicy and " "NetworkPolicy.") if policy: self.k8s.remove_finalizer(policy, constants.NETWORKPOLICY_FINALIZER) self.k8s.remove_finalizer(knp, constants.NETWORKPOLICY_FINALIZER)
def _apply_members_security_groups(self, loadbalancer, port, target_port, protocol, sg_rule_name, listener_id, new_sgs=None): LOG.debug("Applying members security groups.") os_net = clients.get_network_client() lb_sg = None if CONF.octavia_defaults.enforce_sg_rules: vip_port = self._get_vip_port(loadbalancer) if vip_port: try: lb_sg = vip_port.security_group_ids[0] except IndexError: LOG.warning("We still waiting for SG to be created for " "VIP %s", vip_port) raise k_exc.ResourceNotReady(listener_id) else: LOG.debug("Skipping sg update for lb %s", loadbalancer['name']) return # NOTE (maysams) It might happen that the update of LBaaS SG # has been triggered and the LBaaS SG was not created yet. # This update is skiped, until the LBaaS members are created. if not lb_sg: return if self._octavia_acls: self._create_listeners_acls(loadbalancer, port, target_port, protocol, lb_sg, new_sgs, listener_id) return lbaas_sg_rules = os_net.security_group_rules( security_group_id=lb_sg, project_id=loadbalancer['project_id']) all_pod_rules = [] add_default_rules = False if new_sgs: sgs = new_sgs else: sgs = loadbalancer['security_groups'] sg_rule_ethertype = k_const.IPv4 if utils.get_service_subnet_version() == k_const.IP_VERSION_6: sg_rule_ethertype = k_const.IPv6 # Check if Network Policy allows listener on the pods for sg in sgs: if sg != lb_sg: if sg in config.CONF.neutron_defaults.pod_security_groups: # If default sg is set, this means there is no NP # associated to the service, thus falling back to the # default listener rules add_default_rules = True break rules = os_net.security_group_rules(security_group_id=sg) for rule in rules: # copying ingress rules with same protocol onto the # loadbalancer sg rules # NOTE(ltomasbo): NP sg can only have rules with # or without remote_ip_prefix. Rules with remote_group_id # are not possible, therefore only applying the ones # with or without remote_ip_prefix. if (rule.protocol == protocol.lower() and rule.direction == 'ingress'): # If listener port not in allowed range, skip min_port = rule.port_range_min max_port = rule.port_range_max if (min_port and target_port not in range(min_port, max_port+1)): continue all_pod_rules.append(rule) try: LOG.debug("Creating LBaaS sg rule for sg: %r", lb_sg) os_net.create_security_group_rule( direction='ingress', ether_type=sg_rule_ethertype, port_range_min=port, port_range_max=port, protocol=protocol, remote_ip_prefix=rule.remote_ip_prefix, security_group_id=lb_sg, description=sg_rule_name) except os_exc.ConflictException: pass except os_exc.SDKException: LOG.exception('Failed when creating security ' 'group rule for listener %s.', sg_rule_name) # Delete LBaaS sg rules that do not match NP for rule in lbaas_sg_rules: if (rule.protocol != protocol.lower() or rule.port_range_min != port or rule.direction != 'ingress'): if all_pod_rules and self._is_default_rule(rule): LOG.debug("Removing default LBaaS sg rule for sg: %r", lb_sg) os_net.delete_security_group_rule(rule.id) continue self._delete_rule_if_no_match(rule, all_pod_rules) if add_default_rules: try: LOG.debug("Restoring default LBaaS sg rule for sg: %r", lb_sg) os_net.create_security_group_rule(direction='ingress', ether_type=sg_rule_ethertype, port_range_min=port, port_range_max=port, protocol=protocol, security_group_id=lb_sg, description=sg_rule_name) except os_exc.ConflictException: pass except os_exc.SDKException: LOG.exception('Failed when creating security group rule for ' 'listener %s.', sg_rule_name)
def on_present(self, pod, *args, **kwargs): if utils.is_host_network(pod): return pod_name = pod['metadata']['name'] if utils.is_pod_completed(pod): LOG.debug("Pod %s has completed execution, " "removing the vifs", pod_name) self.on_finalize(pod) return if not self._is_pod_scheduled(pod): # REVISIT(ivc): consider an additional configurable check that # would allow skipping pods to enable heterogeneous environments # where certain pods/namespaces/nodes can be managed by other # networking solutions/CNI drivers. return namespace = pod['metadata']['namespace'] kuryrnetwork_path = '{}/{}/kuryrnetworks/{}'.format( constants.K8S_API_CRD_NAMESPACES, namespace, namespace) kuryrnetwork = driver_utils.get_k8s_resource(kuryrnetwork_path) kuryrnetwork_status = kuryrnetwork.get('status', {}) if (CONF.kubernetes.pod_subnets_driver == 'namespace' and (not kuryrnetwork or not kuryrnetwork_status.get('routerId'))): namespace_path = '{}/{}'.format( constants.K8S_API_NAMESPACES, namespace) LOG.debug("Triggering Namespace Handling %s", namespace_path) try: self.k8s.annotate(namespace_path, {'KuryrTrigger': str(uuid.uuid4())}) except k_exc.K8sResourceNotFound: LOG.warning('Ignoring Pod handling, no Namespace %s.', namespace) return raise k_exc.ResourceNotReady(pod) # NOTE(gryf): Set the finalizer as soon, as we have pod created. On # subsequent updates of the pod, add_finalizer will ignore this if # finalizer exist. try: if not self.k8s.add_finalizer(pod, constants.POD_FINALIZER): # NOTE(gryf) It might happen that pod will be deleted even # before we got here. return except k_exc.K8sClientException as ex: self.k8s.add_event(pod, 'FailedToAddFinalizerToPod', f'Adding finalizer to pod has failed: {ex}', 'Warning') LOG.exception("Failed to add finalizer to pod object: %s", ex) raise kp = driver_utils.get_kuryrport(pod) LOG.debug("Got KuryrPort: %r", kp) if not kp: try: self._add_kuryrport_crd(pod) except k_exc.K8sNamespaceTerminating: # The underlying namespace is being terminated, we can # ignore this and let `on_finalize` handle this now. LOG.warning('Namespace %s is being terminated, ignoring Pod ' '%s in that namespace.', pod['metadata']['namespace'], pod_name) return except k_exc.K8sClientException as ex: self.k8s.add_event(pod, 'FailedToCreateKuryrPortCRD', f'Creating corresponding KuryrPort CRD has ' f'failed: {ex}', 'Warning') LOG.exception("Kubernetes Client Exception creating " "KuryrPort CRD: %s", ex) raise k_exc.ResourceNotReady(pod)
def update_lbaas_sg(self, service, sgs): LOG.debug('Setting SG for LBaaS VIP port') svc_namespace = service['metadata']['namespace'] svc_name = service['metadata']['name'] svc_ports = service['spec'].get('ports', []) lbaas_name = c_utils.get_resource_name(svc_name, prefix=svc_namespace + "/") endpoints_link = utils.get_endpoints_link(service) k8s = clients.get_kubernetes_client() try: k8s.get(endpoints_link) except k_exc.K8sResourceNotFound: LOG.debug("Endpoint not Found. Skipping LB SG update for " "%s as the LB resources are not present", lbaas_name) return try: klb = k8s.get(f'{k_const.K8S_API_CRD_NAMESPACES}/{svc_namespace}/' f'kuryrloadbalancers/{svc_name}') except k_exc.K8sResourceNotFound: LOG.debug('No KuryrLoadBalancer for service %s created yet.', lbaas_name) raise k_exc.ResourceNotReady(svc_name) if (not klb.get('status', {}).get('loadbalancer') or klb.get('status', {}).get('listeners') is None): LOG.debug('KuryrLoadBalancer for service %s not populated yet.', lbaas_name) raise k_exc.ResourceNotReady(svc_name) klb['status']['loadbalancer']['security_groups'] = sgs lb = klb['status']['loadbalancer'] try: k8s.patch_crd('status/loadbalancer', utils.get_res_link(klb), {'security_groups': sgs}) except k_exc.K8sResourceNotFound: LOG.debug('KuryrLoadBalancer CRD not found %s', lbaas_name) return except k_exc.K8sClientException: LOG.exception('Error updating KuryLoadBalancer CRD %s', lbaas_name) raise lsnr_ids = {(listener['protocol'], listener['port']): listener['id'] for listener in klb['status']['listeners']} for port in svc_ports: port_protocol = port['protocol'] lbaas_port = port['port'] target_port = port['targetPort'] suffix = f"{port_protocol}:{lbaas_port}" sg_rule_name = c_utils.get_resource_name(lbaas_name, suffix=':' + suffix) listener_id = lsnr_ids.get((port_protocol, lbaas_port)) if listener_id is None: LOG.warning("There is no listener associated to the protocol " "%s and port %s. Skipping", port_protocol, lbaas_port) continue self._apply_members_security_groups(lb, lbaas_port, target_port, port_protocol, sg_rule_name, listener_id, sgs)
def get_vifs(self, kuryrport_crd): try: pod = self.k8s.get(f"{constants.K8S_API_NAMESPACES}" f"/{kuryrport_crd['metadata']['namespace']}" f"/pods" f"/{kuryrport_crd['metadata']['name']}") except k_exc.K8sResourceNotFound as ex: LOG.exception("Failed to get pod: %s", ex) # TODO(gryf): Release resources self.k8s.remove_finalizer(kuryrport_crd, constants.KURYRPORT_FINALIZER) raise project_id = self._drv_project.get_project(pod) security_groups = self._drv_sg.get_security_groups(pod, project_id) try: subnets = self._drv_subnets.get_subnets(pod, project_id) except (os_exc.ResourceNotFound, k_exc.K8sResourceNotFound): LOG.warning("Subnet does not exists. If namespace driver is " "used, probably the namespace for the pod is " "already deleted. So this pod does not need to " "get a port as it will be deleted too. If the " "default subnet driver is used, then you must " "select an existing subnet to be used by Kuryr.") return False # Request the default interface of pod try: main_vif = self._drv_vif_pool.request_vif(pod, project_id, subnets, security_groups) except os_exc.ResourceNotFound: # NOTE(gryf): It might happen, that between getting security # groups above and requesting VIF, network policy is deleted, # hence we will get 404 from OpenStackSDK. Let's retry, to refresh # information regarding SG. LOG.warning("SG not found during VIF requesting. Retrying.") raise k_exc.ResourceNotReady(pod['metadata']['name']) if not main_vif: pod_name = pod['metadata']['name'] LOG.warning("Ignoring event due to pod %s not being " "scheduled yet.", pod_name) return False vifs = {constants.DEFAULT_IFNAME: {'default': True, 'vif': main_vif}} # Request the additional interfaces from multiple drivers index = 0 for driver in self._drv_multi_vif: additional_vifs = driver.request_additional_vifs(pod, project_id, security_groups) for index, vif in enumerate(additional_vifs, start=index+1): ifname = (oslo_cfg.CONF.kubernetes.additional_ifname_prefix + str(index)) vifs[ifname] = {'default': False, 'vif': vif} try: self._update_kuryrport_crd(kuryrport_crd, vifs) except k_exc.K8sClientException as ex: LOG.exception("Kubernetes Client Exception creating " "KuryrPort CRD: %s", ex) for ifname, data in vifs.items(): self._drv_vif_pool.release_vif(pod, data['vif'], project_id, security_groups) return True
def on_present(self, pod): if (driver_utils.is_host_network(pod) or not self._is_pod_scheduled(pod)): # REVISIT(ivc): consider an additional configurable check that # would allow skipping pods to enable heterogeneous environments # where certain pods/namespaces/nodes can be managed by other # networking solutions/CNI drivers. return state = driver_utils.get_pod_state(pod) LOG.debug("Got VIFs from annotation: %r", state) project_id = self._drv_project.get_project(pod) security_groups = self._drv_sg.get_security_groups(pod, project_id) if not state: try: subnets = self._drv_subnets.get_subnets(pod, project_id) except (os_exc.ResourceNotFound, k_exc.K8sResourceNotFound): LOG.warning("Subnet does not exists. If namespace driver is " "used, probably the namespace for the pod is " "already deleted. So this pod does not need to " "get a port as it will be deleted too. If the " "default subnet driver is used, then you must " "select an existing subnet to be used by Kuryr.") return # Request the default interface of pod main_vif = self._drv_vif_pool.request_vif(pod, project_id, subnets, security_groups) if not main_vif: pod_name = pod['metadata']['name'] LOG.warning( "Ignoring event due to pod %s not being " "scheduled yet.", pod_name) return state = objects.vif.PodState(default_vif=main_vif) # Request the additional interfaces from multiple dirvers additional_vifs = [] for driver in self._drv_multi_vif: additional_vifs.extend( driver.request_additional_vifs(pod, project_id, security_groups)) if additional_vifs: state.additional_vifs = {} for i, vif in enumerate(additional_vifs, start=1): k = (oslo_cfg.CONF.kubernetes.additional_ifname_prefix + str(i)) state.additional_vifs[k] = vif try: self._set_pod_state(pod, state) except k_exc.K8sClientException as ex: LOG.debug("Failed to set annotation: %s", ex) # FIXME(ivc): improve granularity of K8sClient exceptions: # only resourceVersion conflict should be ignored for ifname, vif in state.vifs.items(): self._drv_vif_pool.release_vif(pod, vif, project_id, security_groups) else: changed = False try: for ifname, vif in state.vifs.items(): if vif.plugin == constants.KURYR_VIF_TYPE_SRIOV: driver_utils.update_port_pci_info(pod, vif) if not vif.active: try: self._drv_vif_pool.activate_vif(pod, vif) changed = True except n_exc.PortNotFoundClient: LOG.debug("Port not found, possibly already " "deleted. No need to activate it") finally: if changed: try: self._set_pod_state(pod, state) except k_exc.K8sResourceNotFound as ex: LOG.exception("Failed to set annotation: %s", ex) for ifname, vif in state.vifs.items(): self._drv_vif_pool.release_vif( pod, vif, project_id, security_groups) except k_exc.K8sClientException: pod_name = pod['metadata']['name'] raise k_exc.ResourceNotReady(pod_name) if self._is_network_policy_enabled(): crd_pod_selectors = self._drv_sg.create_sg_rules(pod) if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)
def on_present(self, pod): if driver_utils.is_host_network(pod) or not self._is_pending_node(pod): # REVISIT(ivc): consider an additional configurable check that # would allow skipping pods to enable heterogeneous environments # where certain pods/namespaces/nodes can be managed by other # networking solutions/CNI drivers. return state = driver_utils.get_pod_state(pod) LOG.debug("Got VIFs from annotation: %r", state) project_id = self._drv_project.get_project(pod) security_groups = self._drv_sg.get_security_groups(pod, project_id) if not state: subnets = self._drv_subnets.get_subnets(pod, project_id) # Request the default interface of pod main_vif = self._drv_vif_pool.request_vif(pod, project_id, subnets, security_groups) if not main_vif: pod_name = pod['metadata']['name'] LOG.warning( "Ignoring event due to pod %s not being " "scheduled yet.", pod_name) return state = objects.vif.PodState(default_vif=main_vif) # Request the additional interfaces from multiple dirvers additional_vifs = [] for driver in self._drv_multi_vif: additional_vifs.extend( driver.request_additional_vifs(pod, project_id, security_groups)) if additional_vifs: state.additional_vifs = {} for i, vif in enumerate(additional_vifs, start=1): k = constants.ADDITIONAL_IFNAME_PREFIX + str(i) state.additional_vifs[k] = vif try: self._set_pod_state(pod, state) except k_exc.K8sClientException as ex: LOG.debug("Failed to set annotation: %s", ex) # FIXME(ivc): improve granularity of K8sClient exceptions: # only resourceVersion conflict should be ignored for ifname, vif in state.vifs.items(): self._drv_vif_pool.release_vif(pod, vif, project_id, security_groups) else: changed = False try: for ifname, vif in state.vifs.items(): if vif.plugin == constants.KURYR_VIF_TYPE_SRIOV: driver_utils.update_port_pci_info(pod, vif) if not vif.active: self._drv_vif_pool.activate_vif(pod, vif) changed = True finally: if changed: try: self._set_pod_state(pod, state) except k_exc.K8sResourceNotFound as ex: LOG.exception("Failed to set annotation: %s", ex) for ifname, vif in state.vifs.items(): self._drv_vif_pool.release_vif( pod, vif, project_id, security_groups) except k_exc.K8sClientException: pod_name = pod['metadata']['name'] raise k_exc.ResourceNotReady(pod_name) if self._is_network_policy_enabled(): crd_pod_selectors = self._drv_sg.create_sg_rules(pod) if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id)