class VncIngress(VncCommon):
    def __init__(self, tag_mgr=None):
        self._k8s_event_type = 'Ingress'
        super(VncIngress, self).__init__(self._k8s_event_type)
        self._name = type(self).__name__
        self._args = vnc_kube_config.args()
        self._queue = vnc_kube_config.queue()
        self._vnc_lib = vnc_kube_config.vnc_lib()
        self._logger = vnc_kube_config.logger()
        self._kube = vnc_kube_config.kube()
        self._label_cache = vnc_kube_config.label_cache()
        self._labels = XLabelCache(self._k8s_event_type)
        self.tag_mgr = tag_mgr
        self._ingress_label_cache = {}
        self._default_vn_obj = None
        self._fip_pool_obj = None
        self.service_lb_mgr = ServiceLbManager()
        self.service_ll_mgr = ServiceLbListenerManager()
        self.service_lb_pool_mgr = ServiceLbPoolManager()
        self.service_lb_member_mgr = ServiceLbMemberManager()

    def _get_project(self, ns_name):
        proj_fq_name = vnc_kube_config.cluster_project_fq_name(ns_name)
        try:
            proj_obj = self._vnc_lib.project_read(fq_name=proj_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" %
                               (self._name, proj_fq_name))
            return None
        return proj_obj

    def _get_namespace(self, ns_name):
        return NamespaceKM.find_by_name_or_uuid(ns_name)

    def _is_network_isolated(self, ns_name):
        return self._get_namespace(ns_name).is_isolated()

    def _get_ip_fabric_forwarding(self, ns_name):
        ns = self._get_namespace(ns_name)
        if ns:
            return ns.get_ip_fabric_forwarding()
        return None

    def _is_ip_fabric_forwarding_enabled(self, ns_name):
        ip_fabric_forwarding = self._get_ip_fabric_forwarding(ns_name)
        if ip_fabric_forwarding != None:
            return ip_fabric_forwarding
        else:
            return self._args.ip_fabric_forwarding

    def _get_network(self, ns_name):
        set_default_vn = False
        ns = self._get_namespace(ns_name)
        vn_fq_name = ns.get_annotated_network_fq_name()

        if not vn_fq_name:
            if ns.is_isolated():
                vn_fq_name = ns.get_isolated_pod_network_fq_name()

        if not vn_fq_name:
            if self._default_vn_obj:
                return self._default_vn_obj
            set_default_vn = True
            vn_fq_name = vnc_kube_config.cluster_default_pod_network_fq_name()

        try:
            vn_obj = self._vnc_lib.virtual_network_read(fq_name=vn_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" % (self._name, vn_fq_name))
            return None

        if set_default_vn:
            self._default_vn_obj = vn_obj

        return vn_obj

    def _get_pod_ipam_subnet_uuid(self, ns_name, vn_obj):
        pod_ipam_subnet_uuid = None
        if self._is_network_isolated(ns_name):
            vn_namespace = ns_name
        else:
            vn_namespace = 'default'
        if self._is_ip_fabric_forwarding_enabled(vn_namespace):
            ipam_fq_name = vnc_kube_config.ip_fabric_ipam_fq_name()
        else:
            ipam_fq_name = vnc_kube_config.pod_ipam_fq_name()
        vn = VirtualNetworkKM.find_by_name_or_uuid(vn_obj.get_uuid())
        pod_ipam_subnet_uuid = vn.get_ipam_subnet_uuid(ipam_fq_name)
        if pod_ipam_subnet_uuid is None:
            self._logger.error("%s - %s Not Found" %
                               (self._name, ipam_fq_name))
        return pod_ipam_subnet_uuid

    def _get_public_fip_pool(self, fip_pool_fq_name):
        if self._fip_pool_obj:
            return self._fip_pool_obj
        try:
            fip_pool_obj = self._vnc_lib. \
                           floating_ip_pool_read(fq_name=fip_pool_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" \
                 %(self._name, fip_pool_fq_name))
            return None
        self._fip_pool_obj = fip_pool_obj
        return fip_pool_obj

    def _get_floating_ip(self, name, proj_obj, external_ip=None, vmi_obj=None):
        if not vnc_kube_config.is_public_fip_pool_configured():
            return None

        try:
            fip_pool_fq_name = get_fip_pool_fq_name_from_dict_string(
                self._args.public_fip_pool)
        except Exception as e:
            string_buf = StringIO()
            cgitb_hook(file=string_buf, format="text")
            err_msg = string_buf.getvalue()
            self._logger.error("%s - %s" % (self._name, err_msg))
            return None

        if vmi_obj:
            fip_refs = vmi_obj.get_floating_ip_back_refs()
            for ref in fip_refs or []:
                fip = FloatingIpKM.get(ref['uuid'])
                if fip and fip.fq_name[:-1] == fip_pool_fq_name:
                    return fip
                else:
                    break
        fip_pool = self._get_public_fip_pool(fip_pool_fq_name)
        if fip_pool is None:
            return None
        fip_uuid = str(uuid.uuid4())
        fip_name = VncCommon.make_name(name, fip_uuid)
        fip_obj = FloatingIp(fip_name, fip_pool)
        fip_obj.uuid = fip_uuid
        fip_obj.set_project(proj_obj)
        if vmi_obj:
            fip_obj.set_virtual_machine_interface(vmi_obj)
        if external_ip:
            fip_obj.floating_ip_address = external_ip
        try:
            self._vnc_lib.floating_ip_create(fip_obj)
            fip = FloatingIpKM.locate(fip_obj.uuid)
        except Exception as e:
            string_buf = StringIO()
            cgitb_hook(file=string_buf, format="text")
            err_msg = string_buf.getvalue()
            self._logger.error("%s - %s" % (self._name, err_msg))
            return None
        return fip

    def _allocate_floating_ip(self, lb_obj, name, proj_obj, external_ip):
        vmi_id = lb_obj.virtual_machine_interface_refs[0]['uuid']
        vmi_obj = self._vnc_lib.virtual_machine_interface_read(id=vmi_id)
        if vmi_obj is None:
            self._logger.error("%s - %s Vmi %s Not Found" \
                 %(self._name, lb_obj.name, vmi_id))
            return None
        fip = self._get_floating_ip(name, proj_obj, external_ip, vmi_obj)
        return fip

    def _deallocate_floating_ip(self, lb):
        vmi_id = list(lb.virtual_machine_interfaces)[0]
        vmi = VirtualMachineInterfaceKM.get(vmi_id)
        if vmi is None:
            self._logger.error("%s - %s Vmi %s Not Found" \
                 %(self._name, lb.name, vmi_id))
            return
        fip_list = vmi.floating_ips.copy()
        for fip_id in fip_list or []:
            fip_obj = self._vnc_lib.floating_ip_read(id=fip_id)
            fip_obj.set_virtual_machine_interface_list([])
            self._vnc_lib.floating_ip_update(fip_obj)
            self._vnc_lib.floating_ip_delete(id=fip_obj.uuid)
            FloatingIpKM.delete(fip_obj.uuid)

    def _update_floating_ip(self, name, ns_name, external_ip, lb_obj):
        proj_obj = self._get_project(ns_name)
        fip = self._allocate_floating_ip(lb_obj, name, proj_obj, external_ip)
        if fip:
            lb_obj.add_annotations(
                KeyValuePair(key='externalIP', value=external_ip))
            self._vnc_lib.loadbalancer_update(lb_obj)
        return fip

    def _update_kube_api_server(self, name, ns_name, lb_obj, fip):
        vip_dict_list = []
        if fip:
            vip_dict = {}
            vip_dict['ip'] = fip.address
            vip_dict_list.append(vip_dict)
        vip_dict = {}
        vip_dict['ip'] = lb_obj._loadbalancer_properties.vip_address
        vip_dict_list.append(vip_dict)
        patch = {'status': {'loadBalancer': {'ingress': vip_dict_list}}}
        self._kube.patch_resource("ingresses",
                                  name,
                                  patch,
                                  ns_name,
                                  beta=True,
                                  sub_resource_name='status')

    def _find_ingress(self, ingress_cache, ns_name, service_name):
        if not ns_name or not service_name:
            return
        key = 'service'
        value = '-'.join([ns_name, service_name])
        labels = {key: value}
        result = set()
        for label in labels.items():
            key = self._label_cache._get_key(label)
            ingress_ids = ingress_cache.get(key, set())
            #no matching label
            if not ingress_ids:
                return ingress_ids
            if not result:
                result = ingress_ids.copy()
            else:
                result.intersection_update(ingress_ids)
        return result

    def _clear_ingress_cache_uuid(self, ingress_cache, ingress_uuid):
        if not ingress_uuid:
            return
        key_list = [k for k, v in ingress_cache.items() if ingress_uuid in v]
        for key in key_list or []:
            label = tuple(key.split(':'))
            self._label_cache._remove_label(key, ingress_cache, label,
                                            ingress_uuid)

    def _clear_ingress_cache(self, ingress_cache, ns_name, service_name,
                             ingress_uuid):
        if not ns_name or not service_name:
            return
        key = 'service'
        value = '-'.join([ns_name, service_name])
        labels = {key: value}
        for label in labels.items() or []:
            key = self._label_cache._get_key(label)
            self._label_cache._remove_label(key, ingress_cache, label,
                                            ingress_uuid)

    def _update_ingress_cache(self, ingress_cache, ns_name, service_name,
                              ingress_uuid):
        if not ns_name or not service_name:
            return
        key = 'service'
        value = '-'.join([ns_name, service_name])
        labels = {key: value}
        for label in labels.items() or []:
            key = self._label_cache._get_key(label)
            self._label_cache._locate_label(key, ingress_cache, label,
                                            ingress_uuid)

    def _vnc_create_member(self, pool, address, port, annotations):
        pool_obj = self.service_lb_pool_mgr.read(pool.uuid)
        member_obj = self.service_lb_member_mgr.create(pool_obj, address, port,
                                                       annotations)
        return member_obj

    def _vnc_update_member(self, member_id, address, port, annotations):
        member_obj = self.service_lb_member_mgr.update(member_id, address,
                                                       port, annotations)
        return member_obj

    def _vnc_create_pool(self, ns_name, ll, port, lb_algorithm, annotations):
        proj_obj = self._get_project(ns_name)
        ll_obj = self.service_ll_mgr.read(ll.uuid)
        pool_obj = self.service_lb_pool_mgr.create(ll_obj, proj_obj, port,
                                                   lb_algorithm, annotations)
        return pool_obj

    def _vnc_create_listeners(self, ns_name, lb, port):
        proj_obj = self._get_project(ns_name)
        lb_obj = self.service_lb_mgr.read(lb.uuid)
        ll_obj = self.service_ll_mgr.create(lb_obj, proj_obj, port)
        return ll_obj

    def _vnc_create_lb(self, uid, name, ns_name, annotations):
        proj_obj = self._get_project(ns_name)
        vn_obj = self._get_network(ns_name)
        if proj_obj is None or vn_obj is None:
            return None

        vip_address = None
        pod_ipam_subnet_uuid = self._get_pod_ipam_subnet_uuid(ns_name, vn_obj)
        lb_obj = self.service_lb_mgr.create(
            self._k8s_event_type,
            ns_name,
            uid,
            name,
            proj_obj,
            vn_obj,
            vip_address,
            pod_ipam_subnet_uuid,
            tags=self._labels.get_labels_dict(uid))
        if lb_obj:
            external_ip = None
            if annotations and 'externalIP' in annotations:
                external_ip = annotations['externalIP']
            fip = self._update_floating_ip(name, ns_name, external_ip, lb_obj)
            self._update_kube_api_server(name, ns_name, lb_obj, fip)
        else:
            self._logger.error("%s - %s LB Not Created" % (self._name, name))

        return lb_obj

    def _vnc_delete_member(self, member_id):
        self.service_lb_member_mgr.delete(member_id)

    def _vnc_delete_pool(self, pool_id):
        self.service_lb_pool_mgr.delete(pool_id)

    def _vnc_delete_listener(self, ll_id):
        self.service_ll_mgr.delete(ll_id)

    def _vnc_delete_lb(self, lb):
        self._deallocate_floating_ip(lb)
        self.service_lb_mgr.delete(lb.uuid)

    def _get_old_backend_list(self, lb):
        backend_list = []
        listener_list = lb.loadbalancer_listeners
        for ll_id in listener_list:
            backend = {}
            backend['listener_id'] = ll_id
            ll = LoadbalancerListenerKM.get(ll_id)
            backend['listener'] = {}
            backend['listener']['protocol'] = ll.params['protocol']
            if backend['listener']['protocol'] == 'TERMINTED_HTTPS':
                if ll.params['default_tls_container']:
                    backend['listener']['default_tls_container'] = \
                        ll.params['default_tls_container']
                if ll.params['sni_containers']:
                    backend['listener']['sni_containers'] = \
                        ll.params['sni_containers']
            pool_id = ll.loadbalancer_pool
            if pool_id:
                pool = LoadbalancerPoolKM.get(pool_id)
                if pool.annotations is None:
                    annotations = {}
                    kvps = []
                    pool_obj = self._vnc_lib.loadbalancer_pool_read(id=pool_id)
                    pool_obj_kvp = pool_obj.annotations.key_value_pair
                    kvps_len = len(pool_obj_kvp)
                    for count in range(0, kvps_len):
                        kvp = {}
                        kvp['key'] = pool_obj_kvp[count].key
                        kvp['value'] = pool_obj_kvp[count].value
                        kvps.append(kvp)
                    annotations['key_value_pair'] = kvps
                else:
                    annotations = pool.annotations
                backend['pool_id'] = pool_id
                backend['annotations'] = {}
                for kvp in annotations['key_value_pair'] or []:
                    key = kvp['key']
                    value = kvp['value']
                    backend['annotations'][key] = value
                backend['pool'] = {}
                backend['pool']['protocol'] = pool.params['protocol']
                backend['member'] = {}
                if len(pool.members) == 0:
                    continue
                member_id = list(pool.members)[0]
                member = LoadbalancerMemberKM.get(member_id)
                if member.annotations is None:
                    annotations = {}
                    kvps = []
                    member_obj = self._vnc_lib. \
                                 loadbalancer_member_read(id=member_id)
                    member_obj_kvp = member_obj.annotations.key_value_pair
                    kvps_len = len(member_obj_kvp)
                    for count in range(0, kvps_len):
                        kvp = {}
                        kvp['key'] = member_obj_kvp[count].key
                        kvp['value'] = member_obj_kvp[count].value
                        kvps.append(kvp)
                    annotations['key_value_pair'] = kvps
                else:
                    annotations = member.annotations
                backend['member_id'] = member_id
                protocol_port = member.params['protocol_port']
                for kvp in annotations['key_value_pair'] or []:
                    if kvp['key'] == 'serviceName':
                        backend['member']['serviceName'] = kvp['value']
                        backend['member']['servicePort'] = protocol_port
                        break
            backend_list.append(backend)
        return backend_list

    def _get_tls_dict(self, spec, ns_name):
        tls_dict = {}
        if 'tls' in spec:
            tls_list = spec['tls']
            for tls in tls_list:
                if not 'secretName' in tls:
                    continue
                if 'hosts' in tls:
                    hosts = tls['hosts']
                else:
                    hosts = ['ALL']
                for host in hosts:
                    tls_dict[host] = ns_name + '__' + tls['secretName']
        return tls_dict

    def _get_new_backend_list(self, spec, ns_name):
        tls_dict = self._get_tls_dict(spec, ns_name)
        backend_list = []
        rules = []
        if 'rules' in spec:
            rules = spec['rules']
            for rule in rules:
                if 'http' not in rule:
                    continue
                paths = rule['http']['paths']
                for path in paths or []:
                    backend = {}
                    backend['annotations'] = {}
                    backend['listener'] = {}
                    backend['pool'] = {}
                    backend['member'] = {}
                    backend['listener']['protocol'] = 'HTTP'
                    backend['pool']['protocol'] = 'HTTP'
                    secretname = ""
                    virtual_host = False
                    if 'host' in rule:
                        host = rule['host']
                        backend['annotations']['host'] = host
                        if host in tls_dict.keys():
                            secretname = tls_dict[host]
                            virtual_host = True
                    if 'path' in path:
                        backend['annotations']['path'] = path['path']
                        if virtual_host == False and 'ALL' in tls_dict.keys():
                            secretname = 'ALL'
                    service = path['backend']
                    backend['annotations']['type'] = 'acl'
                    backend['member']['serviceName'] = service['serviceName']
                    backend['member']['servicePort'] = service['servicePort']
                    backend_list.append(backend)
                    if secretname:
                        backend_https = copy.deepcopy(backend)
                        backend_https['listener'][
                            'protocol'] = 'TERMINATED_HTTPS'
                        if virtual_host:
                            backend_https['listener']['sni_containers'] = [
                                secretname
                            ]
                        else:
                            backend_https['listener'][
                                'default_tls_container'] = tls_dict['ALL']
                        backend_list.append(backend_https)
        if 'backend' in spec:
            service = spec['backend']
            backend = {}
            backend['annotations'] = {}
            backend['listener'] = {}
            backend['pool'] = {}
            backend['member'] = {}
            backend['listener']['protocol'] = 'HTTP'
            backend['pool']['protocol'] = 'HTTP'
            backend['annotations']['type'] = 'default'
            backend['member']['serviceName'] = service['serviceName']
            backend['member']['servicePort'] = service['servicePort']
            backend_list.append(backend)
            if 'ALL' in tls_dict.keys():
                backend_https = copy.deepcopy(backend)
                backend_https['listener']['protocol'] = 'TERMINATED_HTTPS'
                backend_https['listener']['default_tls_container'] = tls_dict[
                    'ALL']
                backend_list.append(backend_https)
        return backend_list

    def _create_member(self, ns_name, backend_member, pool):
        resource_type = "services"
        service_name = backend_member['serviceName']
        service_port = backend_member['servicePort']
        service_info = self._kube.get_resource(resource_type, service_name,
                                               ns_name)
        member = None
        if service_info and 'clusterIP' in service_info['spec']:
            service_ip = service_info['spec']['clusterIP']
            self._logger.debug("%s - clusterIP for service %s - %s" \
                 %(self._name, service_name, service_ip))
            member_match = False
            annotations = {}
            annotations['serviceName'] = service_name
            for member_id in pool.members:
                member = LoadbalancerMemberKM.get(member_id)
                if member and member.params['address'] == service_ip \
                   and member.params['protocol_port'] == service_port:
                    member_match = True
                    break
            if not member_match:
                member_obj = self._vnc_create_member(pool, service_ip,
                                                     service_port, annotations)
                if member_obj:
                    member = LoadbalancerMemberKM.locate(member_obj.uuid)
                else:
                    self._logger.error(
                         "%s - (%s %s) Member Not Created for Pool %s" \
                         %(self._name, service_name,
                         str(service_port), pool.name))
        else:
            self._logger.error("%s - clusterIP for Service %s Not Found" \
                 %(self._name, service_name))
            self._logger.error(
                 "%s - (%s %s) Member Not Created for Pool %s" \
                 %(self._name, service_name,
                 str(service_port), pool.name))
        return member

    def _update_member(self, ns_name, backend_member, pool):
        resource_type = "services"
        member_id = backend_member['member_id']
        new_service_name = backend_member['serviceName']
        new_service_port = backend_member['servicePort']
        member = LoadbalancerMemberKM.get(member_id)
        annotations = member.annotations
        for kvp in annotations['key_value_pair'] or []:
            if kvp['key'] == 'serviceName':
                old_service_name = kvp['value']
                break
        old_service_port = member.params['protocol_port']
        service_ip = None
        if new_service_name != old_service_name:
            service_info = self._kube.get_resource(resource_type,
                                                   new_service_name, ns_name)
            if service_info and 'clusterIP' in service_info['spec']:
                service_ip = service_info['spec']['clusterIP']
            else:
                self._logger.error("%s - clusterIP for Service %s Not Found" \
                     %(self._name, new_service_name))
                self._logger.error(
                     "%s - (%s %s) Member Not Updated for Pool %s" \
                     %(self._name, new_service_name,
                     str(new_service_port), pool.name))
                self._vnc_delete_member(member_id)
                LoadbalancerMemberKM.delete(member_id)
                self._logger.error(
                     "%s - (%s %s) Member Deleted for Pool %s" \
                     %(self._name, old_service_name,
                     str(old_service_port), pool.name))
                return None
        else:
            service_ip = member.params['address']
        annotations = {}
        annotations['serviceName'] = new_service_name
        member_obj = self._vnc_update_member(member_id, service_ip,
                                             new_service_port, annotations)
        member = LoadbalancerMemberKM.update(member)
        return member

    def _create_pool(self, ns_name, ll, port, lb_algorithm, annotations):
        pool_id = ll.loadbalancer_pool
        pool = LoadbalancerPoolKM.get(pool_id)
        if pool is None:
            pool_obj = self._vnc_create_pool(ns_name, ll, port, lb_algorithm,
                                             annotations)
            pool_id = pool_obj.uuid
            pool = LoadbalancerPoolKM.locate(pool_id)
        else:
            self._logger.error("%s - %s Pool Not Created" \
                 %(self._name, ll.name))
        return pool

    def _create_listener(self, ns_name, lb, port):
        ll_obj = self._vnc_create_listeners(ns_name, lb, port)
        if ll_obj:
            ll = LoadbalancerListenerKM.locate(ll_obj.uuid)
        else:
            self._logger.error("%s - %s Listener for Port %s Not Created" \
                 %(self._name, lb.name, str(port)))
        return ll

    def _create_listener_pool_member(self, ns_name, lb, backend):
        pool_port = {}
        listener_port = {}
        listener_port['port'] = '80'
        listener_port['protocol'] = backend['listener']['protocol']
        if listener_port['protocol'] == 'TERMINATED_HTTPS':
            listener_port['port'] = '443'
            if 'default_tls_container' in backend['listener']:
                listener_port['default_tls_container'] = backend['listener'][
                    'default_tls_container']
            if 'sni_containers' in backend['listener']:
                listener_port['sni_containers'] = backend['listener'][
                    'sni_containers']
        ll = self._create_listener(ns_name, lb, listener_port)
        annotations = {}
        for key in backend['annotations']:
            annotations[key] = backend['annotations'][key]
        lb_algorithm = "ROUND_ROBIN"
        pool_port['port'] = '80'
        pool_port['protocol'] = backend['pool']['protocol']
        pool = self._create_pool(ns_name, ll, pool_port, lb_algorithm,
                                 annotations)
        backend_member = backend['member']
        member = self._create_member(ns_name, backend_member, pool)
        if member is None:
            self._logger.error("%s - Deleting Listener %s and Pool %s" \
                %(self._name, ll.name, pool.name))
            self._vnc_delete_pool(pool.uuid)
            LoadbalancerPoolKM.delete(pool.uuid)
            self._vnc_delete_listener(ll.uuid)
            LoadbalancerListenerKM.delete(ll.uuid)

    def update_ingress_backend(self, ns_name, service_name, oper):
        ingress_ids = self._find_ingress(self._ingress_label_cache, ns_name,
                                         service_name)
        for ingress_id in ingress_ids or []:
            ingress = IngressKM.get(ingress_id)
            lb = LoadbalancerKM.get(ingress_id)
            if not ingress or not lb:
                continue
            if oper == 'ADD':
                new_backend_list = self._get_new_backend_list(
                    ingress.spec, ns_name)
                for new_backend in new_backend_list[:] or []:
                    if new_backend['member']['serviceName'] == service_name:

                        # Create a firewall rule for ingress to this service.
                        fw_uuid = VncIngress.add_ingress_to_service_rule(
                            ns_name, ingress.name, service_name)
                        lb.add_firewall_rule(fw_uuid)

                        self._create_listener_pool_member(
                            ns_name, lb, new_backend)
            else:
                old_backend_list = self._get_old_backend_list(lb)
                for old_backend in old_backend_list[:] or []:
                    if old_backend['member']['serviceName'] == service_name:
                        self._delete_listener(old_backend['listener_id'])

                        # Delete rules created for this ingress to service.
                        deleted_fw_rule_uuid =\
                            VncIngress.delete_ingress_to_service_rule(ns_name,
                                                                  ingress.name,
                                                                  service_name)
                        lb.remove_firewall_rule(deleted_fw_rule_uuid)

    def _create_lb(self, uid, name, ns_name, event):
        annotations = event['object']['metadata'].get('annotations')
        ingress_controller = 'opencontrail'
        if annotations:
            if 'kubernetes.io/ingress.class' in annotations:
                ingress_controller = annotations['kubernetes.io/ingress.class']
        if ingress_controller != 'opencontrail':
            self._logger.warning(
                "%s - ingress controller is not opencontrail for ingress %s" %
                (self._name, name))
            self._delete_ingress(uid)
            return
        lb = LoadbalancerKM.get(uid)
        if not lb:
            lb_obj = self._vnc_create_lb(uid, name, ns_name, annotations)
            if lb_obj is None:
                return
            lb = LoadbalancerKM.locate(uid)
        else:
            external_ip = None
            if annotations and 'externalIP' in annotations:
                external_ip = annotations['externalIP']
            if external_ip != lb.external_ip:
                self._deallocate_floating_ip(lb)
                lb_obj = self._vnc_lib.loadbalancer_read(id=lb.uuid)
                fip = self._update_floating_ip(name, ns_name, external_ip,
                                               lb_obj)
                if fip:
                    lb.external_ip = external_ip
                self._update_kube_api_server(name, ns_name, lb_obj, fip)

        self._clear_ingress_cache_uuid(self._ingress_label_cache, uid)

        spec = event['object']['spec']
        new_backend_list = self._get_new_backend_list(spec, ns_name)
        old_backend_list = self._get_old_backend_list(lb)

        # find the unchanged backends
        for new_backend in new_backend_list[:] or []:
            self._update_ingress_cache(self._ingress_label_cache, ns_name,
                                       new_backend['member']['serviceName'],
                                       uid)
            for old_backend in old_backend_list[:] or []:
                if new_backend['annotations'] == old_backend['annotations'] \
                    and new_backend['listener'] == old_backend['listener'] \
                    and new_backend['pool'] == old_backend['pool'] \
                    and new_backend['member'] == old_backend['member']:

                    # Create a firewall rule for this member.
                    fw_uuid = VncIngress.add_ingress_to_service_rule(
                        ns_name, name, new_backend['member']['serviceName'])
                    lb.add_firewall_rule(fw_uuid)

                    old_backend_list.remove(old_backend)
                    new_backend_list.remove(new_backend)
                    break
        if len(old_backend_list) == 0 and len(new_backend_list) == 0:
            return lb

        # find the updated backends and update
        backend_update_list = []
        for new_backend in new_backend_list[:] or []:
            for old_backend in old_backend_list[:] or []:
                if new_backend['annotations'] == old_backend['annotations'] \
                    and new_backend['listener'] == old_backend['listener'] \
                    and new_backend['pool'] == old_backend['pool']:
                    backend = old_backend
                    backend['member']['member_id'] = \
                                     old_backend['member_id']
                    backend['member']['serviceName'] = \
                                     new_backend['member']['serviceName']
                    backend['member']['servicePort'] = \
                                     new_backend['member']['servicePort']
                    backend_update_list.append(backend)
                    old_backend_list.remove(old_backend)
                    new_backend_list.remove(new_backend)
        for backend in backend_update_list or []:
            ll = LoadbalancerListenerKM.get(backend['listener_id'])
            pool = LoadbalancerPoolKM.get(backend['pool_id'])
            backend_member = backend['member']
            member = self._update_member(ns_name, backend_member, pool)
            if member is None:
                self._logger.error("%s - Deleting Listener %s and Pool %s" \
                     %(self._name, ll.name, pool.name))
                self._vnc_delete_pool(pool.uuid)
                LoadbalancerPoolKM.delete(pool.uuid)
                self._vnc_delete_listener(ll.uuid)
                LoadbalancerListenerKM.delete(ll.uuid)
        if len(old_backend_list) == 0 and len(new_backend_list) == 0:
            return lb

        # delete the old backends
        for backend in old_backend_list or []:
            self._delete_listener(backend['listener_id'])

            deleted_fw_rule_uuid =\
                VncIngress.delete_ingress_to_service_rule(ns_name,
                    name, backend['member']['serviceName'])
            lb.remove_firewall_rule(deleted_fw_rule_uuid)

        # create the new backends
        for backend in new_backend_list:

            # Create a firewall rule for this member.
            fw_uuid = VncIngress.add_ingress_to_service_rule(
                ns_name, name, backend['member']['serviceName'])
            lb.add_firewall_rule(fw_uuid)

            self._create_listener_pool_member(ns_name, lb, backend)

        return lb

    def _delete_all_listeners(self, lb):
        listener_list = lb.loadbalancer_listeners.copy()
        for ll_id in listener_list:
            ll = LoadbalancerListenerKM.get(ll_id)
            pool_id = ll.loadbalancer_pool
            if pool_id:
                pool = LoadbalancerPoolKM.get(pool_id)
                member_list = pool.members.copy()
                for member_id in member_list:
                    self._vnc_delete_member(member_id)
                    LoadbalancerMemberKM.delete(member_id)
                self._vnc_delete_pool(pool_id)
                LoadbalancerPoolKM.delete(pool_id)
            self._vnc_delete_listener(ll_id)
            LoadbalancerListenerKM.delete(ll_id)

    def _delete_listener(self, ll_id):
        ll = LoadbalancerListenerKM.get(ll_id)
        pool_id = ll.loadbalancer_pool
        if pool_id:
            pool = LoadbalancerPoolKM.get(pool_id)
            member_list = pool.members.copy()
            for member_id in member_list:
                self._vnc_delete_member(member_id)
                LoadbalancerMemberKM.delete(member_id)
            self._vnc_delete_pool(pool_id)
            LoadbalancerPoolKM.delete(pool_id)
        self._vnc_delete_listener(ll_id)
        LoadbalancerListenerKM.delete(ll_id)

    def _delete_lb(self, uid):
        lb = LoadbalancerKM.get(uid)
        if not lb:
            return
        # Delete rules created for this member.
        firewall_rules = set(lb.get_firewall_rules())
        for fw_rule_uuid in firewall_rules:
            VncIngress.delete_ingress_to_service_rule_by_id(fw_rule_uuid)
            lb.remove_firewall_rule(fw_rule_uuid)

        self._delete_all_listeners(lb)
        self._vnc_delete_lb(lb)
        LoadbalancerKM.delete(uid)

    def _update_ingress(self, name, uid, event):
        ns_name = event['object']['metadata'].get('namespace')
        self._create_lb(uid, name, ns_name, event)

    def _delete_ingress(self, uid):
        self._delete_lb(uid)
        self._clear_ingress_cache_uuid(self._ingress_label_cache, uid)

    def _create_ingress_event(self, event_type, ingress_id, lb):
        event = {}
        object = {}
        object['kind'] = 'Ingress'
        object['spec'] = {}
        object['metadata'] = {}
        object['metadata']['uid'] = ingress_id
        if event_type == 'delete':
            event['type'] = 'DELETED'
            event['object'] = object
            self._queue.put(event)
        return

    def _sync_ingress_lb(self):
        lb_uuid_set = set(LoadbalancerKM.keys())
        ingress_uuid_set = set(IngressKM.keys())
        deleted_ingress_set = lb_uuid_set - ingress_uuid_set
        for uuid in deleted_ingress_set:
            lb = LoadbalancerKM.get(uuid)
            if not lb:
                continue
            if not lb.annotations:
                continue
            owner = None
            kind = None
            cluster = None
            for kvp in lb.annotations['key_value_pair'] or []:
                if kvp['key'] == 'cluster':
                    cluster = kvp['value']
                elif kvp['key'] == 'owner':
                    owner = kvp['value']
                elif kvp['key'] == 'kind':
                    kind = kvp['value']

                if cluster == vnc_kube_config.cluster_name() and \
                   owner == 'k8s' and \
                   kind == self._k8s_event_type:
                    self._create_ingress_event('delete', uuid, lb)
                    break
        return

    def ingress_timer(self):
        self._sync_ingress_lb()

    @classmethod
    def get_ingress_label_name(self, ns_name, name):
        return "-".join([vnc_kube_config.cluster_name(), ns_name, name])

    def process(self, event):
        event_type = event['type']
        kind = event['object'].get('kind')
        ns_name = event['object']['metadata'].get('namespace')
        name = event['object']['metadata'].get('name')
        uid = event['object']['metadata'].get('uid')

        print("%s - Got %s %s %s:%s:%s" %
              (self._name, event_type, kind, ns_name, name, uid))
        self._logger.debug("%s - Got %s %s %s:%s:%s" %
                           (self._name, event_type, kind, ns_name, name, uid))

        if event['type'] == 'ADDED' or event['type'] == 'MODIFIED':

            #
            # Construct and add labels for this ingress.
            # Following labels are added by infra:
            #
            # 1. A label for the ingress object.
            # 2. A label for the namespace of ingress object.
            #
            labels = self._labels.get_ingress_label(
                self.get_ingress_label_name(ns_name, name))
            labels.update(self._labels.get_namespace_label(ns_name))
            self._labels.process(uid, labels)

            self._update_ingress(name, uid, event)

        elif event['type'] == 'DELETED':
            # Dis-associate infra labels from refernced VMI's.
            self.remove_ingress_labels(ns_name, name)

            self._delete_ingress(uid)

            # Delete labels added by infra for this ingress.
            self._labels.process(uid)
        else:
            self._logger.warning('Unknown event type: "{}" Ignoring'.format(
                event['type']))

    def remove_ingress_labels(self, ns_name, name):
        """
        Remove ingress infra label/tag from VMI's corresponding to the services of
        this ingress.

        For each ingress service, kube-manager will create a infra label to add
        rules that allow traffic from ingress VMI to backend service VMI's.

        Ingress is a special case where tags created by kube-manager are attached
        to VMI's that are not created/managed by kube-manager. Since the ingress
        label/tag is being deleted, dis-associate this tag from all VMI's on which
        it is referred.
        """
        if not self.tag_mgr or not ns_name or not name:
            return

        # Get labels for this ingress service.
        labels = self._labels.get_ingress_label(
            self.get_ingress_label_name(ns_name, name))
        for type, value in labels.iteritems():
            tag_obj = self.tag_mgr.read(type, value)
            if tag_obj:
                vmi_refs = tag_obj.get_virtual_machine_interface_back_refs()
                for vmi in vmi_refs if vmi_refs else []:
                    vmi_obj = self._vnc_lib.virtual_machine_interface_read(
                        id=vmi['uuid'])
                    self._vnc_lib.unset_tag(vmi_obj, type)

    def create_ingress_security_policy(self):
        """
        Create a FW policy to house all ingress-to-service rules.
        """
        if not VncSecurityPolicy.ingress_svc_fw_policy_uuid:
            VncSecurityPolicy.ingress_svc_fw_policy_uuid =\
              VncSecurityPolicy.create_firewall_policy(
                "-".join([vnc_kube_config.cluster_name(), self._k8s_event_type]),
                None, None, is_global=True)
            VncSecurityPolicy.add_firewall_policy(
                VncSecurityPolicy.ingress_svc_fw_policy_uuid)

    @classmethod
    def _get_ingress_firewall_rule_name(cls, ns_name, ingress_name, svc_name):
        return "-".join([
            vnc_kube_config.cluster_name(), "Ingress", ns_name, ingress_name,
            svc_name
        ])

    @classmethod
    def add_ingress_to_service_rule(cls, ns_name, ingress_name, service_name):
        """
        Add a ingress-to-service allow rule to ingress firewall policy.
        """
        if VncSecurityPolicy.ingress_svc_fw_policy_uuid:

            ingress_labels = XLabelCache.get_ingress_label(
                cls.get_ingress_label_name(ns_name, ingress_name))
            service_labels = XLabelCache.get_service_label(service_name)

            rule_name = VncIngress._get_ingress_firewall_rule_name(
                ns_name, ingress_name, service_name)

            fw_rule_uuid = VncSecurityPolicy.create_firewall_rule_allow_all(
                rule_name, service_labels, ingress_labels)

            VncSecurityPolicy.add_firewall_rule(
                VncSecurityPolicy.ingress_svc_fw_policy_uuid, fw_rule_uuid)

            return fw_rule_uuid

    @classmethod
    def delete_ingress_to_service_rule(cls, ns_name, ingress_name,
                                       service_name):
        """
        Delete the ingress-to-service allow rule added to ingress firewall
        policy.
        """
        rule_uuid = None
        if VncSecurityPolicy.ingress_svc_fw_policy_uuid:
            rule_name = VncIngress._get_ingress_firewall_rule_name(
                ns_name, ingress_name, service_name)

            # Get the rule id of the rule to be deleted.
            rule_uuid = VncSecurityPolicy.get_firewall_rule_uuid(rule_name)
            if rule_uuid:
                # Delete the rule.
                VncSecurityPolicy.delete_firewall_rule(
                    VncSecurityPolicy.ingress_svc_fw_policy_uuid, rule_uuid)

        return rule_uuid

    @classmethod
    def delete_ingress_to_service_rule_by_id(cls, rule_uuid):
        if VncSecurityPolicy.ingress_svc_fw_policy_uuid:
            # Delete the rule.
            VncSecurityPolicy.delete_firewall_rule(
                VncSecurityPolicy.ingress_svc_fw_policy_uuid, rule_uuid)
class VncIngress(VncCommon):
    def __init__(self):
        super(VncIngress,self).__init__('Ingress')
        self._name = type(self).__name__
        self._args = vnc_kube_config.args()
        self._queue = vnc_kube_config.queue()
        self._vnc_lib = vnc_kube_config.vnc_lib()
        self._logger = vnc_kube_config.logger()
        self._kube = vnc_kube_config.kube()
        self._default_vn_obj = None
        self._fip_pool_obj = None
        self.service_lb_mgr = ServiceLbManager()
        self.service_ll_mgr = ServiceLbListenerManager()
        self.service_lb_pool_mgr = ServiceLbPoolManager()
        self.service_lb_member_mgr = ServiceLbMemberManager()

    def _get_namespace(self, ns_name):
        return NamespaceKM.find_by_name_or_uuid(ns_name)

    def _get_project(self, ns_name):
        proj_fq_name = vnc_kube_config.cluster_project_fq_name(ns_name)
        try:
            proj_obj = self._vnc_lib.project_read(fq_name=proj_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" %(self._name, proj_fq_name))
            return None
        return proj_obj

    def _get_network(self, ns_name):
        ns = self._get_namespace(ns_name)
        if ns.is_isolated():
            vn_fq_name = ns.get_network_fq_name()
        else:
            if self._default_vn_obj:
                return self._default_vn_obj
            proj_fq_name = vnc_kube_config.cluster_default_project_fq_name()
            vn_fq_name = proj_fq_name +\
                [vnc_kube_config.cluster_default_network_name()]
        try:
            vn_obj = self._vnc_lib.virtual_network_read(fq_name=vn_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" %(self._name, vn_fq_name))
            return None
        if not ns.is_isolated():
            self._default_vn_obj = vn_obj
        return vn_obj

    def _get_pod_ipam_subnet_uuid(self, vn_obj):
        pod_ipam_subnet_uuid = None
        fq_name = vnc_kube_config.pod_ipam_fq_name()
        vn = VirtualNetworkKM.find_by_name_or_uuid(vn_obj.get_uuid())
        pod_ipam_subnet_uuid = vn.get_ipam_subnet_uuid(fq_name)
        if pod_ipam_subnet_uuid is None:
            self._logger.error("%s - %s Not Found" %(self._name, fq_name))
        return pod_ipam_subnet_uuid

    def _get_public_fip_pool(self):
        if self._fip_pool_obj:
            return self._fip_pool_obj
        fip_pool_fq_name = [vnc_kube_config.cluster_domain(),
                            self._args.public_network_project,
                            self._args.public_network,
                            self._args.public_fip_pool]
        try:
            fip_pool_obj = self._vnc_lib. \
                           floating_ip_pool_read(fq_name=fip_pool_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" \
                 %(self._name, fip_pool_fq_name))
            return None
        self._fip_pool_obj = fip_pool_obj
        return fip_pool_obj

    def _get_floating_ip(self, name, proj_obj, vmi_obj=None):
        fip_refs = vmi_obj.get_floating_ip_back_refs()
        for ref in fip_refs or []:
            fip = FloatingIpKM.get(ref['uuid'])
            if fip:
                return fip
            else:
                break
        fip_pool = self._get_public_fip_pool()
        if fip_pool is None:
            return None
        fip_uuid = str(uuid.uuid4())
        fip_name = VncCommon.make_name(name, fip_uuid)
        fip_obj = FloatingIp(fip_name, fip_pool)
        fip_obj.uuid = fip_uuid
        fip_obj.set_project(proj_obj)
        if vmi_obj:
            fip_obj.set_virtual_machine_interface(vmi_obj)
        self._vnc_lib.floating_ip_create(fip_obj)
        fip = FloatingIpKM.locate(fip_obj.uuid)
        return fip

    def _allocate_floating_ip(self, lb_obj, name, proj_obj):
        vmi_id = lb_obj.virtual_machine_interface_refs[0]['uuid']
        vmi_obj = self._vnc_lib.virtual_machine_interface_read(id=vmi_id)
        if vmi_obj is None:
            self._logger.error("%s - %s Vmi %s Not Found" \
                 %(self._name, lb_obj.name, vmi_id))
            return None
        fip = self._get_floating_ip(name, proj_obj, vmi_obj)
        return fip

    def _deallocate_floating_ip(self, lb):
        vmi_id = list(lb.virtual_machine_interfaces)[0]
        vmi = VirtualMachineInterfaceKM.get(vmi_id)
        if vmi is None:
            self._logger.error("%s - %s Vmi %s Not Found" \
                 %(self._name, lb.name, vmi_id))
            return
        fip_list = vmi.floating_ips.copy()
        for fip_id in fip_list or []:
            fip_obj = self._vnc_lib.floating_ip_read(id=fip_id)
            fip_obj.set_virtual_machine_interface_list([])
            self._vnc_lib.floating_ip_update(fip_obj)
            self._vnc_lib.floating_ip_delete(id=fip_obj.uuid)
            FloatingIpKM.delete(fip_obj.uuid)

    def _vnc_create_member(self, pool, address, port, annotations):
        pool_obj = self.service_lb_pool_mgr.read(pool.uuid)
        member_obj = self.service_lb_member_mgr.create(pool_obj,
                          address, port, annotations)
        return member_obj

    def _vnc_update_member(self, member_id, address, port, annotations):
        member_obj = self.service_lb_member_mgr.update(member_id,
                          address, port, annotations)
        return member_obj

    def _vnc_create_pool(self, ns_name, ll, port, lb_algorithm, annotations):
        proj_obj = self._get_project(ns_name)
        ll_obj = self.service_ll_mgr.read(ll.uuid)
        pool_obj = self.service_lb_pool_mgr.create(ll_obj, proj_obj,
                                            port, lb_algorithm, annotations)
        return pool_obj

    def _vnc_create_listeners(self, ns_name, lb, port):
        proj_obj = self._get_project(ns_name)
        lb_obj = self.service_lb_mgr.read(lb.uuid)
        ll_obj = self.service_ll_mgr.create(lb_obj, proj_obj, port)
        return ll_obj

    def _vnc_create_lb(self, uid, name, ns_name):
        lb_provider = 'opencontrail'
        proj_obj = self._get_project(ns_name)
        vn_obj = self._get_network(ns_name)
        if proj_obj is None or vn_obj is None:
            return None

        vip_address = None
        annotations = {}
        annotations['owner'] = 'k8s'
        pod_ipam_subnet_uuid = self._get_pod_ipam_subnet_uuid(vn_obj)
        lb_obj = self.service_lb_mgr.create(lb_provider, vn_obj,
                            ns_name, uid, name, proj_obj, vip_address,
                            pod_ipam_subnet_uuid, annotations=annotations)
        if lb_obj:
            vip_info = {}
            vip_info['clusterIP'] = lb_obj._loadbalancer_properties.vip_address
            fip_obj = self._allocate_floating_ip(lb_obj, name, proj_obj)
            if fip_obj:
                vip_info['externalIP'] = fip_obj.address
            patch = {'metadata': {'annotations': vip_info}}
            self._kube.patch_resource("ingresses", name,
                                      patch, ns_name, beta=True)
        else:
            self._logger.error("%s - %s LB Not Created" %(self._name, name))

        return lb_obj

    def _vnc_delete_member(self, member_id):
        self.service_lb_member_mgr.delete(member_id)

    def _vnc_delete_pool(self, pool_id):
        self.service_lb_pool_mgr.delete(pool_id)

    def _vnc_delete_listener(self, ll_id):
        self.service_ll_mgr.delete(ll_id)

    def _vnc_delete_lb(self, lb):
        self._deallocate_floating_ip(lb)
        self.service_lb_mgr.delete(lb.uuid)

    def _get_old_backend_list(self, lb):
        backend_list = []
        listener_list = lb.loadbalancer_listeners
        for ll_id in listener_list:
            backend = {}
            backend['listener_id'] = ll_id
            ll = LoadbalancerListenerKM.get(ll_id)
            pool_id = ll.loadbalancer_pool
            if pool_id:
                pool = LoadbalancerPoolKM.get(pool_id)
                if pool.annotations is None:
                    annotations = {}
                    kvps = []
                    pool_obj = self._vnc_lib.loadbalancer_pool_read(id=pool_id)
                    pool_obj_kvp = pool_obj.annotations.key_value_pair
                    kvps_len = len(pool_obj_kvp)
                    for count in range(0, kvps_len):
                         kvp = {}
                         kvp['key'] = pool_obj_kvp[count].key
                         kvp['value'] = pool_obj_kvp[count].value
                         kvps.append(kvp)
                    annotations['key_value_pair'] = kvps
                else:
                    annotations = pool.annotations
                backend['pool_id'] = pool_id
                backend['annotations'] = {}
                for kvp in annotations['key_value_pair'] or []:
                    key = kvp['key']
                    value = kvp['value']
                    backend['annotations'][key] = value
                backend['member'] = {}
                if len(pool.members) == 0:
                    continue
                member_id = list(pool.members)[0]
                member = LoadbalancerMemberKM.get(member_id)
                if member.annotations is None:
                    annotations = {}
                    kvps = []
                    member_obj = self._vnc_lib. \
                                 loadbalancer_member_read(id=member_id)
                    member_obj_kvp = member_obj.annotations.key_value_pair
                    kvps_len = len(member_obj_kvp)
                    for count in range(0, kvps_len):
                         kvp = {}
                         kvp['key'] = member_obj_kvp[count].key
                         kvp['value'] = member_obj_kvp[count].value
                         kvps.append(kvp)
                    annotations['key_value_pair'] = kvps
                else:
                    annotations = member.annotations
                backend['member_id'] = member_id
                protocol_port = member.params['protocol_port']
                for kvp in annotations['key_value_pair'] or []:
                    if kvp['key'] == 'serviceName':
                        backend['member']['serviceName'] = kvp['value']
                        backend['member']['servicePort'] = protocol_port
                        break
            backend_list.append(backend)
        return backend_list

    def _get_new_backend_list(self, spec):
        backend_list = []
        rules = []
        if 'rules' in spec:
            rules = spec['rules']
            for rule in rules:
                if 'http' not in rule:
                    continue
                paths = rule['http']['paths']
                for path in paths or []:
                    backend = {}
                    backend['annotations'] = {}
                    backend['member'] = {}
                    backend['protocol'] = 'HTTP'
                    if 'host' in rule:
                        backend['annotations']['host'] = rule['host']
                    if 'path' in path:
                        backend['annotations']['path'] = path['path']
                    service = path['backend']
                    backend['annotations']['type'] = 'acl'
                    backend['member']['serviceName'] = service['serviceName']
                    backend['member']['servicePort'] = service['servicePort']
                    backend_list.append(backend)
        if 'backend' in spec:
            service = spec['backend']
            backend = {}
            backend['annotations'] = {}
            backend['member'] = {}
            backend['protocol'] = 'HTTP'
            backend['annotations']['type'] = 'default'
            backend['member']['serviceName'] = service['serviceName']
            backend['member']['servicePort'] = service['servicePort']
            backend_list.append(backend)
        return backend_list

    def _create_member(self, ns_name, backend_member, pool):
        resource_type = "services"
        service_name = backend_member['serviceName']
        service_port = backend_member['servicePort']
        service_info = self._kube.get_resource(resource_type,
                       service_name, ns_name)
        member = None
        if service_info and 'clusterIP' in service_info['spec']:
            service_ip = service_info['spec']['clusterIP']
            self._logger.debug("%s - clusterIP for service %s - %s" \
                 %(self._name, service_name, service_ip))
            member_match = False
            annotations = {}
            annotations['serviceName'] = service_name
            for member_id in pool.members:
                member = LoadbalancerMemberKM.get(member_id)
                if member and member.params['address'] == service_ip \
                   and member.params['protocol_port'] == service_port:
                    member_match = True
                    break
            if not member_match:
                member_obj = self._vnc_create_member(pool,
                                  service_ip, service_port, annotations)
                if member_obj:
                    member = LoadbalancerMemberKM.locate(member_obj.uuid)
                else:
                    self._logger.error(
                         "%s - (%s %s) Member Not Created for Pool %s" \
                         %(self._name, service_name,
                         str(service_port), pool.name))
        else:
            self._logger.error("%s - clusterIP for Service %s Not Found" \
                 %(self._name, service_name))
            self._logger.error(
                 "%s - (%s %s) Member Not Created for Pool %s" \
                 %(self._name, service_name,
                 str(service_port), pool.name))
        return member

    def _update_member(self, ns_name, backend_member, pool):
        resource_type = "services"
        member_id = backend_member['member_id']
        new_service_name = backend_member['serviceName']
        new_service_port = backend_member['servicePort']
        member = LoadbalancerMemberKM.get(member_id)
        annotations = member.annotations
        for kvp in annotations['key_value_pair'] or []:
            if kvp['key'] == 'serviceName':
                old_service_name = kvp['value']
                break
        old_service_port = member.params['protocol_port']
        service_ip = None
        if new_service_name != old_service_name:
            service_info = self._kube.get_resource(resource_type,
                           new_service_name, ns_name)
            if service_info and 'clusterIP' in service_info['spec']:
                service_ip = service_info['spec']['clusterIP']
            else:
                self._logger.error("%s - clusterIP for Service %s Not Found" \
                     %(self._name, new_service_name))
                self._logger.error(
                     "%s - (%s %s) Member Not Updated for Pool %s" \
                     %(self._name, new_service_name,
                     str(new_service_port), pool.name))
                self._vnc_delete_member(member_id)
                LoadbalancerMemberKM.delete(member_id)
                self._logger.error(
                     "%s - (%s %s) Member Deleted for Pool %s" \
                     %(self._name, old_service_name,
                     str(old_service_port), pool.name))
                return None
        else:
            service_ip = member.params['address']
        annotations = {}
        annotations['serviceName'] = new_service_name
        member_obj = self._vnc_update_member(member_id,
                     service_ip, new_service_port, annotations)
        member = LoadbalancerMemberKM.update(member)
        return member

    def _create_pool(self, ns_name, ll, port, lb_algorithm, annotations):
        pool_id = ll.loadbalancer_pool
        pool = LoadbalancerPoolKM.get(pool_id)
        if pool is None:
            pool_obj = self._vnc_create_pool(ns_name, ll,
                            port, lb_algorithm, annotations)
            pool_id = pool_obj.uuid
            pool = LoadbalancerPoolKM.locate(pool_id)
        else:
            self._logger.error("%s - %s Pool Not Created" \
                 %(self._name, ll.name))
        return pool

    def _create_listener(self, ns_name, lb, port):
        ll_obj = self._vnc_create_listeners(ns_name, lb, port)
        if ll_obj:
            ll = LoadbalancerListenerKM.locate(ll_obj.uuid)
        else:
            self._logger.error("%s - %s Listener for Port %s Not Created" \
                 %(self._name, lb.name, str(port)))
        return ll

    def _create_lb(self, uid, name, ns_name, event):
        lb = LoadbalancerKM.get(uid)
        if not lb:
            lb_obj = self._vnc_create_lb(uid, name, ns_name)
            if lb_obj is None:
                return
            lb = LoadbalancerKM.locate(uid)

        spec = event['object']['spec']
        new_backend_list = self._get_new_backend_list(spec)
        old_backend_list = self._get_old_backend_list(lb)

        # find the unchanged backends
        for new_backend in new_backend_list[:] or []:
            for old_backend in old_backend_list[:] or []:
                if new_backend['annotations'] == old_backend['annotations'] \
                    and new_backend['member'] == old_backend['member']:
                    old_backend_list.remove(old_backend)
                    new_backend_list.remove(new_backend)
        if len(old_backend_list) == 0 and len(new_backend_list) == 0:
            return lb

        # find the updated backends and update
        backend_update_list = []
        for new_backend in new_backend_list[:] or []:
            for old_backend in old_backend_list[:] or []:
                if new_backend['annotations'] == old_backend['annotations']:
                    backend = old_backend
                    backend['member']['member_id'] = \
                                     old_backend['member_id']
                    backend['member']['serviceName'] = \
                                     new_backend['member']['serviceName']
                    backend['member']['servicePort'] = \
                                     new_backend['member']['servicePort']
                    backend_update_list.append(backend)
                    old_backend_list.remove(old_backend)
                    new_backend_list.remove(new_backend)
        for backend in backend_update_list or []:
            ll = LoadbalancerListenerKM.get(backend['listener_id'])
            pool = LoadbalancerPoolKM.get(backend['pool_id'])
            backend_member = backend['member']
            member = self._update_member(ns_name, backend_member, pool)
            if member is None:
                self._logger.error("%s - Deleting Listener %s and Pool %s" \
                     %(self._name, ll.name, pool.name))
                self._vnc_delete_pool(pool.uuid)
                LoadbalancerPoolKM.delete(pool.uuid)
                self._vnc_delete_listener(ll.uuid)
                LoadbalancerListenerKM.delete(ll.uuid)
        if len(old_backend_list) == 0 and len(new_backend_list) == 0:
            return lb

        # delete the old backends
        for backend in old_backend_list or []:
            self._delete_listener(backend['listener_id'])

        # create the new backends
        port = {}
        lb_algorithm = "ROUND_ROBIN"
        port['protocol'] = 'HTTP'
        port['port'] = '80'
        for backend in new_backend_list:
            ll = self._create_listener(ns_name, lb, port)
            annotations = {}
            for key in backend['annotations']:
                annotations[key] = backend['annotations'][key]
            port['protocol'] = backend['protocol']
            pool = self._create_pool(ns_name, ll, port, lb_algorithm, annotations)
            backend_member = backend['member']
            member = self._create_member(ns_name, backend_member, pool)
            if member is None:
                self._logger.error("%s - Deleting Listener %s and Pool %s" \
                     %(self._name, ll.name, pool.name))
                self._vnc_delete_pool(pool.uuid)
                LoadbalancerPoolKM.delete(pool.uuid)
                self._vnc_delete_listener(ll.uuid)
                LoadbalancerListenerKM.delete(ll.uuid)

        return lb

    def _delete_all_listeners(self, lb):
        listener_list = lb.loadbalancer_listeners.copy()
        for ll_id in listener_list:
            ll = LoadbalancerListenerKM.get(ll_id)
            pool_id = ll.loadbalancer_pool
            if pool_id:
                pool = LoadbalancerPoolKM.get(pool_id)
                member_list = pool.members.copy()
                for member_id in member_list:
                    self._vnc_delete_member(member_id)
                    LoadbalancerMemberKM.delete(member_id)
                self._vnc_delete_pool(pool_id)
                LoadbalancerPoolKM.delete(pool_id)
            self._vnc_delete_listener(ll_id)
            LoadbalancerListenerKM.delete(ll_id)

    def _delete_listener(self, ll_id):
        ll = LoadbalancerListenerKM.get(ll_id)
        pool_id = ll.loadbalancer_pool
        if pool_id:
            pool = LoadbalancerPoolKM.get(pool_id)
            member_list = pool.members.copy()
            for member_id in member_list:
                self._vnc_delete_member(member_id)
                LoadbalancerMemberKM.delete(member_id)
            self._vnc_delete_pool(pool_id)
            LoadbalancerPoolKM.delete(pool_id)
        self._vnc_delete_listener(ll_id)
        LoadbalancerListenerKM.delete(ll_id)

    def _delete_lb(self, uid):
        lb = LoadbalancerKM.get(uid)
        if not lb:
            return
        self._delete_all_listeners(lb)
        self._vnc_delete_lb(lb)
        LoadbalancerKM.delete(uid)

    def _update_ingress(self, name, uid, event):
        ns_name = event['object']['metadata'].get('namespace')
        self._create_lb(uid, name, ns_name, event)

    def _delete_ingress(self, uid):
        self._delete_lb(uid)

    def _create_ingress_event(self, event_type, ingress_id, lb):
        event = {}
        object = {}
        object['kind'] = 'Ingress'
        object['spec'] = {}
        object['metadata'] = {}
        object['metadata']['uid'] = ingress_id
        if event_type == 'delete':
            event['type'] = 'DELETED'
            event['object'] = object
            self._queue.put(event)
        return

    def _sync_ingress_lb(self):
        lb_uuid_list = list(LoadbalancerKM.keys())
        ingress_uuid_list = list(IngressKM.keys())
        for uuid in lb_uuid_list:
            if uuid in ingress_uuid_list:
                continue
            lb = LoadbalancerKM.get(uuid)
            if not lb:
                continue
            if not lb.annotations:
                continue
            for kvp in lb.annotations['key_value_pair'] or []:
                if kvp['key'] == 'owner' \
                   and kvp['value'] == 'k8s':
                    self._create_ingress_event('delete', uuid, lb)
                    break
        return

    def ingress_timer(self):
        self._sync_ingress_lb()

    def process(self, event):
        event_type = event['type']
        kind = event['object'].get('kind')
        name = event['object']['metadata'].get('name')
        uid = event['object']['metadata'].get('uid')
        ns_name = event['object']['metadata'].get('namespace')

        print("%s - Got %s %s %s:%s"
              %(self._name, event_type, kind, ns_name, name))
        self._logger.debug("%s - Got %s %s %s:%s"
              %(self._name, event_type, kind, ns_name, name))

        if event['type'] == 'ADDED' or event['type'] == 'MODIFIED':
            self._update_ingress(name, uid, event)
        elif event['type'] == 'DELETED':
            self._delete_ingress(uid)
Beispiel #3
0
class VncIngress(VncCommon):
    def __init__(self):
        self._k8s_event_type = 'Ingress'
        super(VncIngress, self).__init__(self._k8s_event_type)
        self._name = type(self).__name__
        self._args = vnc_kube_config.args()
        self._queue = vnc_kube_config.queue()
        self._vnc_lib = vnc_kube_config.vnc_lib()
        self._logger = vnc_kube_config.logger()
        self._kube = vnc_kube_config.kube()
        self._label_cache = vnc_kube_config.label_cache()
        self._service_fip_pool = vnc_kube_config.service_fip_pool()
        self._ingress_label_cache = {}
        self._default_vn_obj = None
        self._fip_pool_obj = None
        self.service_lb_mgr = ServiceLbManager()
        self.service_ll_mgr = ServiceLbListenerManager()
        self.service_lb_pool_mgr = ServiceLbPoolManager()
        self.service_lb_member_mgr = ServiceLbMemberManager()

    def _get_project(self, ns_name):
        proj_fq_name = vnc_kube_config.cluster_project_fq_name(ns_name)
        try:
            proj_obj = self._vnc_lib.project_read(fq_name=proj_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" %
                               (self._name, proj_fq_name))
            return None
        return proj_obj

    def _get_namespace(self, ns_name):
        return NamespaceKM.find_by_name_or_uuid(ns_name)

    def _is_network_isolated(self, ns_name):
        return self._get_namespace(ns_name).is_isolated()

    def _get_network(self, ns_name):
        ns = self._get_namespace(ns_name)
        if ns.is_isolated():
            vn_fq_name = ns.get_isolated_network_fq_name()
        else:
            if self._default_vn_obj:
                return self._default_vn_obj
            vn_fq_name = vnc_kube_config.cluster_default_network_fq_name()
        try:
            vn_obj = self._vnc_lib.virtual_network_read(fq_name=vn_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" % (self._name, vn_fq_name))
            return None
        if not ns.is_isolated():
            self._default_vn_obj = vn_obj
        return vn_obj

    def _get_pod_ipam_subnet_uuid(self, vn_obj):
        pod_ipam_subnet_uuid = None
        fq_name = vnc_kube_config.pod_ipam_fq_name()
        vn = VirtualNetworkKM.find_by_name_or_uuid(vn_obj.get_uuid())
        pod_ipam_subnet_uuid = vn.get_ipam_subnet_uuid(fq_name)
        if pod_ipam_subnet_uuid is None:
            self._logger.error("%s - %s Not Found" % (self._name, fq_name))
        return pod_ipam_subnet_uuid

    def _get_cluster_service_fip(self, name, ns_name, lb_obj):
        if not self._service_fip_pool:
            return None

        fip_pool = FloatingIpPool()
        fip_pool.uuid = self._service_fip_pool.uuid
        fip_pool.fq_name = self._service_fip_pool.fq_name
        fip_pool.name = self._service_fip_pool.name

        fip_uuid = str(uuid.uuid4())
        fip_name = VncCommon.make_name(name, fip_uuid)
        display_name = VncCommon.make_display_name(ns_name, name)
        fip_obj = FloatingIp(name="cluster-svc-fip-%s" % (fip_name),
                             parent_obj=fip_pool,
                             floating_ip_traffic_direction='egress',
                             display_name=display_name)
        fip_obj.uuid = fip_uuid

        proj_obj = self._get_project(ns_name)
        fip_obj.set_project(proj_obj)

        vmi_id = lb_obj.virtual_machine_interface_refs[0]['uuid']
        vmi_obj = self._vnc_lib.virtual_machine_interface_read(id=vmi_id)
        if vmi_obj:
            fip_obj.set_virtual_machine_interface(vmi_obj)

        FloatingIpKM.add_annotations(self, fip_obj, name, ns_name)
        try:
            self._vnc_lib.floating_ip_create(fip_obj)
            fip = FloatingIpKM.locate(fip_obj.uuid)
        except Exception as e:
            string_buf = StringIO()
            cgitb_hook(file=string_buf, format="text")
            err_msg = string_buf.getvalue()
            self._logger.error("%s - %s" % (self._name, err_msg))

        return

    def _get_public_fip_pool(self, fip_pool_fq_name):
        if self._fip_pool_obj:
            return self._fip_pool_obj
        try:
            fip_pool_obj = self._vnc_lib. \
                           floating_ip_pool_read(fq_name=fip_pool_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" \
                 %(self._name, fip_pool_fq_name))
            return None
        self._fip_pool_obj = fip_pool_obj
        return fip_pool_obj

    def _get_floating_ip(self, name, proj_obj, external_ip=None, vmi_obj=None):
        if not vnc_kube_config.is_public_fip_pool_configured():
            return None

        try:
            fip_pool_fq_name = get_fip_pool_fq_name_from_dict_string(
                self._args.public_fip_pool)
        except Exception as e:
            string_buf = StringIO()
            cgitb_hook(file=string_buf, format="text")
            err_msg = string_buf.getvalue()
            self._logger.error("%s - %s" % (self._name, err_msg))
            return None

        if vmi_obj:
            fip_refs = vmi_obj.get_floating_ip_back_refs()
            for ref in fip_refs or []:
                fip = FloatingIpKM.get(ref['uuid'])
                if fip and fip.fq_name[:-1] == fip_pool_fq_name:
                    return fip
                else:
                    break
        fip_pool = self._get_public_fip_pool(fip_pool_fq_name)
        if fip_pool is None:
            return None
        fip_uuid = str(uuid.uuid4())
        fip_name = VncCommon.make_name(name, fip_uuid)
        fip_obj = FloatingIp(fip_name, fip_pool)
        fip_obj.uuid = fip_uuid
        fip_obj.set_project(proj_obj)
        if vmi_obj:
            fip_obj.set_virtual_machine_interface(vmi_obj)
        if external_ip:
            fip_obj.floating_ip_address = external_ip
        try:
            self._vnc_lib.floating_ip_create(fip_obj)
            fip = FloatingIpKM.locate(fip_obj.uuid)
        except Exception as e:
            string_buf = StringIO()
            cgitb_hook(file=string_buf, format="text")
            err_msg = string_buf.getvalue()
            self._logger.error("%s - %s" % (self._name, err_msg))
            return None
        return fip

    def _allocate_floating_ip(self, lb_obj, name, proj_obj, external_ip):
        vmi_id = lb_obj.virtual_machine_interface_refs[0]['uuid']
        vmi_obj = self._vnc_lib.virtual_machine_interface_read(id=vmi_id)
        if vmi_obj is None:
            self._logger.error("%s - %s Vmi %s Not Found" \
                 %(self._name, lb_obj.name, vmi_id))
            return None
        fip = self._get_floating_ip(name, proj_obj, external_ip, vmi_obj)
        return fip

    def _deallocate_floating_ip(self, lb, delete_svc_fip=True):
        vmi_id = list(lb.virtual_machine_interfaces)[0]
        vmi = VirtualMachineInterfaceKM.get(vmi_id)
        if vmi is None:
            self._logger.error("%s - %s Vmi %s Not Found" \
                 %(self._name, lb.name, vmi_id))
            return
        fip_list = vmi.floating_ips.copy()
        for fip_id in fip_list or []:
            fip_obj = self._vnc_lib.floating_ip_read(id=fip_id)
            if delete_svc_fip == False and \
               self._service_fip_pool.fq_name == fip_obj.fq_name[:-1]:
                continue
            fip_obj.set_virtual_machine_interface_list([])
            self._vnc_lib.floating_ip_update(fip_obj)
            self._vnc_lib.floating_ip_delete(id=fip_obj.uuid)
            FloatingIpKM.delete(fip_obj.uuid)

    def _update_floating_ip(self, name, ns_name, external_ip, lb_obj):
        proj_obj = self._get_project(ns_name)
        fip = self._allocate_floating_ip(lb_obj, name, proj_obj, external_ip)
        if fip:
            lb_obj.add_annotations(
                KeyValuePair(key='externalIP', value=external_ip))
            self._vnc_lib.loadbalancer_update(lb_obj)
        return fip

    def _update_kube_api_server(self, name, ns_name, lb_obj, fip):
        vip_dict_list = []
        if fip:
            vip_dict = {}
            vip_dict['ip'] = fip.address
            vip_dict_list.append(vip_dict)
        vip_dict = {}
        vip_dict['ip'] = lb_obj._loadbalancer_properties.vip_address
        vip_dict_list.append(vip_dict)
        patch = {'status': {'loadBalancer': {'ingress': vip_dict_list}}}
        self._kube.patch_resource("ingresses",
                                  name,
                                  patch,
                                  ns_name,
                                  beta=True,
                                  sub_resource_name='status')

    def _find_ingress(self, ingress_cache, ns_name, service_name):
        if not ns_name or not service_name:
            return
        key = 'service'
        value = '-'.join([ns_name, service_name])
        labels = {key: value}
        result = set()
        for label in labels.items():
            key = self._label_cache._get_key(label)
            ingress_ids = ingress_cache.get(key, set())
            #no matching label
            if not ingress_ids:
                return ingress_ids
            if not result:
                result = ingress_ids.copy()
            else:
                result.intersection_update(ingress_ids)
        return result

    def _clear_ingress_cache_uuid(self, ingress_cache, ingress_uuid):
        if not ingress_uuid:
            return
        key_list = [k for k, v in ingress_cache.items() if ingress_uuid in v]
        for key in key_list or []:
            label = tuple(key.split(':'))
            self._label_cache._remove_label(key, ingress_cache, label,
                                            ingress_uuid)

    def _clear_ingress_cache(self, ingress_cache, ns_name, service_name,
                             ingress_uuid):
        if not ns_name or not service_name:
            return
        key = 'service'
        value = '-'.join([ns_name, service_name])
        labels = {key: value}
        for label in labels.items() or []:
            key = self._label_cache._get_key(label)
            self._label_cache._remove_label(key, ingress_cache, label,
                                            ingress_uuid)

    def _update_ingress_cache(self, ingress_cache, ns_name, service_name,
                              ingress_uuid):
        if not ns_name or not service_name:
            return
        key = 'service'
        value = '-'.join([ns_name, service_name])
        labels = {key: value}
        for label in labels.items() or []:
            key = self._label_cache._get_key(label)
            self._label_cache._locate_label(key, ingress_cache, label,
                                            ingress_uuid)

    def _vnc_create_member(self, pool, address, port, annotations):
        pool_obj = self.service_lb_pool_mgr.read(pool.uuid)
        member_obj = self.service_lb_member_mgr.create(pool_obj, address, port,
                                                       annotations)
        return member_obj

    def _vnc_update_member(self, member_id, address, port, annotations):
        member_obj = self.service_lb_member_mgr.update(member_id, address,
                                                       port, annotations)
        return member_obj

    def _vnc_create_pool(self, ns_name, ll, port, lb_algorithm, annotations):
        proj_obj = self._get_project(ns_name)
        ll_obj = self.service_ll_mgr.read(ll.uuid)
        pool_obj = self.service_lb_pool_mgr.create(ll_obj, proj_obj, port,
                                                   lb_algorithm, annotations)
        return pool_obj

    def _vnc_create_listeners(self, ns_name, lb, port):
        proj_obj = self._get_project(ns_name)
        lb_obj = self.service_lb_mgr.read(lb.uuid)
        ll_obj = self.service_ll_mgr.create(lb_obj, proj_obj, port)
        return ll_obj

    def _vnc_create_lb(self, uid, name, ns_name, annotations):
        proj_obj = self._get_project(ns_name)
        vn_obj = self._get_network(ns_name)
        if proj_obj is None or vn_obj is None:
            return None

        vip_address = None
        pod_ipam_subnet_uuid = self._get_pod_ipam_subnet_uuid(vn_obj)
        lb_obj = self.service_lb_mgr.create(self._k8s_event_type, ns_name, uid,
                                            name, proj_obj, vn_obj,
                                            vip_address, pod_ipam_subnet_uuid)
        if lb_obj:
            if self._is_network_isolated(ns_name):
                self._get_cluster_service_fip(name, ns_name, lb_obj)
            external_ip = None
            if annotations and 'externalIP' in annotations:
                external_ip = annotations['externalIP']
            fip = self._update_floating_ip(name, ns_name, external_ip, lb_obj)
            self._update_kube_api_server(name, ns_name, lb_obj, fip)
        else:
            self._logger.error("%s - %s LB Not Created" % (self._name, name))

        return lb_obj

    def _vnc_delete_member(self, member_id):
        self.service_lb_member_mgr.delete(member_id)

    def _vnc_delete_pool(self, pool_id):
        self.service_lb_pool_mgr.delete(pool_id)

    def _vnc_delete_listener(self, ll_id):
        self.service_ll_mgr.delete(ll_id)

    def _vnc_delete_lb(self, lb):
        self._deallocate_floating_ip(lb)
        self.service_lb_mgr.delete(lb.uuid)

    def _get_old_backend_list(self, lb):
        backend_list = []
        listener_list = lb.loadbalancer_listeners
        for ll_id in listener_list:
            backend = {}
            backend['listener_id'] = ll_id
            ll = LoadbalancerListenerKM.get(ll_id)
            backend['listener'] = {}
            backend['listener']['protocol'] = ll.params['protocol']
            if backend['listener']['protocol'] == 'TERMINTED_HTTPS':
                if ll.params['default_tls_container']:
                    backend['listener']['default_tls_container'] = \
                        ll.params['default_tls_container']
                if ll.params['sni_containers']:
                    backend['listener']['sni_containers'] = \
                        ll.params['sni_containers']
            pool_id = ll.loadbalancer_pool
            if pool_id:
                pool = LoadbalancerPoolKM.get(pool_id)
                if pool.annotations is None:
                    annotations = {}
                    kvps = []
                    pool_obj = self._vnc_lib.loadbalancer_pool_read(id=pool_id)
                    pool_obj_kvp = pool_obj.annotations.key_value_pair
                    kvps_len = len(pool_obj_kvp)
                    for count in range(0, kvps_len):
                        kvp = {}
                        kvp['key'] = pool_obj_kvp[count].key
                        kvp['value'] = pool_obj_kvp[count].value
                        kvps.append(kvp)
                    annotations['key_value_pair'] = kvps
                else:
                    annotations = pool.annotations
                backend['pool_id'] = pool_id
                backend['annotations'] = {}
                for kvp in annotations['key_value_pair'] or []:
                    key = kvp['key']
                    value = kvp['value']
                    backend['annotations'][key] = value
                backend['pool'] = {}
                backend['pool']['protocol'] = pool.params['protocol']
                backend['member'] = {}
                if len(pool.members) == 0:
                    continue
                member_id = list(pool.members)[0]
                member = LoadbalancerMemberKM.get(member_id)
                if member.annotations is None:
                    annotations = {}
                    kvps = []
                    member_obj = self._vnc_lib. \
                                 loadbalancer_member_read(id=member_id)
                    member_obj_kvp = member_obj.annotations.key_value_pair
                    kvps_len = len(member_obj_kvp)
                    for count in range(0, kvps_len):
                        kvp = {}
                        kvp['key'] = member_obj_kvp[count].key
                        kvp['value'] = member_obj_kvp[count].value
                        kvps.append(kvp)
                    annotations['key_value_pair'] = kvps
                else:
                    annotations = member.annotations
                backend['member_id'] = member_id
                protocol_port = member.params['protocol_port']
                for kvp in annotations['key_value_pair'] or []:
                    if kvp['key'] == 'serviceName':
                        backend['member']['serviceName'] = kvp['value']
                        backend['member']['servicePort'] = protocol_port
                        break
            backend_list.append(backend)
        return backend_list

    def _get_tls_dict(self, spec, ns_name):
        tls_dict = {}
        if 'tls' in spec:
            tls_list = spec['tls']
            for tls in tls_list:
                if not 'secretName' in tls:
                    continue
                if 'hosts' in tls:
                    hosts = tls['hosts']
                else:
                    hosts = ['ALL']
                for host in hosts:
                    tls_dict[host] = ns_name + '__' + tls['secretName']
        return tls_dict

    def _get_new_backend_list(self, spec, ns_name):
        tls_dict = self._get_tls_dict(spec, ns_name)
        backend_list = []
        rules = []
        if 'rules' in spec:
            rules = spec['rules']
            for rule in rules:
                if 'http' not in rule:
                    continue
                paths = rule['http']['paths']
                for path in paths or []:
                    backend = {}
                    backend['annotations'] = {}
                    backend['listener'] = {}
                    backend['pool'] = {}
                    backend['member'] = {}
                    backend['listener']['protocol'] = 'HTTP'
                    backend['pool']['protocol'] = 'HTTP'
                    secretname = ""
                    if 'host' in rule:
                        host = rule['host']
                        backend['annotations']['host'] = host
                        if host in tls_dict.keys():
                            secretname = tls_dict[host]
                    if 'path' in path:
                        backend['annotations']['path'] = path['path']
                    service = path['backend']
                    backend['annotations']['type'] = 'acl'
                    backend['member']['serviceName'] = service['serviceName']
                    backend['member']['servicePort'] = service['servicePort']
                    backend_list.append(backend)
                    if secretname:
                        backend_https = copy.deepcopy(backend)
                        backend_https['listener'][
                            'protocol'] = 'TERMINATED_HTTPS'
                        backend_https['listener']['sni_containers'] = [
                            secretname
                        ]
                        backend_list.append(backend_https)
        if 'backend' in spec:
            service = spec['backend']
            backend = {}
            backend['annotations'] = {}
            backend['listener'] = {}
            backend['pool'] = {}
            backend['member'] = {}
            backend['listener']['protocol'] = 'HTTP'
            backend['pool']['protocol'] = 'HTTP'
            backend['annotations']['type'] = 'default'
            backend['member']['serviceName'] = service['serviceName']
            backend['member']['servicePort'] = service['servicePort']
            backend_list.append(backend)
            if 'ALL' in tls_dict.keys():
                backend_https = copy.deepcopy(backend)
                backend_https['listener']['protocol'] = 'TERMINATED_HTTPS'
                backend_https['listener']['default_tls_container'] = tls_dict[
                    'ALL']
                backend_list.append(backend_https)
        return backend_list

    def _create_member(self, ns_name, backend_member, pool):
        resource_type = "services"
        service_name = backend_member['serviceName']
        service_port = backend_member['servicePort']
        service_info = self._kube.get_resource(resource_type, service_name,
                                               ns_name)
        member = None
        if service_info and 'clusterIP' in service_info['spec']:
            service_ip = service_info['spec']['clusterIP']
            self._logger.debug("%s - clusterIP for service %s - %s" \
                 %(self._name, service_name, service_ip))
            member_match = False
            annotations = {}
            annotations['serviceName'] = service_name
            for member_id in pool.members:
                member = LoadbalancerMemberKM.get(member_id)
                if member and member.params['address'] == service_ip \
                   and member.params['protocol_port'] == service_port:
                    member_match = True
                    break
            if not member_match:
                member_obj = self._vnc_create_member(pool, service_ip,
                                                     service_port, annotations)
                if member_obj:
                    member = LoadbalancerMemberKM.locate(member_obj.uuid)
                else:
                    self._logger.error(
                         "%s - (%s %s) Member Not Created for Pool %s" \
                         %(self._name, service_name,
                         str(service_port), pool.name))
        else:
            self._logger.error("%s - clusterIP for Service %s Not Found" \
                 %(self._name, service_name))
            self._logger.error(
                 "%s - (%s %s) Member Not Created for Pool %s" \
                 %(self._name, service_name,
                 str(service_port), pool.name))
        return member

    def _update_member(self, ns_name, backend_member, pool):
        resource_type = "services"
        member_id = backend_member['member_id']
        new_service_name = backend_member['serviceName']
        new_service_port = backend_member['servicePort']
        member = LoadbalancerMemberKM.get(member_id)
        annotations = member.annotations
        for kvp in annotations['key_value_pair'] or []:
            if kvp['key'] == 'serviceName':
                old_service_name = kvp['value']
                break
        old_service_port = member.params['protocol_port']
        service_ip = None
        if new_service_name != old_service_name:
            service_info = self._kube.get_resource(resource_type,
                                                   new_service_name, ns_name)
            if service_info and 'clusterIP' in service_info['spec']:
                service_ip = service_info['spec']['clusterIP']
            else:
                self._logger.error("%s - clusterIP for Service %s Not Found" \
                     %(self._name, new_service_name))
                self._logger.error(
                     "%s - (%s %s) Member Not Updated for Pool %s" \
                     %(self._name, new_service_name,
                     str(new_service_port), pool.name))
                self._vnc_delete_member(member_id)
                LoadbalancerMemberKM.delete(member_id)
                self._logger.error(
                     "%s - (%s %s) Member Deleted for Pool %s" \
                     %(self._name, old_service_name,
                     str(old_service_port), pool.name))
                return None
        else:
            service_ip = member.params['address']
        annotations = {}
        annotations['serviceName'] = new_service_name
        member_obj = self._vnc_update_member(member_id, service_ip,
                                             new_service_port, annotations)
        member = LoadbalancerMemberKM.update(member)
        return member

    def _create_pool(self, ns_name, ll, port, lb_algorithm, annotations):
        pool_id = ll.loadbalancer_pool
        pool = LoadbalancerPoolKM.get(pool_id)
        if pool is None:
            pool_obj = self._vnc_create_pool(ns_name, ll, port, lb_algorithm,
                                             annotations)
            pool_id = pool_obj.uuid
            pool = LoadbalancerPoolKM.locate(pool_id)
        else:
            self._logger.error("%s - %s Pool Not Created" \
                 %(self._name, ll.name))
        return pool

    def _create_listener(self, ns_name, lb, port):
        ll_obj = self._vnc_create_listeners(ns_name, lb, port)
        if ll_obj:
            ll = LoadbalancerListenerKM.locate(ll_obj.uuid)
        else:
            self._logger.error("%s - %s Listener for Port %s Not Created" \
                 %(self._name, lb.name, str(port)))
        return ll

    def _create_listener_pool_member(self, ns_name, lb, backend):
        pool_port = {}
        listener_port = {}
        listener_port['port'] = '80'
        listener_port['protocol'] = backend['listener']['protocol']
        if listener_port['protocol'] == 'TERMINATED_HTTPS':
            listener_port['port'] = '443'
            if 'default_tls_container' in backend['listener']:
                listener_port['default_tls_container'] = backend['listener'][
                    'default_tls_container']
            if 'sni_containers' in backend['listener']:
                listener_port['sni_containers'] = backend['listener'][
                    'sni_containers']
        ll = self._create_listener(ns_name, lb, listener_port)
        annotations = {}
        for key in backend['annotations']:
            annotations[key] = backend['annotations'][key]
        lb_algorithm = "ROUND_ROBIN"
        pool_port['port'] = '80'
        pool_port['protocol'] = backend['pool']['protocol']
        pool = self._create_pool(ns_name, ll, pool_port, lb_algorithm,
                                 annotations)
        backend_member = backend['member']
        member = self._create_member(ns_name, backend_member, pool)
        if member is None:
            self._logger.error("%s - Deleting Listener %s and Pool %s" \
                %(self._name, ll.name, pool.name))
            self._vnc_delete_pool(pool.uuid)
            LoadbalancerPoolKM.delete(pool.uuid)
            self._vnc_delete_listener(ll.uuid)
            LoadbalancerListenerKM.delete(ll.uuid)

    def update_ingress_backend(self, ns_name, service_name, oper):
        ingress_ids = self._find_ingress(self._ingress_label_cache, ns_name,
                                         service_name)
        for ingress_id in ingress_ids or []:
            ingress = IngressKM.get(ingress_id)
            lb = LoadbalancerKM.get(ingress_id)
            if not ingress or not lb:
                continue
            if oper == 'ADD':
                new_backend_list = self._get_new_backend_list(
                    ingress.spec, ns_name)
                for new_backend in new_backend_list[:] or []:
                    if new_backend['member']['serviceName'] == service_name:
                        self._create_listener_pool_member(
                            ns_name, lb, new_backend)
            else:
                old_backend_list = self._get_old_backend_list(lb)
                for old_backend in old_backend_list[:] or []:
                    if old_backend['member']['serviceName'] == service_name:
                        self._delete_listener(old_backend['listener_id'])

    def _create_lb(self, uid, name, ns_name, event):
        annotations = event['object']['metadata'].get('annotations')
        lb = LoadbalancerKM.get(uid)
        if not lb:
            lb_obj = self._vnc_create_lb(uid, name, ns_name, annotations)
            if lb_obj is None:
                return
            lb = LoadbalancerKM.locate(uid)
        else:
            external_ip = None
            if annotations and 'externalIP' in annotations:
                external_ip = annotations['externalIP']
            if external_ip != lb.external_ip:
                self._deallocate_floating_ip(lb, delete_svc_fip=False)
                lb_obj = self._vnc_lib.loadbalancer_read(id=lb.uuid)
                fip = self._update_floating_ip(name, ns_name, external_ip,
                                               lb_obj)
                if fip:
                    lb.external_ip = external_ip
                self._update_kube_api_server(name, ns_name, lb_obj, fip)

        self._clear_ingress_cache_uuid(self._ingress_label_cache, uid)

        spec = event['object']['spec']
        new_backend_list = self._get_new_backend_list(spec, ns_name)
        old_backend_list = self._get_old_backend_list(lb)

        # find the unchanged backends
        for new_backend in new_backend_list[:] or []:
            self._update_ingress_cache(self._ingress_label_cache, ns_name,
                                       new_backend['member']['serviceName'],
                                       uid)
            for old_backend in old_backend_list[:] or []:
                if new_backend['annotations'] == old_backend['annotations'] \
                    and new_backend['listener'] == old_backend['listener'] \
                    and new_backend['pool'] == old_backend['pool'] \
                    and new_backend['member'] == old_backend['member']:
                    old_backend_list.remove(old_backend)
                    new_backend_list.remove(new_backend)
                    break
        if len(old_backend_list) == 0 and len(new_backend_list) == 0:
            return lb

        # find the updated backends and update
        backend_update_list = []
        for new_backend in new_backend_list[:] or []:
            for old_backend in old_backend_list[:] or []:
                if new_backend['annotations'] == old_backend['annotations'] \
                    and new_backend['listener'] == old_backend['listener'] \
                    and new_backend['pool'] == old_backend['pool']:
                    backend = old_backend
                    backend['member']['member_id'] = \
                                     old_backend['member_id']
                    backend['member']['serviceName'] = \
                                     new_backend['member']['serviceName']
                    backend['member']['servicePort'] = \
                                     new_backend['member']['servicePort']
                    backend_update_list.append(backend)
                    old_backend_list.remove(old_backend)
                    new_backend_list.remove(new_backend)
        for backend in backend_update_list or []:
            ll = LoadbalancerListenerKM.get(backend['listener_id'])
            pool = LoadbalancerPoolKM.get(backend['pool_id'])
            backend_member = backend['member']
            member = self._update_member(ns_name, backend_member, pool)
            if member is None:
                self._logger.error("%s - Deleting Listener %s and Pool %s" \
                     %(self._name, ll.name, pool.name))
                self._vnc_delete_pool(pool.uuid)
                LoadbalancerPoolKM.delete(pool.uuid)
                self._vnc_delete_listener(ll.uuid)
                LoadbalancerListenerKM.delete(ll.uuid)
        if len(old_backend_list) == 0 and len(new_backend_list) == 0:
            return lb

        # delete the old backends
        for backend in old_backend_list or []:
            self._delete_listener(backend['listener_id'])

        # create the new backends
        for backend in new_backend_list:
            self._create_listener_pool_member(ns_name, lb, backend)

        return lb

    def _delete_all_listeners(self, lb):
        listener_list = lb.loadbalancer_listeners.copy()
        for ll_id in listener_list:
            ll = LoadbalancerListenerKM.get(ll_id)
            pool_id = ll.loadbalancer_pool
            if pool_id:
                pool = LoadbalancerPoolKM.get(pool_id)
                member_list = pool.members.copy()
                for member_id in member_list:
                    self._vnc_delete_member(member_id)
                    LoadbalancerMemberKM.delete(member_id)
                self._vnc_delete_pool(pool_id)
                LoadbalancerPoolKM.delete(pool_id)
            self._vnc_delete_listener(ll_id)
            LoadbalancerListenerKM.delete(ll_id)

    def _delete_listener(self, ll_id):
        ll = LoadbalancerListenerKM.get(ll_id)
        pool_id = ll.loadbalancer_pool
        if pool_id:
            pool = LoadbalancerPoolKM.get(pool_id)
            member_list = pool.members.copy()
            for member_id in member_list:
                self._vnc_delete_member(member_id)
                LoadbalancerMemberKM.delete(member_id)
            self._vnc_delete_pool(pool_id)
            LoadbalancerPoolKM.delete(pool_id)
        self._vnc_delete_listener(ll_id)
        LoadbalancerListenerKM.delete(ll_id)

    def _delete_lb(self, uid):
        lb = LoadbalancerKM.get(uid)
        if not lb:
            return
        self._delete_all_listeners(lb)
        self._vnc_delete_lb(lb)
        LoadbalancerKM.delete(uid)

    def _update_ingress(self, name, uid, event):
        ns_name = event['object']['metadata'].get('namespace')
        self._create_lb(uid, name, ns_name, event)

    def _delete_ingress(self, uid):
        self._delete_lb(uid)
        self._clear_ingress_cache_uuid(self._ingress_label_cache, uid)

    def _create_ingress_event(self, event_type, ingress_id, lb):
        event = {}
        object = {}
        object['kind'] = 'Ingress'
        object['spec'] = {}
        object['metadata'] = {}
        object['metadata']['uid'] = ingress_id
        if event_type == 'delete':
            event['type'] = 'DELETED'
            event['object'] = object
            self._queue.put(event)
        return

    def _sync_ingress_lb(self):
        lb_uuid_set = set(LoadbalancerKM.keys())
        ingress_uuid_set = set(IngressKM.keys())
        deleted_ingress_set = lb_uuid_set - ingress_uuid_set
        for uuid in deleted_ingress_set:
            lb = LoadbalancerKM.get(uuid)
            if not lb:
                continue
            if not lb.annotations:
                continue
            owner = None
            kind = None
            for kvp in lb.annotations['key_value_pair'] or []:
                if kvp['key'] == 'owner':
                    owner = kvp['value']
                elif kvp['key'] == 'kind':
                    kind = kvp['value']
                if owner == 'k8s' and kind == self._k8s_event_type:
                    self._create_ingress_event('delete', uuid, lb)
                    break
        return

    def ingress_timer(self):
        self._sync_ingress_lb()

    def process(self, event):
        event_type = event['type']
        kind = event['object'].get('kind')
        ns_name = event['object']['metadata'].get('namespace')
        name = event['object']['metadata'].get('name')
        uid = event['object']['metadata'].get('uid')

        print("%s - Got %s %s %s:%s:%s" %
              (self._name, event_type, kind, ns_name, name, uid))
        self._logger.debug("%s - Got %s %s %s:%s:%s" %
                           (self._name, event_type, kind, ns_name, name, uid))

        if event['type'] == 'ADDED' or event['type'] == 'MODIFIED':
            self._update_ingress(name, uid, event)
        elif event['type'] == 'DELETED':
            self._delete_ingress(uid)
class VncIngress(VncCommon):
    def __init__(self):
        self._k8s_event_type = 'Ingress'
        super(VncIngress,self).__init__(self._k8s_event_type)
        self._name = type(self).__name__
        self._args = vnc_kube_config.args()
        self._queue = vnc_kube_config.queue()
        self._vnc_lib = vnc_kube_config.vnc_lib()
        self._logger = vnc_kube_config.logger()
        self._kube = vnc_kube_config.kube()
        self._label_cache = vnc_kube_config.label_cache()
        self._service_fip_pool = vnc_kube_config.service_fip_pool()
        self._ingress_label_cache = {}
        self._default_vn_obj = None
        self._fip_pool_obj = None
        self.service_lb_mgr = ServiceLbManager()
        self.service_ll_mgr = ServiceLbListenerManager()
        self.service_lb_pool_mgr = ServiceLbPoolManager()
        self.service_lb_member_mgr = ServiceLbMemberManager()

    def _get_project(self, ns_name):
        proj_fq_name = vnc_kube_config.cluster_project_fq_name(ns_name)
        try:
            proj_obj = self._vnc_lib.project_read(fq_name=proj_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" %(self._name, proj_fq_name))
            return None
        return proj_obj

    def _get_namespace(self, ns_name):
        return NamespaceKM.find_by_name_or_uuid(ns_name)

    def _is_network_isolated(self, ns_name):
        return self._get_namespace(ns_name).is_isolated()

    def _get_network(self, ns_name):
        ns = self._get_namespace(ns_name)
        if ns.is_isolated():
            vn_fq_name = ns.get_isolated_network_fq_name()
        else:
            if self._default_vn_obj:
                return self._default_vn_obj
            vn_fq_name = vnc_kube_config.cluster_default_network_fq_name()
        try:
            vn_obj = self._vnc_lib.virtual_network_read(fq_name=vn_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" %(self._name, vn_fq_name))
            return None
        if not ns.is_isolated():
            self._default_vn_obj = vn_obj
        return vn_obj

    def _get_pod_ipam_subnet_uuid(self, vn_obj):
        pod_ipam_subnet_uuid = None
        fq_name = vnc_kube_config.pod_ipam_fq_name()
        vn = VirtualNetworkKM.find_by_name_or_uuid(vn_obj.get_uuid())
        pod_ipam_subnet_uuid = vn.get_ipam_subnet_uuid(fq_name)
        if pod_ipam_subnet_uuid is None:
            self._logger.error("%s - %s Not Found" %(self._name, fq_name))
        return pod_ipam_subnet_uuid

    def _get_cluster_service_fip(self, name, ns_name, lb_obj):
        if not self._service_fip_pool:
            return None

        fip_pool = FloatingIpPool()
        fip_pool.uuid = self._service_fip_pool.uuid
        fip_pool.fq_name = self._service_fip_pool.fq_name
        fip_pool.name = self._service_fip_pool.name

        fip_uuid = str(uuid.uuid4())
        fip_name = VncCommon.make_name(name, fip_uuid)
        display_name=VncCommon.make_display_name(ns_name, name)
        fip_obj = FloatingIp(name="cluster-svc-fip-%s"% (fip_name),
                    parent_obj=fip_pool,
                    floating_ip_traffic_direction='egress',
                    display_name=display_name)
        fip_obj.uuid = fip_uuid

        proj_obj = self._get_project(ns_name)
        fip_obj.set_project(proj_obj)

        vmi_id = lb_obj.virtual_machine_interface_refs[0]['uuid']
        vmi_obj = self._vnc_lib.virtual_machine_interface_read(id=vmi_id)
        if vmi_obj:
            fip_obj.set_virtual_machine_interface(vmi_obj)

        FloatingIpKM.add_annotations(self, fip_obj, name, ns_name)
        try:
            self._vnc_lib.floating_ip_create(fip_obj)
            fip = FloatingIpKM.locate(fip_obj.uuid)
        except Exception as e:
            string_buf = StringIO()
            cgitb_hook(file=string_buf, format="text")
            err_msg = string_buf.getvalue()
            self._logger.error("%s - %s" %(self._name, err_msg))

        return

    def _get_public_fip_pool(self, fip_pool_fq_name):
        if self._fip_pool_obj:
            return self._fip_pool_obj
        try:
            fip_pool_obj = self._vnc_lib. \
                           floating_ip_pool_read(fq_name=fip_pool_fq_name)
        except NoIdError:
            self._logger.error("%s - %s Not Found" \
                 %(self._name, fip_pool_fq_name))
            return None
        self._fip_pool_obj = fip_pool_obj
        return fip_pool_obj

    def _get_floating_ip(self, name,
            proj_obj, external_ip=None, vmi_obj=None):
        if not vnc_kube_config.is_public_fip_pool_configured():
            return None

        try:
            fip_pool_fq_name = get_fip_pool_fq_name_from_dict_string(
                self._args.public_fip_pool)
        except Exception as e:
            string_buf = StringIO()
            cgitb_hook(file=string_buf, format="text")
            err_msg = string_buf.getvalue()
            self._logger.error("%s - %s" %(self._name, err_msg))
            return None

        if vmi_obj:
            fip_refs = vmi_obj.get_floating_ip_back_refs()
            for ref in fip_refs or []:
                fip = FloatingIpKM.get(ref['uuid'])
                if fip and fip.fq_name[:-1] == fip_pool_fq_name:
                    return fip
                else:
                    break
        fip_pool = self._get_public_fip_pool(fip_pool_fq_name)
        if fip_pool is None:
            return None
        fip_uuid = str(uuid.uuid4())
        fip_name = VncCommon.make_name(name, fip_uuid)
        fip_obj = FloatingIp(fip_name, fip_pool)
        fip_obj.uuid = fip_uuid
        fip_obj.set_project(proj_obj)
        if vmi_obj:
            fip_obj.set_virtual_machine_interface(vmi_obj)
        if external_ip:
            fip_obj.floating_ip_address = external_ip
        try:
            self._vnc_lib.floating_ip_create(fip_obj)
            fip = FloatingIpKM.locate(fip_obj.uuid)
        except Exception as e:
            string_buf = StringIO()
            cgitb_hook(file=string_buf, format="text")
            err_msg = string_buf.getvalue()
            self._logger.error("%s - %s" %(self._name, err_msg))
            return None
        return fip

    def _allocate_floating_ip(self, lb_obj, name, proj_obj, external_ip):
        vmi_id = lb_obj.virtual_machine_interface_refs[0]['uuid']
        vmi_obj = self._vnc_lib.virtual_machine_interface_read(id=vmi_id)
        if vmi_obj is None:
            self._logger.error("%s - %s Vmi %s Not Found" \
                 %(self._name, lb_obj.name, vmi_id))
            return None
        fip = self._get_floating_ip(name, proj_obj, external_ip, vmi_obj)
        return fip

    def _deallocate_floating_ip(self, lb, delete_svc_fip=True):
        vmi_id = list(lb.virtual_machine_interfaces)[0]
        vmi = VirtualMachineInterfaceKM.get(vmi_id)
        if vmi is None:
            self._logger.error("%s - %s Vmi %s Not Found" \
                 %(self._name, lb.name, vmi_id))
            return
        fip_list = vmi.floating_ips.copy()
        for fip_id in fip_list or []:
            fip_obj = self._vnc_lib.floating_ip_read(id=fip_id)
            if delete_svc_fip == False and \
               self._service_fip_pool.fq_name == fip_obj.fq_name[:-1]:
                continue
            fip_obj.set_virtual_machine_interface_list([])
            self._vnc_lib.floating_ip_update(fip_obj)
            self._vnc_lib.floating_ip_delete(id=fip_obj.uuid)
            FloatingIpKM.delete(fip_obj.uuid)

    def _update_floating_ip(self, name, ns_name, external_ip, lb_obj):
        proj_obj = self._get_project(ns_name)
        fip = self._allocate_floating_ip(lb_obj,
                        name, proj_obj, external_ip)
        if fip:
            lb_obj.add_annotations(
                KeyValuePair(key='externalIP', value=external_ip))
            self._vnc_lib.loadbalancer_update(lb_obj)
        return fip

    def _update_kube_api_server(self, name, ns_name, lb_obj, fip):
        vip_dict_list = []
        if fip:
            vip_dict = {}
            vip_dict['ip'] = fip.address
            vip_dict_list.append(vip_dict)
        vip_dict = {}
        vip_dict['ip'] = lb_obj._loadbalancer_properties.vip_address
        vip_dict_list.append(vip_dict)
        patch = {'status': {'loadBalancer': {'ingress': vip_dict_list}}}
        self._kube.patch_resource("ingresses", name, patch,
                ns_name, beta=True, sub_resource_name='status')

    def _find_ingress(self, ingress_cache, ns_name, service_name):
        if not ns_name or not service_name:
            return
        key = 'service'
        value = '-'.join([ns_name, service_name])
        labels = {key:value}
        result = set()
        for label in labels.items():
            key = self._label_cache._get_key(label)
            ingress_ids = ingress_cache.get(key, set())
            #no matching label
            if not ingress_ids:
                return ingress_ids
            if not result:
                result = ingress_ids.copy()
            else:
                result.intersection_update(ingress_ids)
        return result

    def _clear_ingress_cache_uuid(self, ingress_cache, ingress_uuid):
        if not ingress_uuid:
            return
        key_list = [k for k,v in ingress_cache.items() if ingress_uuid in v]
        for key in key_list or []:
            label = tuple(key.split(':'))
            self._label_cache._remove_label(key, ingress_cache, label, ingress_uuid)

    def _clear_ingress_cache(self, ingress_cache,
            ns_name, service_name, ingress_uuid):
        if not ns_name or not service_name:
            return
        key = 'service'
        value = '-'.join([ns_name, service_name])
        labels = {key:value}
        for label in labels.items() or []:
            key = self._label_cache._get_key(label)
            self._label_cache._remove_label(key,
                ingress_cache, label, ingress_uuid)

    def _update_ingress_cache(self, ingress_cache,
            ns_name, service_name, ingress_uuid):
        if not ns_name or not service_name:
            return
        key = 'service'
        value = '-'.join([ns_name, service_name])
        labels = {key:value}
        for label in labels.items() or []:
            key = self._label_cache._get_key(label)
            self._label_cache._locate_label(key,
                ingress_cache, label, ingress_uuid)

    def _vnc_create_member(self, pool, address, port, annotations):
        pool_obj = self.service_lb_pool_mgr.read(pool.uuid)
        member_obj = self.service_lb_member_mgr.create(pool_obj,
                          address, port, annotations)
        return member_obj

    def _vnc_update_member(self, member_id, address, port, annotations):
        member_obj = self.service_lb_member_mgr.update(member_id,
                          address, port, annotations)
        return member_obj

    def _vnc_create_pool(self, ns_name, ll, port, lb_algorithm, annotations):
        proj_obj = self._get_project(ns_name)
        ll_obj = self.service_ll_mgr.read(ll.uuid)
        pool_obj = self.service_lb_pool_mgr.create(ll_obj, proj_obj,
                                            port, lb_algorithm, annotations)
        return pool_obj

    def _vnc_create_listeners(self, ns_name, lb, port):
        proj_obj = self._get_project(ns_name)
        lb_obj = self.service_lb_mgr.read(lb.uuid)
        ll_obj = self.service_ll_mgr.create(lb_obj, proj_obj, port)
        return ll_obj

    def _vnc_create_lb(self, uid, name, ns_name, annotations):
        proj_obj = self._get_project(ns_name)
        vn_obj = self._get_network(ns_name)
        if proj_obj is None or vn_obj is None:
            return None

        vip_address = None
        pod_ipam_subnet_uuid = self._get_pod_ipam_subnet_uuid(vn_obj)
        lb_obj = self.service_lb_mgr.create(self._k8s_event_type, ns_name, uid,
                    name, proj_obj, vn_obj, vip_address, pod_ipam_subnet_uuid)
        if lb_obj:
            if self._is_network_isolated(ns_name):
                self._get_cluster_service_fip(name, ns_name, lb_obj)
            external_ip = None
            if annotations and 'externalIP' in annotations:
                external_ip = annotations['externalIP']
            fip = self._update_floating_ip(name,
                            ns_name, external_ip, lb_obj)
            self._update_kube_api_server(name, ns_name, lb_obj, fip)
        else:
            self._logger.error("%s - %s LB Not Created" %(self._name, name))

        return lb_obj

    def _vnc_delete_member(self, member_id):
        self.service_lb_member_mgr.delete(member_id)

    def _vnc_delete_pool(self, pool_id):
        self.service_lb_pool_mgr.delete(pool_id)

    def _vnc_delete_listener(self, ll_id):
        self.service_ll_mgr.delete(ll_id)

    def _vnc_delete_lb(self, lb):
        self._deallocate_floating_ip(lb)
        self.service_lb_mgr.delete(lb.uuid)

    def _get_old_backend_list(self, lb):
        backend_list = []
        listener_list = lb.loadbalancer_listeners
        for ll_id in listener_list:
            backend = {}
            backend['listener_id'] = ll_id
            ll = LoadbalancerListenerKM.get(ll_id)
            backend['listener'] = {}
            backend['listener']['protocol'] = ll.params['protocol']
            if backend['listener']['protocol'] == 'TERMINTED_HTTPS':
                if ll.params['default_tls_container']:
                    backend['listener']['default_tls_container'] = \
                        ll.params['default_tls_container']
                if ll.params['sni_containers']:
                    backend['listener']['sni_containers'] = \
                        ll.params['sni_containers']
            pool_id = ll.loadbalancer_pool
            if pool_id:
                pool = LoadbalancerPoolKM.get(pool_id)
                if pool.annotations is None:
                    annotations = {}
                    kvps = []
                    pool_obj = self._vnc_lib.loadbalancer_pool_read(id=pool_id)
                    pool_obj_kvp = pool_obj.annotations.key_value_pair
                    kvps_len = len(pool_obj_kvp)
                    for count in range(0, kvps_len):
                         kvp = {}
                         kvp['key'] = pool_obj_kvp[count].key
                         kvp['value'] = pool_obj_kvp[count].value
                         kvps.append(kvp)
                    annotations['key_value_pair'] = kvps
                else:
                    annotations = pool.annotations
                backend['pool_id'] = pool_id
                backend['annotations'] = {}
                for kvp in annotations['key_value_pair'] or []:
                    key = kvp['key']
                    value = kvp['value']
                    backend['annotations'][key] = value
                backend['pool'] = {}
                backend['pool']['protocol'] = pool.params['protocol']
                backend['member'] = {}
                if len(pool.members) == 0:
                    continue
                member_id = list(pool.members)[0]
                member = LoadbalancerMemberKM.get(member_id)
                if member.annotations is None:
                    annotations = {}
                    kvps = []
                    member_obj = self._vnc_lib. \
                                 loadbalancer_member_read(id=member_id)
                    member_obj_kvp = member_obj.annotations.key_value_pair
                    kvps_len = len(member_obj_kvp)
                    for count in range(0, kvps_len):
                         kvp = {}
                         kvp['key'] = member_obj_kvp[count].key
                         kvp['value'] = member_obj_kvp[count].value
                         kvps.append(kvp)
                    annotations['key_value_pair'] = kvps
                else:
                    annotations = member.annotations
                backend['member_id'] = member_id
                protocol_port = member.params['protocol_port']
                for kvp in annotations['key_value_pair'] or []:
                    if kvp['key'] == 'serviceName':
                        backend['member']['serviceName'] = kvp['value']
                        backend['member']['servicePort'] = protocol_port
                        break
            backend_list.append(backend)
        return backend_list

    def _get_tls_dict(self, spec, ns_name):
        tls_dict = {}
        if 'tls' in spec:
            tls_list = spec['tls']
            for tls in tls_list:
                if not 'secretName' in tls:
                    continue
                if 'hosts' in tls:
                    hosts = tls['hosts']
                else:
                    hosts = ['ALL']
                for host in hosts:
                    tls_dict[host] = ns_name + '__' + tls['secretName']
        return tls_dict

    def _get_new_backend_list(self, spec, ns_name):
        tls_dict = self._get_tls_dict(spec, ns_name)
        backend_list = []
        rules = []
        if 'rules' in spec:
            rules = spec['rules']
            for rule in rules:
                if 'http' not in rule:
                    continue
                paths = rule['http']['paths']
                for path in paths or []:
                    backend = {}
                    backend['annotations'] = {}
                    backend['listener'] = {}
                    backend['pool'] = {}
                    backend['member'] = {}
                    backend['listener']['protocol'] = 'HTTP'
                    backend['pool']['protocol'] = 'HTTP'
                    secretname = ""
                    virtual_host = False
                    if 'host' in rule:
                        host = rule['host']
                        backend['annotations']['host'] = host
                        if host in tls_dict.keys():
                            secretname = tls_dict[host]
                            virtual_host = True
                    if 'path' in path:
                        backend['annotations']['path'] = path['path']
                        if virtual_host == False and 'ALL' in tls_dict.keys():
                            secretname = 'ALL'
                    service = path['backend']
                    backend['annotations']['type'] = 'acl'
                    backend['member']['serviceName'] = service['serviceName']
                    backend['member']['servicePort'] = service['servicePort']
                    backend_list.append(backend)
                    if secretname:
                        backend_https = copy.deepcopy(backend)
                        backend_https['listener']['protocol'] = 'TERMINATED_HTTPS'
                        if virtual_host:
                            backend_https['listener']['sni_containers'] = [secretname]
                        else:
                            backend_https['listener']['default_tls_container'] = tls_dict['ALL']
                        backend_list.append(backend_https)
        if 'backend' in spec:
            service = spec['backend']
            backend = {}
            backend['annotations'] = {}
            backend['listener'] = {}
            backend['pool'] = {}
            backend['member'] = {}
            backend['listener']['protocol'] = 'HTTP'
            backend['pool']['protocol'] = 'HTTP'
            backend['annotations']['type'] = 'default'
            backend['member']['serviceName'] = service['serviceName']
            backend['member']['servicePort'] = service['servicePort']
            backend_list.append(backend)
            if 'ALL' in tls_dict.keys():
                backend_https = copy.deepcopy(backend)
                backend_https['listener']['protocol'] = 'TERMINATED_HTTPS'
                backend_https['listener']['default_tls_container'] = tls_dict['ALL']
                backend_list.append(backend_https)
        return backend_list

    def _create_member(self, ns_name, backend_member, pool):
        resource_type = "services"
        service_name = backend_member['serviceName']
        service_port = backend_member['servicePort']
        service_info = self._kube.get_resource(resource_type,
                       service_name, ns_name)
        member = None
        if service_info and 'clusterIP' in service_info['spec']:
            service_ip = service_info['spec']['clusterIP']
            self._logger.debug("%s - clusterIP for service %s - %s" \
                 %(self._name, service_name, service_ip))
            member_match = False
            annotations = {}
            annotations['serviceName'] = service_name
            for member_id in pool.members:
                member = LoadbalancerMemberKM.get(member_id)
                if member and member.params['address'] == service_ip \
                   and member.params['protocol_port'] == service_port:
                    member_match = True
                    break
            if not member_match:
                member_obj = self._vnc_create_member(pool,
                                  service_ip, service_port, annotations)
                if member_obj:
                    member = LoadbalancerMemberKM.locate(member_obj.uuid)
                else:
                    self._logger.error(
                         "%s - (%s %s) Member Not Created for Pool %s" \
                         %(self._name, service_name,
                         str(service_port), pool.name))
        else:
            self._logger.error("%s - clusterIP for Service %s Not Found" \
                 %(self._name, service_name))
            self._logger.error(
                 "%s - (%s %s) Member Not Created for Pool %s" \
                 %(self._name, service_name,
                 str(service_port), pool.name))
        return member

    def _update_member(self, ns_name, backend_member, pool):
        resource_type = "services"
        member_id = backend_member['member_id']
        new_service_name = backend_member['serviceName']
        new_service_port = backend_member['servicePort']
        member = LoadbalancerMemberKM.get(member_id)
        annotations = member.annotations
        for kvp in annotations['key_value_pair'] or []:
            if kvp['key'] == 'serviceName':
                old_service_name = kvp['value']
                break
        old_service_port = member.params['protocol_port']
        service_ip = None
        if new_service_name != old_service_name:
            service_info = self._kube.get_resource(resource_type,
                           new_service_name, ns_name)
            if service_info and 'clusterIP' in service_info['spec']:
                service_ip = service_info['spec']['clusterIP']
            else:
                self._logger.error("%s - clusterIP for Service %s Not Found" \
                     %(self._name, new_service_name))
                self._logger.error(
                     "%s - (%s %s) Member Not Updated for Pool %s" \
                     %(self._name, new_service_name,
                     str(new_service_port), pool.name))
                self._vnc_delete_member(member_id)
                LoadbalancerMemberKM.delete(member_id)
                self._logger.error(
                     "%s - (%s %s) Member Deleted for Pool %s" \
                     %(self._name, old_service_name,
                     str(old_service_port), pool.name))
                return None
        else:
            service_ip = member.params['address']
        annotations = {}
        annotations['serviceName'] = new_service_name
        member_obj = self._vnc_update_member(member_id,
                     service_ip, new_service_port, annotations)
        member = LoadbalancerMemberKM.update(member)
        return member

    def _create_pool(self, ns_name, ll, port, lb_algorithm, annotations):
        pool_id = ll.loadbalancer_pool
        pool = LoadbalancerPoolKM.get(pool_id)
        if pool is None:
            pool_obj = self._vnc_create_pool(ns_name, ll,
                            port, lb_algorithm, annotations)
            pool_id = pool_obj.uuid
            pool = LoadbalancerPoolKM.locate(pool_id)
        else:
            self._logger.error("%s - %s Pool Not Created" \
                 %(self._name, ll.name))
        return pool

    def _create_listener(self, ns_name, lb, port):
        ll_obj = self._vnc_create_listeners(ns_name, lb, port)
        if ll_obj:
            ll = LoadbalancerListenerKM.locate(ll_obj.uuid)
        else:
            self._logger.error("%s - %s Listener for Port %s Not Created" \
                 %(self._name, lb.name, str(port)))
        return ll

    def _create_listener_pool_member(self, ns_name, lb, backend):
        pool_port = {}
        listener_port = {}
        listener_port['port'] = '80'
        listener_port['protocol'] = backend['listener']['protocol']
        if listener_port['protocol'] == 'TERMINATED_HTTPS':
            listener_port['port'] = '443'
            if 'default_tls_container' in backend['listener']:
                listener_port['default_tls_container'] = backend['listener']['default_tls_container']
            if 'sni_containers' in backend['listener']:
                listener_port['sni_containers'] = backend['listener']['sni_containers']
        ll = self._create_listener(ns_name, lb, listener_port)
        annotations = {}
        for key in backend['annotations']:
            annotations[key] = backend['annotations'][key]
        lb_algorithm = "ROUND_ROBIN"
        pool_port['port'] = '80'
        pool_port['protocol'] = backend['pool']['protocol']
        pool = self._create_pool(ns_name, ll, pool_port, lb_algorithm, annotations)
        backend_member = backend['member']
        member = self._create_member(ns_name, backend_member, pool)
        if member is None:
            self._logger.error("%s - Deleting Listener %s and Pool %s" \
                %(self._name, ll.name, pool.name))
            self._vnc_delete_pool(pool.uuid)
            LoadbalancerPoolKM.delete(pool.uuid)
            self._vnc_delete_listener(ll.uuid)
            LoadbalancerListenerKM.delete(ll.uuid)

    def update_ingress_backend(self, ns_name, service_name, oper):
        ingress_ids = self._find_ingress(
            self._ingress_label_cache, ns_name, service_name)
        for ingress_id in ingress_ids or []:
            ingress = IngressKM.get(ingress_id)
            lb = LoadbalancerKM.get(ingress_id)
            if not ingress or not lb:
                continue
            if oper == 'ADD':
                new_backend_list = self._get_new_backend_list(ingress.spec, ns_name)
                for new_backend in new_backend_list[:] or []:
                    if new_backend['member']['serviceName'] == service_name:
                        self._create_listener_pool_member(
                            ns_name, lb, new_backend)
            else:
                old_backend_list = self._get_old_backend_list(lb)
                for old_backend in old_backend_list[:] or []:
                    if old_backend['member']['serviceName'] == service_name:
                        self._delete_listener(old_backend['listener_id'])

    def _create_lb(self, uid, name, ns_name, event):
        annotations = event['object']['metadata'].get('annotations')
        lb = LoadbalancerKM.get(uid)
        if not lb:
            lb_obj = self._vnc_create_lb(uid, name, ns_name, annotations)
            if lb_obj is None:
                return
            lb = LoadbalancerKM.locate(uid)
        else:
            external_ip = None
            if annotations and 'externalIP' in annotations:
                external_ip = annotations['externalIP']
            if external_ip != lb.external_ip:
                self._deallocate_floating_ip(lb, delete_svc_fip=False)
                lb_obj = self._vnc_lib.loadbalancer_read(id=lb.uuid)
                fip = self._update_floating_ip(name, ns_name,
                            external_ip, lb_obj)
                if fip:
                    lb.external_ip = external_ip
                self._update_kube_api_server(name, ns_name, lb_obj, fip)

        self._clear_ingress_cache_uuid(self._ingress_label_cache, uid)

        spec = event['object']['spec']
        new_backend_list = self._get_new_backend_list(spec, ns_name)
        old_backend_list = self._get_old_backend_list(lb)

        # find the unchanged backends
        for new_backend in new_backend_list[:] or []:
            self._update_ingress_cache(self._ingress_label_cache,
                ns_name, new_backend['member']['serviceName'], uid)
            for old_backend in old_backend_list[:] or []:
                if new_backend['annotations'] == old_backend['annotations'] \
                    and new_backend['listener'] == old_backend['listener'] \
                    and new_backend['pool'] == old_backend['pool'] \
                    and new_backend['member'] == old_backend['member']:
                    old_backend_list.remove(old_backend)
                    new_backend_list.remove(new_backend)
                    break
        if len(old_backend_list) == 0 and len(new_backend_list) == 0:
            return lb

        # find the updated backends and update
        backend_update_list = []
        for new_backend in new_backend_list[:] or []:
            for old_backend in old_backend_list[:] or []:
                if new_backend['annotations'] == old_backend['annotations'] \
                    and new_backend['listener'] == old_backend['listener'] \
                    and new_backend['pool'] == old_backend['pool']:
                    backend = old_backend
                    backend['member']['member_id'] = \
                                     old_backend['member_id']
                    backend['member']['serviceName'] = \
                                     new_backend['member']['serviceName']
                    backend['member']['servicePort'] = \
                                     new_backend['member']['servicePort']
                    backend_update_list.append(backend)
                    old_backend_list.remove(old_backend)
                    new_backend_list.remove(new_backend)
        for backend in backend_update_list or []:
            ll = LoadbalancerListenerKM.get(backend['listener_id'])
            pool = LoadbalancerPoolKM.get(backend['pool_id'])
            backend_member = backend['member']
            member = self._update_member(ns_name, backend_member, pool)
            if member is None:
                self._logger.error("%s - Deleting Listener %s and Pool %s" \
                     %(self._name, ll.name, pool.name))
                self._vnc_delete_pool(pool.uuid)
                LoadbalancerPoolKM.delete(pool.uuid)
                self._vnc_delete_listener(ll.uuid)
                LoadbalancerListenerKM.delete(ll.uuid)
        if len(old_backend_list) == 0 and len(new_backend_list) == 0:
            return lb

        # delete the old backends
        for backend in old_backend_list or []:
            self._delete_listener(backend['listener_id'])

        # create the new backends
        for backend in new_backend_list:
            self._create_listener_pool_member(ns_name, lb, backend)

        return lb

    def _delete_all_listeners(self, lb):
        listener_list = lb.loadbalancer_listeners.copy()
        for ll_id in listener_list:
            ll = LoadbalancerListenerKM.get(ll_id)
            pool_id = ll.loadbalancer_pool
            if pool_id:
                pool = LoadbalancerPoolKM.get(pool_id)
                member_list = pool.members.copy()
                for member_id in member_list:
                    self._vnc_delete_member(member_id)
                    LoadbalancerMemberKM.delete(member_id)
                self._vnc_delete_pool(pool_id)
                LoadbalancerPoolKM.delete(pool_id)
            self._vnc_delete_listener(ll_id)
            LoadbalancerListenerKM.delete(ll_id)

    def _delete_listener(self, ll_id):
        ll = LoadbalancerListenerKM.get(ll_id)
        pool_id = ll.loadbalancer_pool
        if pool_id:
            pool = LoadbalancerPoolKM.get(pool_id)
            member_list = pool.members.copy()
            for member_id in member_list:
                self._vnc_delete_member(member_id)
                LoadbalancerMemberKM.delete(member_id)
            self._vnc_delete_pool(pool_id)
            LoadbalancerPoolKM.delete(pool_id)
        self._vnc_delete_listener(ll_id)
        LoadbalancerListenerKM.delete(ll_id)

    def _delete_lb(self, uid):
        lb = LoadbalancerKM.get(uid)
        if not lb:
            return
        self._delete_all_listeners(lb)
        self._vnc_delete_lb(lb)
        LoadbalancerKM.delete(uid)

    def _update_ingress(self, name, uid, event):
        ns_name = event['object']['metadata'].get('namespace')
        self._create_lb(uid, name, ns_name, event)

    def _delete_ingress(self, uid):
        self._delete_lb(uid)
        self._clear_ingress_cache_uuid(self._ingress_label_cache, uid)

    def _create_ingress_event(self, event_type, ingress_id, lb):
        event = {}
        object = {}
        object['kind'] = 'Ingress'
        object['spec'] = {}
        object['metadata'] = {}
        object['metadata']['uid'] = ingress_id
        if event_type == 'delete':
            event['type'] = 'DELETED'
            event['object'] = object
            self._queue.put(event)
        return

    def _sync_ingress_lb(self):
        lb_uuid_set = set(LoadbalancerKM.keys())
        ingress_uuid_set = set(IngressKM.keys())
        deleted_ingress_set = lb_uuid_set - ingress_uuid_set
        for uuid in deleted_ingress_set:
            lb = LoadbalancerKM.get(uuid)
            if not lb:
                continue
            if not lb.annotations:
                continue
            owner = None
            kind = None
            for kvp in lb.annotations['key_value_pair'] or []:
                if kvp['key'] == 'owner':
                    owner = kvp['value']
                elif kvp['key'] == 'kind':
                    kind = kvp['value']
                if owner == 'k8s' and kind == self._k8s_event_type:
                    self._create_ingress_event('delete', uuid, lb)
                    break
        return

    def ingress_timer(self):
        self._sync_ingress_lb()

    def process(self, event):
        event_type = event['type']
        kind = event['object'].get('kind')
        ns_name = event['object']['metadata'].get('namespace')
        name = event['object']['metadata'].get('name')
        uid = event['object']['metadata'].get('uid')

        print("%s - Got %s %s %s:%s:%s"
              %(self._name, event_type, kind, ns_name, name, uid))
        self._logger.debug("%s - Got %s %s %s:%s:%s"
              %(self._name, event_type, kind, ns_name, name, uid))

        if event['type'] == 'ADDED' or event['type'] == 'MODIFIED':
            self._update_ingress(name, uid, event)
        elif event['type'] == 'DELETED':
            self._delete_ingress(uid)