예제 #1
0
class SvcMonitor(object):

    """
    data + methods used/referred to by ssrc and arc greenlets
    """

    def __init__(self, args=None):
        self._args = args

        # create database and logger
        self.db = ServiceMonitorDB(args)

        # initialize discovery client
        self._disc = None
        if self._args.disc_server_ip and self._args.disc_server_port:
            self._disc = client.DiscoveryClient(self._args.disc_server_ip,
                                                self._args.disc_server_port,
                                                client_type='Service Monitor')

        # initialize logger
        self.logger = ServiceMonitorLogger(self.db, self._disc, args)
        self.db.add_logger(self.logger)

        # rotating log file for catchall errors
        self._err_file = '/var/log/contrail/svc-monitor.err'
        self._tmp_file = '/var/log/contrail/svc-monitor.tmp'
        try:
            with open(self._err_file, 'a'):
                pass
            with open(self._tmp_file, 'a'):
                pass
        except IOError:
            self._err_file = './svc-monitor.err'
            self._tmp_file = './svc-monitor.tmp'
        self._svc_err_logger = logging.getLogger('SvcErrLogger')
        self._svc_err_logger.setLevel(logging.ERROR)
        handler = logging.handlers.RotatingFileHandler(
            self._err_file, maxBytes=64*1024, backupCount=2)
        self._svc_err_logger.addHandler(handler)


    def post_init(self, vnc_lib, args=None):
        # api server
        self._vnc_lib = vnc_lib

        # create default analyzer template
        self._create_default_template('analyzer-template', 'analyzer',
                                      flavor='m1.medium',
                                      image_name='analyzer')
        # create default NAT template
        self._create_default_template('nat-template', 'firewall',
                                      svc_mode='in-network-nat',
                                      image_name='analyzer',
                                      flavor='m1.medium')
        # create default netns SNAT template
        self._create_default_template('netns-snat-template', 'source-nat',
                                      svc_mode='in-network-nat',
                                      hypervisor_type='network-namespace',
                                      scaling=True)
        # create default loadbalancer template
        self._create_default_template('haproxy-loadbalancer-template', 'loadbalancer',
                                      image_name='in-network',
                                      hypervisor_type='network-namespace',
                                      scaling=True)

        # load vrouter scheduler
        self.vrouter_scheduler = importutils.import_object(
            self._args.si_netns_scheduler_driver,
            self._vnc_lib,
            self._args)

        # load virtual machine instance manager
        self.vm_manager = importutils.import_object(
            'svc_monitor.instance_manager.VirtualMachineManager',
            self._vnc_lib, self.db, self.logger,
            self.vrouter_scheduler, self._args)

        # load network namespace instance manager
        self.netns_manager = importutils.import_object(
            'svc_monitor.instance_manager.NetworkNamespaceManager',
            self._vnc_lib, self.db, self.logger,
            self.vrouter_scheduler, self._args)


    # create service template
    def _create_default_template(self, st_name, svc_type, svc_mode=None,
                                 hypervisor_type='virtual-machine',
                                 image_name=None, flavor=None, scaling=False):
        domain_name = 'default-domain'
        domain_fq_name = [domain_name]
        st_fq_name = [domain_name, st_name]
        self.logger.log("Creating %s %s hypervisor %s" %
                         (domain_name, st_name, hypervisor_type))

        try:
            st_obj = self._vnc_lib.service_template_read(fq_name=st_fq_name)
            st_uuid = st_obj.uuid
            self.logger.log("%s exists uuid %s" % (st_name, str(st_uuid)))
            return
        except NoIdError:
            domain = self._vnc_lib.domain_read(fq_name=domain_fq_name)
            st_obj = ServiceTemplate(name=st_name, domain_obj=domain)
            st_uuid = self._vnc_lib.service_template_create(st_obj)

        svc_properties = ServiceTemplateType()
        svc_properties.set_service_type(svc_type)
        svc_properties.set_service_mode(svc_mode)
        svc_properties.set_service_virtualization_type(hypervisor_type)
        svc_properties.set_image_name(image_name)
        svc_properties.set_flavor(flavor)
        svc_properties.set_ordered_interfaces(True)
        svc_properties.set_service_scaling(scaling)

        # set interface list
        if svc_type == 'analyzer':
            if_list = [['left', False]]
        elif hypervisor_type == 'network-namespace':
            if_list = [['left', False], ['right', False]]
        else:
            if_list = [
                ['management', False], ['left', False], ['right', False]]

        for itf in if_list:
            if_type = ServiceTemplateInterfaceType(shared_ip=itf[1])
            if_type.set_service_interface_type(itf[0])
            svc_properties.add_interface_type(if_type)

        try:
            st_obj.set_service_template_properties(svc_properties)
            self._vnc_lib.service_template_update(st_obj)
        except Exception as e:
            print e

        self.logger.log("%s created with uuid %s" % (st_name, str(st_uuid)))
    #_create_default_analyzer_template

    def cleanup(self):
        # TODO cleanup sandesh context
        pass
    # end cleanup

    def _get_proj_name_from_si_fq_str(self, si_fq_str):
        return si_fq_str.split(':')[1]
    # enf _get_si_fq_str_to_proj_name

    def _get_virtualization_type(self, st_props):
        return st_props.get_service_virtualization_type() or 'virtual-machine'
    # end _get_virtualization_type

    def _check_svc_vm_exists(self, si_obj):
        vm_back_refs = si_obj.get_virtual_machine_back_refs()
        if not vm_back_refs:
            return False
        si_props = si_obj.get_service_instance_properties()
        if si_props.get_scale_out():
            max_instances = si_props.get_scale_out().get_max_instances()
        else:
            max_instances = 1
        if max_instances == len(vm_back_refs):
            return True
        return False
    #end _check_svc_vm_exists

    def _check_store_si_info(self, st_obj, si_obj):
        config_complete = True
        st_props = st_obj.get_service_template_properties()
        st_if_list = st_props.get_interface_type()
        si_props = si_obj.get_service_instance_properties()
        si_if_list = si_props.get_interface_list()
        if si_if_list and (len(si_if_list) != len(st_if_list)):
            self.logger.log("Error: IF mismatch template %s instance %s" %
                             (len(st_if_list), len(si_if_list)))
            return

        # read existing si_entry
        si_entry = self.db.service_instance_get(si_obj.get_fq_name_str())
        if not si_entry:
            si_entry = {}
        si_entry['instance_type'] = self._get_virtualization_type(st_props)

        # walk the interface list
        for idx in range(0, len(st_if_list)):
            st_if = st_if_list[idx]
            itf_type = st_if.service_interface_type

            si_if = None
            if si_if_list and st_props.get_ordered_interfaces():
                si_if = si_if_list[idx]
                si_vn_str = si_if.get_virtual_network()
            else:
                funcname = "get_" + itf_type + "_virtual_network"
                func = getattr(si_props, funcname)
                si_vn_str = func()

            if not si_vn_str:
                continue

            si_entry[itf_type + '-vn'] = si_vn_str
            try:
                vn_obj = self._vnc_lib.virtual_network_read(
                    fq_name_str=si_vn_str)
                if vn_obj.uuid != si_entry.get(si_vn_str, None):
                    si_entry[si_vn_str] = vn_obj.uuid
            except NoIdError:
                self.logger.log("Warn: VN %s add is pending" % si_vn_str)
                si_entry[si_vn_str] = 'pending'
                config_complete = False

        if config_complete:
            self.logger.log("SI %s info is complete" %
                             si_obj.get_fq_name_str())
        else:
            self.logger.log("Warn: SI %s info is not complete" %
                             si_obj.get_fq_name_str())

        #insert entry
        self.db.service_instance_insert(si_obj.get_fq_name_str(), si_entry)
        return config_complete
    #end _check_store_si_info

    def _restart_svc(self, vm_uuid, si_fq_str):
        proj_name = self._get_proj_name_from_si_fq_str(si_fq_str)
        self._delete_svc_instance(vm_uuid, proj_name, si_fq_str=si_fq_str)

        # Wait the clean phase was completely done before recreate the SI
        gevent.sleep(_CHECK_CLEANUP_INTERVAL + 1)

        si_obj = self._vnc_lib.service_instance_read(fq_name_str=si_fq_str)
        st_list = si_obj.get_service_template_refs()
        if st_list is not None:
            fq_name = st_list[0]['to']
            st_obj = self._vnc_lib.service_template_read(fq_name=fq_name)
            self._create_svc_instance(st_obj, si_obj)
    # end _restart_svc

    def _delete_shared_vn(self, vn_uuid, proj_name):
        try:
            self.logger.log("Deleting VN %s %s" % (proj_name, vn_uuid))
            self._vnc_lib.virtual_network_delete(id=vn_uuid)
        except RefsExistError:
            self._svc_err_logger.error("Delete failed refs exist VN %s %s" %
                                       (proj_name, vn_uuid))
        except NoIdError:
            # remove from cleanup list
            self.db.cleanup_table_remove(vn_uuid)
    # end _delete_shared_vn

    def _create_svc_instance(self, st_obj, si_obj):
        #check if all config received before launch
        if not self._check_store_si_info(st_obj, si_obj):
            return

        #check if VMs already exist
        if self._check_svc_vm_exists(si_obj):
            return

        st_props = st_obj.get_service_template_properties()
        if st_props is None:
            self.logger.log("Cannot find service template associated to "
                             "service instance %s" % si_obj.get_fq_name_str())
        virt_type = self._get_virtualization_type(st_props)
        if virt_type == 'virtual-machine':
            self.vm_manager.create_service(st_obj, si_obj)
        elif virt_type == 'network-namespace':
            self.netns_manager.create_service(st_obj, si_obj)

    def _delete_svc_instance(self, vm_uuid, proj_name, si_fq_str=None):
        vm_entry = self.db.virtual_machine_get(vm_uuid)
        if not vm_entry: 
            self.db.cleanup_table_remove(vm_uuid)
            return

        self.logger.log("Deleting VM %s %s" % (proj_name, vm_uuid))
        virt_type = vm_entry['instance_type']
        if virt_type == 'virtual-machine':
            self.vm_manager.delete_service(vm_uuid, proj_name)
        elif virt_type == 'network-namespace':
            self.netns_manager.delete_service(vm_uuid)

        # remove from launch table and queue into cleanup list
        self.db.virtual_machine_remove(vm_uuid)
        self.db.cleanup_table_insert(
            vm_uuid, {'proj_name': proj_name, 'type': 'vm'})
        self.logger.uve_svc_instance(si_fq_str, status='DELETE', vm_uuid=vm_uuid)

    def _delmsg_project_service_instance(self, idents):
        proj_fq_str = idents['project']
        si_fq_str = idents['service-instance']

        si_info = self.db.service_instance_get(si_fq_str)
        if not si_info:
            return

        virt_type = si_info['instance_type']
        if virt_type == 'virtual-machine':
            self.vm_manager.clean_resources(proj_fq_str, si_fq_str)
        elif virt_type == 'network-namespace':
            self.netns_manager.clean_resources(proj_fq_str, si_fq_str)

        #delete si info
        self.db.service_instance_remove(si_fq_str)
    # end _delmsg_project_service_instance

    def _delmsg_service_instance_service_template(self, idents):
        si_fq_str = idents['service-instance']
        si_info = self.db.service_instance_get(si_fq_str)
        if not si_info:
            return

        vm_list = self.db.virtual_machine_list()
        for vm_uuid, si in vm_list:
            if si_fq_str != si['si_fq_str']:
                continue

            proj_name = self._get_proj_name_from_si_fq_str(si_fq_str)
            self._delete_svc_instance(vm_uuid, proj_name, si_fq_str=si_fq_str)

            #insert shared instance IP uuids into cleanup list if present
            for itf_str in svc_info.get_if_str_list():
                iip_uuid_str = itf_str + '-iip-uuid'
                if not iip_uuid_str in si_info:
                    continue
                self.db.cleanup_table_insert(
                    si_info[iip_uuid_str], {'proj_name': proj_name,
                                            'type': 'iip'})
    #end _delmsg_service_instance_service_template

    def _delmsg_virtual_machine_service_instance(self, idents):
        vm_uuid = idents['virtual-machine']
        si_fq_str = idents['service-instance']
        proj_name = self._get_proj_name_from_si_fq_str(si_fq_str)
        vm_list = self.db.virtual_machine_list()
        for vm_uuid, si in vm_list:
            if si_fq_str != si['si_fq_str']:
                continue

            self._delete_svc_instance(vm_uuid, proj_name, si_fq_str=si_fq_str)
    # end _delmsg_service_instance_virtual_machine

    def _delmsg_virtual_machine_interface_route_table(self, idents):
        rt_fq_str = idents['interface-route-table']

        rt_obj = self._vnc_lib.interface_route_table_read(
            fq_name_str=rt_fq_str)
        vmi_list = rt_obj.get_virtual_machine_interface_back_refs()
        if vmi_list is None:
            self._vnc_lib.interface_route_table_delete(id=rt_obj.uuid)
    # end _delmsg_virtual_machine_interface_route_table

    def _addmsg_virtual_machine_interface_virtual_machine(self, idents):
        vm_fq_str = idents['virtual-machine']

        try:
            vm_obj = self._vnc_lib.virtual_machine_read(
                fq_name_str=vm_fq_str)
        except NoIdError:
            return

        # check if this is a service vm
        si_list = vm_obj.get_service_instance_refs()
        if not si_list:
            return

        vm_info = self.db.virtual_machine_get(vm_obj.uuid)
        if not vm_info:
            return
        si_fq_str = vm_info['si_fq_str']

        try:
            si_obj = self._vnc_lib.service_instance_read(
                fq_name_str=si_fq_str)
        except NoIdError:
            proj_name = self._get_proj_name_from_si_fq_str(si_fq_str)
            self._delete_svc_instance(vm_obj.uuid, proj_name,
                                      si_fq_str=si_fq_str)
            return

        # create service instance to service vm link
        vm_obj.add_service_instance(si_obj)
        self._vnc_lib.virtual_machine_update(vm_obj)
        self.logger.log("Info: VM %s updated SI %s" %
                         (vm_obj.get_fq_name_str(), si_fq_str))
    # end _addmsg_virtual_machine_interface_virtual_machine

    def _addmsg_service_instance_service_template(self, idents):
        st_fq_str = idents['service-template']
        si_fq_str = idents['service-instance']

        try:
            st_obj = self._vnc_lib.service_template_read(
                fq_name_str=st_fq_str)
            si_obj = self._vnc_lib.service_instance_read(
                fq_name_str=si_fq_str)
        except NoIdError:
            return

        #launch VMs
        self._create_svc_instance(st_obj, si_obj)
    # end _addmsg_service_instance_service_template

    def _addmsg_service_instance_properties(self, idents):
        si_fq_str = idents['service-instance']

        try:
            si_obj = self._vnc_lib.service_instance_read(
                fq_name_str=si_fq_str)
        except NoIdError:
            return

        #update static routes
        self._update_static_routes(si_obj)
    # end _addmsg_service_instance_service_template

    def _addmsg_project_virtual_network(self, idents):
        vn_fq_str = idents['virtual-network']

        si_list = self.db.service_instance_list()
        if not si_list:
            return

        for si_fq_str, si_info in si_list:
            if vn_fq_str not in si_info.keys():
                continue

            try:
                si_obj = self._vnc_lib.service_instance_read(
                    fq_name_str=si_fq_str)
                if si_obj.get_virtual_machine_back_refs():
                    continue

                st_refs = si_obj.get_service_template_refs()
                fq_name = st_refs[0]['to']
                st_obj = self._vnc_lib.service_template_read(fq_name=fq_name)

                #launch VMs
                self._create_svc_instance(st_obj, si_obj)
            except Exception:
                continue
    #end _addmsg_project_virtual_network

    def process_poll_result(self, poll_result_str):
        result_list = parse_poll_result(poll_result_str)

        # process ifmap message
        for (result_type, idents, metas) in result_list:
            if 'ERROR' in idents.values():
                continue
            for meta in metas:
                meta_name = re.sub('{.*}', '', meta.tag)
                if result_type == 'deleteResult':
                    funcname = "_delmsg_" + meta_name.replace('-', '_')
                elif result_type in ['searchResult', 'updateResult']:
                    funcname = "_addmsg_" + meta_name.replace('-', '_')
                # end if result_type
                try:
                    func = getattr(self, funcname)
                except AttributeError:
                    pass
                else:
                    self.logger.log("%s with %s/%s"
                                     % (funcname, meta_name, idents))
                    func(idents)
            # end for meta
        # end for result_type
    # end process_poll_result

    def _update_static_routes(self, si_obj):
        # get service instance interface list
        si_props = si_obj.get_service_instance_properties()
        si_if_list = si_props.get_interface_list()
        if not si_if_list:
            return

        for idx in range(0, len(si_if_list)):
            si_if = si_if_list[idx]
            static_routes = si_if.get_static_routes()
            if not static_routes:
                static_routes = {'route':[]}

            # update static routes
            try:
                domain_name, proj_name = si_obj.get_parent_fq_name()
                rt_name = si_obj.uuid + ' ' + str(idx)
                rt_fq_name = [domain_name, proj_name, rt_name]
                rt_obj = self._vnc_lib.interface_route_table_read(
                    fq_name=rt_fq_name)
                rt_obj.set_interface_route_table_routes(static_routes)
                self._vnc_lib.interface_route_table_update(rt_obj)
            except NoIdError:
                pass
class SvcMonitor(object):

    """
    data + methods used/referred to by ssrc and arc greenlets
    """

    def __init__(self, args=None):
        self._args = args

        # create database and logger
        self.db = ServiceMonitorDB(args)

        # initialize discovery client
        self._disc = None
        if self._args.disc_server_ip and self._args.disc_server_port:
            self._disc = client.DiscoveryClient(
                self._args.disc_server_ip, self._args.disc_server_port, client_type="Service Monitor"
            )

        # initialize logger
        self.logger = ServiceMonitorLogger(self.db, self._disc, args)
        self.db.add_logger(self.logger)

        # rotating log file for catchall errors
        self._err_file = "/var/log/contrail/svc-monitor.err"
        self._tmp_file = "/var/log/contrail/svc-monitor.tmp"
        try:
            with open(self._err_file, "a"):
                pass
            with open(self._tmp_file, "a"):
                pass
        except IOError:
            self._err_file = "./svc-monitor.err"
            self._tmp_file = "./svc-monitor.tmp"
        self._svc_err_logger = logging.getLogger("SvcErrLogger")
        self._svc_err_logger.setLevel(logging.ERROR)
        handler = logging.handlers.RotatingFileHandler(self._err_file, maxBytes=64 * 1024, backupCount=2)
        self._svc_err_logger.addHandler(handler)

    def post_init(self, vnc_lib, args=None):
        # api server
        self._vnc_lib = vnc_lib

        # create default analyzer template
        self._create_default_template("analyzer-template", "analyzer", flavor="m1.medium", image_name="analyzer")
        # create default NAT template
        self._create_default_template(
            "nat-template", "firewall", svc_mode="in-network-nat", image_name="analyzer", flavor="m1.medium"
        )
        # create default netns SNAT template
        self._create_default_template(
            "netns-snat-template",
            "source-nat",
            svc_mode="in-network-nat",
            hypervisor_type="network-namespace",
            scaling=True,
        )
        # create default loadbalancer template
        self._create_default_template(
            "haproxy-loadbalancer-template",
            "loadbalancer",
            svc_mode="in-network-nat",
            hypervisor_type="network-namespace",
            scaling=True,
        )

        # load vrouter scheduler
        self.vrouter_scheduler = importutils.import_object(
            self._args.si_netns_scheduler_driver, self._vnc_lib, self._args
        )

        # load virtual machine instance manager
        self.vm_manager = importutils.import_object(
            "svc_monitor.instance_manager.VirtualMachineManager",
            self._vnc_lib,
            self.db,
            self.logger,
            self.vrouter_scheduler,
            self._args,
        )

        # load network namespace instance manager
        self.netns_manager = importutils.import_object(
            "svc_monitor.instance_manager.NetworkNamespaceManager",
            self._vnc_lib,
            self.db,
            self.logger,
            self.vrouter_scheduler,
            self._args,
        )

    # create service template
    def _create_default_template(
        self,
        st_name,
        svc_type,
        svc_mode=None,
        hypervisor_type="virtual-machine",
        image_name=None,
        flavor=None,
        scaling=False,
    ):
        domain_name = "default-domain"
        domain_fq_name = [domain_name]
        st_fq_name = [domain_name, st_name]
        self.logger.log("Creating %s %s hypervisor %s" % (domain_name, st_name, hypervisor_type))

        try:
            st_obj = self._vnc_lib.service_template_read(fq_name=st_fq_name)
            st_uuid = st_obj.uuid
            self.logger.log("%s exists uuid %s" % (st_name, str(st_uuid)))
            return
        except NoIdError:
            domain = self._vnc_lib.domain_read(fq_name=domain_fq_name)
            st_obj = ServiceTemplate(name=st_name, domain_obj=domain)
            st_uuid = self._vnc_lib.service_template_create(st_obj)

        svc_properties = ServiceTemplateType()
        svc_properties.set_service_type(svc_type)
        svc_properties.set_service_mode(svc_mode)
        svc_properties.set_service_virtualization_type(hypervisor_type)
        svc_properties.set_image_name(image_name)
        svc_properties.set_flavor(flavor)
        svc_properties.set_ordered_interfaces(True)
        svc_properties.set_service_scaling(scaling)

        # set interface list
        if svc_type == "analyzer":
            if_list = [["left", False]]
        elif hypervisor_type == "network-namespace":
            if_list = [["right", True], ["left", True]]
        else:
            if_list = [["management", False], ["left", False], ["right", False]]

        for itf in if_list:
            if_type = ServiceTemplateInterfaceType(shared_ip=itf[1])
            if_type.set_service_interface_type(itf[0])
            svc_properties.add_interface_type(if_type)

        try:
            st_obj.set_service_template_properties(svc_properties)
            self._vnc_lib.service_template_update(st_obj)
        except Exception as e:
            print e

        self.logger.log("%s created with uuid %s" % (st_name, str(st_uuid)))

    # _create_default_analyzer_template

    def cleanup(self):
        # TODO cleanup sandesh context
        pass

    # end cleanup

    def _get_proj_name_from_si_fq_str(self, si_fq_str):
        return si_fq_str.split(":")[1]

    # enf _get_si_fq_str_to_proj_name

    def _get_virtualization_type(self, st_props):
        return st_props.get_service_virtualization_type() or "virtual-machine"

    # end _get_virtualization_type

    def _check_store_si_info(self, st_obj, si_obj):
        config_complete = True
        st_props = st_obj.get_service_template_properties()
        st_if_list = st_props.get_interface_type()
        si_props = si_obj.get_service_instance_properties()
        si_if_list = si_props.get_interface_list()
        # for lb relax the check because vip and pool could be in same net
        if (
            (st_props.get_service_type() != svc_info.get_lb_service_type())
            and si_if_list
            and (len(si_if_list) != len(st_if_list))
        ):
            self.logger.log("Error: IF mismatch template %s instance %s" % (len(st_if_list), len(si_if_list)))
            return

        # read existing si_entry
        si_entry = self.db.service_instance_get(si_obj.get_fq_name_str())
        if not si_entry:
            si_entry = {}
        si_entry["instance_type"] = self._get_virtualization_type(st_props)
        si_entry["uuid"] = si_obj.uuid

        # walk the interface list
        for idx in range(0, len(st_if_list)):
            st_if = st_if_list[idx]
            itf_type = st_if.service_interface_type

            si_if = None
            if si_if_list and st_props.get_ordered_interfaces():
                try:
                    si_if = si_if_list[idx]
                except IndexError:
                    continue
                si_vn_str = si_if.get_virtual_network()
            else:
                funcname = "get_" + itf_type + "_virtual_network"
                func = getattr(si_props, funcname)
                si_vn_str = func()

            if not si_vn_str:
                continue

            si_entry[itf_type + "-vn"] = si_vn_str
            try:
                vn_obj = self._vnc_lib.virtual_network_read(fq_name_str=si_vn_str)
                if vn_obj.uuid != si_entry.get(si_vn_str, None):
                    si_entry[si_vn_str] = vn_obj.uuid
            except NoIdError:
                self.logger.log("Warn: VN %s add is pending" % si_vn_str)
                si_entry[si_vn_str] = "pending"
                config_complete = False

        if config_complete:
            self.logger.log("SI %s info is complete" % si_obj.get_fq_name_str())
        else:
            self.logger.log("Warn: SI %s info is not complete" % si_obj.get_fq_name_str())

        # insert entry
        self.db.service_instance_insert(si_obj.get_fq_name_str(), si_entry)
        return config_complete

    # end _check_store_si_info

    def _restart_svc(self, si_fq_str):
        si_obj = self._vnc_lib.service_instance_read(fq_name_str=si_fq_str)
        st_list = si_obj.get_service_template_refs()
        if st_list is not None:
            fq_name = st_list[0]["to"]
            st_obj = self._vnc_lib.service_template_read(fq_name=fq_name)
            self._create_svc_instance(st_obj, si_obj)

    # end _restart_svc

    def _delete_shared_vn(self, vn_uuid, proj_name):
        try:
            self.logger.log("Deleting VN %s %s" % (proj_name, vn_uuid))
            self._vnc_lib.virtual_network_delete(id=vn_uuid)
        except RefsExistError:
            self._svc_err_logger.error("Delete failed refs exist VN %s %s" % (proj_name, vn_uuid))
        except NoIdError:
            # remove from cleanup list
            self.db.cleanup_table_remove(vn_uuid)

    # end _delete_shared_vn

    def _create_svc_instance(self, st_obj, si_obj):
        # check if all config received before launch
        if not self._check_store_si_info(st_obj, si_obj):
            return

        st_props = st_obj.get_service_template_properties()
        if st_props is None:
            self.logger.log(
                "Cannot find service template associated to " "service instance %s" % si_obj.get_fq_name_str()
            )
        virt_type = self._get_virtualization_type(st_props)
        if virt_type == "virtual-machine":
            self.vm_manager.create_service(st_obj, si_obj)
        elif virt_type == "network-namespace":
            self.netns_manager.create_service(st_obj, si_obj)

    def _delete_svc_instance(self, vm_uuid, proj_name, si_fq_str=None, virt_type=None):
        self.logger.log("Deleting VM %s %s" % (proj_name, vm_uuid))

        try:
            if virt_type == svc_info.get_vm_instance_type():
                self.vm_manager.delete_service(vm_uuid, proj_name)
            elif virt_type == svc_info.get_netns_instance_type():
                self.netns_manager.delete_service(vm_uuid)
        except KeyError:
            self.db.cleanup_table_remove(vm_uuid)
            return

        # remove from launch table and queue into cleanup list
        self.db.virtual_machine_remove(vm_uuid)
        self.db.cleanup_table_insert(vm_uuid, {"proj_name": proj_name, "type": virt_type})
        self.logger.uve_svc_instance(si_fq_str, status="DELETE", vm_uuid=vm_uuid)

    def _check_si_status(self, si_fq_name_str, si_info):
        try:
            si_obj = self._vnc_lib.service_instance_read(id=si_info["uuid"])
        except NoIdError:
            self.db.service_instance_remove(si_fq_name_str)
            return "ACTIVE"

        if si_info["instance_type"] == "virtual-machine":
            proj_name = self._get_proj_name_from_si_fq_str(si_fq_name_str)
            status = self.vm_manager.check_service(si_obj, proj_name)
        elif si_info["instance_type"] == "network-namespace":
            status = self.netns_manager.check_service(si_obj)

        return status

    def _check_vm_si_link(self, vm_uuid, si_info):
        try:
            vm_obj = self._vnc_lib.virtual_machine_read(id=vm_uuid)
        except NoIdError:
            return

        si_fq_str = si_info["si_fq_str"]
        si_refs = vm_obj.get_service_instance_refs()
        if (si_refs is None) or (si_refs[0]["to"][0] == "ERROR"):
            proj_name = self._get_proj_name_from_si_fq_str(si_fq_str)
            self._delete_svc_instance(vm_uuid, proj_name, si_fq_str, si_info["instance_type"])

    def _delmsg_project_service_instance(self, idents):
        proj_fq_str = idents["project"]
        si_fq_str = idents["service-instance"]

        si_info = self.db.service_instance_get(si_fq_str)
        if not si_info:
            return

        virt_type = si_info["instance_type"]
        if virt_type == "virtual-machine":
            self.vm_manager.clean_resources(proj_fq_str, si_fq_str)
        elif virt_type == "network-namespace":
            self.netns_manager.clean_resources(proj_fq_str, si_fq_str)

        # delete si info
        self.db.service_instance_remove(si_fq_str)

    # end _delmsg_project_service_instance

    def _delmsg_service_instance_service_template(self, idents):
        si_fq_str = idents["service-instance"]
        si_info = self.db.service_instance_get(si_fq_str)
        if not si_info:
            return

        vm_list = self.db.virtual_machine_list()
        for vm_uuid, si in vm_list:
            if si_fq_str != si["si_fq_str"]:
                continue

            proj_name = self._get_proj_name_from_si_fq_str(si_fq_str)
            self._delete_svc_instance(vm_uuid, proj_name, si_fq_str=si_fq_str, virt_type=si["instance_type"])

            # insert shared instance IP uuids into cleanup list if present
            for itf_str in svc_info.get_if_str_list():
                iip_uuid_str = itf_str + "-iip-uuid"
                if not iip_uuid_str in si_info:
                    continue
                self.db.cleanup_table_insert(si_info[iip_uuid_str], {"proj_name": proj_name, "type": "iip"})

        # delete si info
        self.db.service_instance_remove(si_fq_str)

    # end _delmsg_service_instance_service_template

    def _delmsg_virtual_machine_service_instance(self, idents):
        vm_uuid = idents["virtual-machine"]
        si_fq_str = idents["service-instance"]
        proj_name = self._get_proj_name_from_si_fq_str(si_fq_str)
        vm_info = self.db.virtual_machine_get(vm_uuid)
        if vm_info:
            self._delete_svc_instance(
                vm_uuid, proj_name, si_fq_str=vm_info["si_fq_str"], virt_type=vm_info["instance_type"]
            )
        return

    # end _delmsg_service_instance_virtual_machine

    def _delmsg_virtual_machine_interface_route_table(self, idents):
        rt_fq_str = idents["interface-route-table"]

        rt_obj = self._vnc_lib.interface_route_table_read(fq_name_str=rt_fq_str)
        vmi_list = rt_obj.get_virtual_machine_interface_back_refs()
        if vmi_list is None:
            self._vnc_lib.interface_route_table_delete(id=rt_obj.uuid)

    # end _delmsg_virtual_machine_interface_route_table

    def _addmsg_virtual_machine_interface_virtual_machine(self, idents):
        vm_fq_str = idents["virtual-machine"]

        try:
            vm_obj = self._vnc_lib.virtual_machine_read(fq_name_str=vm_fq_str)
        except NoIdError:
            return

        # check if this is a service vm
        si_list = vm_obj.get_service_instance_refs()
        if si_list:
            return

        vm_info = self.db.virtual_machine_get(vm_obj.uuid)
        if not vm_info:
            return
        si_fq_str = vm_info["si_fq_str"]

        try:
            si_obj = self._vnc_lib.service_instance_read(fq_name_str=si_fq_str)
        except NoIdError:
            proj_name = self._get_proj_name_from_si_fq_str(si_fq_str)
            self._delete_svc_instance(vm_obj.uuid, proj_name, si_fq_str=si_fq_str, virt_type=vm_info["instance_type"])
            return

        # create service instance to service vm link
        vm_obj.add_service_instance(si_obj)
        self._vnc_lib.virtual_machine_update(vm_obj)
        self.logger.log("Info: VM %s updated SI %s" % (vm_obj.get_fq_name_str(), si_fq_str))

    # end _addmsg_virtual_machine_interface_virtual_machine

    def _addmsg_service_instance_service_template(self, idents):
        st_fq_str = idents["service-template"]
        si_fq_str = idents["service-instance"]

        try:
            st_obj = self._vnc_lib.service_template_read(fq_name_str=st_fq_str)
            si_obj = self._vnc_lib.service_instance_read(fq_name_str=si_fq_str)
        except NoIdError:
            return

        # launch VMs
        self._create_svc_instance(st_obj, si_obj)

    # end _addmsg_service_instance_service_template

    def _addmsg_service_instance_properties(self, idents):
        si_fq_str = idents["service-instance"]

        try:
            si_obj = self._vnc_lib.service_instance_read(fq_name_str=si_fq_str)
        except NoIdError:
            return

        # update static routes
        self._update_static_routes(si_obj)

    # end _addmsg_service_instance_service_template

    def _addmsg_project_virtual_network(self, idents):
        vn_fq_str = idents["virtual-network"]

        si_list = self.db.service_instance_list()
        if not si_list:
            return

        for si_fq_str, si_info in si_list:
            if vn_fq_str not in si_info.keys():
                continue

            try:
                si_obj = self._vnc_lib.service_instance_read(fq_name_str=si_fq_str)
                if si_obj.get_virtual_machine_back_refs():
                    continue

                st_refs = si_obj.get_service_template_refs()
                fq_name = st_refs[0]["to"]
                st_obj = self._vnc_lib.service_template_read(fq_name=fq_name)

                # launch VMs
                self._create_svc_instance(st_obj, si_obj)
            except Exception:
                continue

    # end _addmsg_project_virtual_network

    def _addmsg_floating_ip_virtual_machine_interface(self, idents):
        fip_fq_str = idents["floating-ip"]
        vmi_fq_str = idents["virtual-machine-interface"]

        try:
            fip_obj = self._vnc_lib.floating_ip_read(fq_name_str=fip_fq_str)
            vmi_obj = self._vnc_lib.virtual_machine_interface_read(fq_name_str=vmi_fq_str)
        except NoIdError:
            return

        # handle only if VIP back ref exists
        vip_back_refs = vmi_obj.get_virtual_ip_back_refs()
        if vip_back_refs is None:
            return

        # associate fip to all VMIs
        iip_back_refs = vmi_obj.get_instance_ip_back_refs()
        try:
            iip_obj = self._vnc_lib.instance_ip_read(id=iip_back_refs[0]["uuid"])
        except NoIdError:
            return

        fip_updated = False
        vmi_refs_iip = iip_obj.get_virtual_machine_interface_refs()
        vmi_refs_fip = fip_obj.get_virtual_machine_interface_refs()
        for vmi_ref_iip in vmi_refs_iip:
            if vmi_ref_iip in vmi_refs_fip:
                continue
            try:
                vmi_obj = self._vnc_lib.virtual_machine_interface_read(id=vmi_ref_iip["uuid"])
            except NoIdError:
                continue
            fip_obj.add_virtual_machine_interface(vmi_obj)
            fip_updated = True

        if fip_updated:
            self._vnc_lib.floating_ip_update(fip_obj)

    def process_poll_result(self, poll_result_str):
        result_list = parse_poll_result(poll_result_str)

        # process ifmap message
        for (result_type, idents, metas) in result_list:
            if "ERROR" in idents.values():
                continue
            for meta in metas:
                meta_name = re.sub("{.*}", "", meta.tag)
                if result_type == "deleteResult":
                    funcname = "_delmsg_" + meta_name.replace("-", "_")
                elif result_type in ["searchResult", "updateResult"]:
                    funcname = "_addmsg_" + meta_name.replace("-", "_")
                # end if result_type
                try:
                    func = getattr(self, funcname)
                except AttributeError:
                    pass
                else:
                    self.logger.log("%s with %s/%s" % (funcname, meta_name, idents))
                    func(idents)
            # end for meta
        # end for result_type

    # end process_poll_result

    def _update_static_routes(self, si_obj):
        # get service instance interface list
        si_props = si_obj.get_service_instance_properties()
        si_if_list = si_props.get_interface_list()
        if not si_if_list:
            return

        for idx in range(0, len(si_if_list)):
            si_if = si_if_list[idx]
            static_routes = si_if.get_static_routes()
            if not static_routes:
                static_routes = {"route": []}

            # update static routes
            try:
                domain_name, proj_name = si_obj.get_parent_fq_name()
                rt_name = si_obj.uuid + " " + str(idx)
                rt_fq_name = [domain_name, proj_name, rt_name]
                rt_obj = self._vnc_lib.interface_route_table_read(fq_name=rt_fq_name)
                rt_obj.set_interface_route_table_routes(static_routes)
                self._vnc_lib.interface_route_table_update(rt_obj)
            except NoIdError:
                pass